YOLO V5 and V8 are working

Changes to be committed:
    modified:   lib/main.dart

1 files changed, 536 insertions, 4 deletions

diff --git a/lib/main.dart b/lib/main.dart
index b59e921..a8f0620 100644
--- a/lib/main.dart
+++ b/lib/main.dart
@@ -3,7 +3,7 @@ import 'package:permission_handler/permission_handler.dart';
 import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart';
 // import 'package:google_mlkit_object_detection/google_mlkit_object_detection.dart';
 // import 'package:flutter_tflite/flutter_tflite.dart';
-import 'package:flutter_speed_dial/flutter_speed_dial.dart';
+// import 'package:flutter_speed_dial/flutter_speed_dial.dart';
 import 'package:flutter_tts/flutter_tts.dart';
 import 'package:flutter_vision/flutter_vision.dart';
 import 'package:image_picker/image_picker.dart';
@@ -13,8 +13,12 @@ import 'package:flutter/material.dart';
 import 'dart:typed_data';
 import 'dart:async';
 import 'dart:io';
+import 'dart:ui';
 
+late List<CameraDescription> cameras;
 void main() {
+  WidgetsFlutterBinding.ensureInitialized();
+  DartPluginRegistrant.ensureInitialized();
   runApp(const App());
 }
 
@@ -30,6 +34,7 @@ class App extends StatelessWidget {
         useMaterial3: true,
       ),
       debugShowCheckedModeBanner: false,
+      // home: const MainScreen(),
       home: const MainScreen(),
     );
   }
@@ -48,9 +53,9 @@ class _MainScreenState extends State<MainScreen> with WidgetsBindingObserver {
   late final Future<void> _future;
   CameraController? _cameraController;
 
-  late FlutterVision vision;
+  late FlutterVision vision; // YOLO
 
-  final textRecognizer = TextRecognizer();
+  final textRecognizer = TextRecognizer(); // OCR
 
   FlutterTts flutterTts = FlutterTts(); // TTS
 
@@ -244,7 +249,8 @@ class _MainScreenState extends State<MainScreen> with WidgetsBindingObserver {
       await navigator.push(
         MaterialPageRoute(
             builder: (BuildContext context) =>
-                ResultScreen(text: recognizedText.text)
+                //ResultScreen(text: recognizedText.text)
+                YoloVideo(vision: vision)
         ),
       );
     } catch (e) {
@@ -256,4 +262,530 @@ class _MainScreenState extends State<MainScreen> with WidgetsBindingObserver {
       );
     }
   }
+}
+
+// YOLO V5 REAL-TIME OBJECT DETECTION
+
+class YoloVideo extends StatefulWidget {
+  final FlutterVision vision;
+  const YoloVideo({Key? key, required this.vision}) : super(key: key);
+
+  @override
+  State<YoloVideo> createState() => _YoloVideoState();
+}
+
+class _YoloVideoState extends State<YoloVideo> {
+  late CameraController controller;
+  late List<Map<String, dynamic>> yoloResults;
+  CameraImage? cameraImage;
+  bool isLoaded = false;
+  bool isDetecting = false;
+
+  @override
+  void initState() {
+    super.initState();
+    init();
+  }
+
+  init() async {
+    cameras = await availableCameras();
+    controller = CameraController(cameras[0], ResolutionPreset.high);
+    controller.initialize().then((value) {
+      loadYoloModel().then((value) {
+        setState(() {
+          isLoaded = true;
+          isDetecting = false;
+          yoloResults = [];
+        });
+      });
+    });
+  }
+
+  @override
+  void dispose() async {
+    super.dispose();
+    controller.dispose();
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    final Size size = MediaQuery.of(context).size;
+    if (!isLoaded) {
+      return const Scaffold(
+        body: Center(
+          child: Text("Model not loaded, waiting for it"),
+        ),
+      );
+    }
+    return Stack(
+      fit: StackFit.expand,
+      children: [
+        AspectRatio(
+          aspectRatio: controller.value.aspectRatio,
+          child: CameraPreview(
+            controller,
+          ),
+        ),
+        ...displayBoxesAroundRecognizedObjects(size),
+        Positioned(
+          bottom: 75,
+          width: MediaQuery.of(context).size.width,
+          child: Container(
+            height: 80,
+            width: 80,
+            decoration: BoxDecoration(
+              shape: BoxShape.circle,
+              border: Border.all(
+                  width: 5, color: Colors.white, style: BorderStyle.solid),
+            ),
+            child: isDetecting
+                ? IconButton(
+              onPressed: () async {
+                stopDetection();
+              },
+              icon: const Icon(
+                Icons.stop,
+                color: Colors.red,
+              ),
+              iconSize: 50,
+            )
+                : IconButton(
+              onPressed: () async {
+                await startDetection();
+              },
+              icon: const Icon(
+                Icons.play_arrow,
+                color: Colors.white,
+              ),
+              iconSize: 50,
+            ),
+          ),
+        ),
+      ],
+    );
+  }
+
+  Future<void> loadYoloModel() async {
+    await widget.vision.loadYoloModel(
+        labels: 'assets/labels.txt',
+        modelPath: 'assets/yolov5n.tflite',
+        modelVersion: "yolov5",
+        numThreads: 2,
+        useGpu: true);
+    setState(() {
+      isLoaded = true;
+    });
+  }
+
+  Future<void> yoloOnFrame(CameraImage cameraImage) async {
+    final result = await widget.vision.yoloOnFrame(
+        bytesList: cameraImage.planes.map((plane) => plane.bytes).toList(),
+        imageHeight: cameraImage.height,
+        imageWidth: cameraImage.width,
+        iouThreshold: 0.4,
+        confThreshold: 0.4,
+        classThreshold: 0.5);
+    if (result.isNotEmpty) {
+      setState(() {
+        yoloResults = result;
+      });
+    }
+  }
+
+  Future<void> startDetection() async {
+    setState(() {
+      isDetecting = true;
+    });
+    if (controller.value.isStreamingImages) {
+      return;
+    }
+    await controller.startImageStream((image) async {
+      if (isDetecting) {
+        cameraImage = image;
+        yoloOnFrame(image);
+      }
+    });
+  }
+
+  Future<void> stopDetection() async {
+    setState(() {
+      isDetecting = false;
+      yoloResults.clear();
+    });
+  }
+
+  List<Widget> displayBoxesAroundRecognizedObjects(Size screen) {
+    if (yoloResults.isEmpty) return [];
+    double factorX = screen.width / (cameraImage?.height ?? 1);
+    double factorY = screen.height / (cameraImage?.width ?? 1);
+
+    Color colorPick = const Color.fromARGB(255, 50, 233, 30);
+
+    return yoloResults.map((result) {
+      return Positioned(
+        left: result["box"][0] * factorX,
+        top: result["box"][1] * factorY,
+        width: (result["box"][2] - result["box"][0]) * factorX,
+        height: (result["box"][3] - result["box"][1]) * factorY,
+        child: Container(
+          decoration: BoxDecoration(
+            borderRadius: const BorderRadius.all(Radius.circular(10.0)),
+            border: Border.all(color: Colors.pink, width: 2.0),
+          ),
+          child: Text(
+            "${result['tag']} ${(result['box'][4] * 100).toStringAsFixed(0)}%",
+            style: TextStyle(
+              background: Paint()..color = colorPick,
+              color: Colors.white,
+              fontSize: 18.0,
+            ),
+          ),
+        ),
+      );
+    }).toList();
+  }
+}
+
+// YOLO V5 OBJECT DETECTION
+
+class YoloImageV5 extends StatefulWidget {
+  final FlutterVision vision;
+  const YoloImageV5({Key? key, required this.vision}) : super(key: key);
+
+  @override
+  State<YoloImageV5> createState() => _YoloImageV5State();
+}
+
+class _YoloImageV5State extends State<YoloImageV5> {
+  late List<Map<String, dynamic>> yoloResults;
+  File? imageFile;
+  int imageHeight = 1;
+  int imageWidth = 1;
+  bool isLoaded = false;
+
+  @override
+  void initState() {
+    super.initState();
+    loadYoloModel().then((value) {
+      setState(() {
+        yoloResults = [];
+        isLoaded = true;
+      });
+    });
+  }
+
+  @override
+  void dispose() async {
+    super.dispose();
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    final Size size = MediaQuery.of(context).size;
+    if (!isLoaded) {
+      return const Scaffold(
+        body: Center(
+          child: Text("Model not loaded, waiting for it"),
+        ),
+      );
+    }
+    return Stack(
+      fit: StackFit.expand,
+      children: [
+        imageFile != null ? Image.file(imageFile!) : const SizedBox(),
+        Align(
+          alignment: Alignment.bottomCenter,
+          child: Row(
+            mainAxisAlignment: MainAxisAlignment.center,
+            children: [
+              TextButton(
+                onPressed: pickImage,
+                child: const Text("Pick image"),
+              ),
+              ElevatedButton(
+                onPressed: yoloOnImage,
+                child: const Text("Detect"),
+              )
+            ],
+          ),
+        ),
+        ...displayBoxesAroundRecognizedObjects(size),
+      ],
+    );
+  }
+
+  Future<void> loadYoloModel() async {
+    await widget.vision.loadYoloModel(
+        labels: 'assets/labels.txt',
+        modelPath: 'assets/yolov5n.tflite',
+        modelVersion: "yolov5",
+        quantization: false,
+        numThreads: 2,
+        useGpu: true);
+    setState(() {
+      isLoaded = true;
+    });
+  }
+
+  Future<void> pickImage() async {
+    final ImagePicker picker = ImagePicker();
+    // Capture a photo
+    final XFile? photo = await picker.pickImage(source: ImageSource.gallery);
+    if (photo != null) {
+      setState(() {
+        imageFile = File(photo.path);
+      });
+    }
+  }
+
+  yoloOnImage() async {
+    yoloResults.clear();
+    Uint8List byte = await imageFile!.readAsBytes();
+    final image = await decodeImageFromList(byte);
+    imageHeight = image.height;
+    imageWidth = image.width;
+    final result = await widget.vision.yoloOnImage(
+        bytesList: byte,
+        imageHeight: image.height,
+        imageWidth: image.width,
+        iouThreshold: 0.8,
+        confThreshold: 0.4,
+        classThreshold: 0.5);
+    if (result.isNotEmpty) {
+      setState(() {
+        yoloResults = result;
+      });
+    }
+  }
+
+  List<Widget> displayBoxesAroundRecognizedObjects(Size screen) {
+    if (yoloResults.isEmpty) return [];
+
+    double factorX = screen.width / (imageWidth);
+    double imgRatio = imageWidth / imageHeight;
+    double newWidth = imageWidth * factorX;
+    double newHeight = newWidth / imgRatio;
+    double factorY = newHeight / (imageHeight);
+
+    double pady = (screen.height - newHeight) / 2;
+
+    Color colorPick = const Color.fromARGB(255, 50, 233, 30);
+    return yoloResults.map((result) {
+      return Positioned(
+        left: result["box"][0] * factorX,
+        top: result["box"][1] * factorY + pady,
+        width: (result["box"][2] - result["box"][0]) * factorX,
+        height: (result["box"][3] - result["box"][1]) * factorY,
+        child: Container(
+          decoration: BoxDecoration(
+            borderRadius: const BorderRadius.all(Radius.circular(10.0)),
+            border: Border.all(color: Colors.pink, width: 2.0),
+          ),
+          child: Text(
+            "${result['tag']} ${(result['box'][4] * 100).toStringAsFixed(0)}%",
+            style: TextStyle(
+              background: Paint()..color = colorPick,
+              color: Colors.white,
+              fontSize: 18.0,
+            ),
+          ),
+        ),
+      );
+    }).toList();
+  }
+}
+
+// YOLO V8 SEMANTIC SEGMENTATION
+
+class YoloImageV8Seg extends StatefulWidget {
+  final FlutterVision vision;
+  const YoloImageV8Seg({Key? key, required this.vision}) : super(key: key);
+
+  @override
+  State<YoloImageV8Seg> createState() => _YoloImageV8SegState();
+}
+
+class _YoloImageV8SegState extends State<YoloImageV8Seg> {
+  late List<Map<String, dynamic>> yoloResults;
+  File? imageFile;
+  int imageHeight = 1;
+  int imageWidth = 1;
+  bool isLoaded = false;
+
+  @override
+  void initState() {
+    super.initState();
+    loadYoloModel().then((value) {
+      setState(() {
+        yoloResults = [];
+        isLoaded = true;
+      });
+    });
+  }
+
+  @override
+  void dispose() async {
+    super.dispose();
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    final Size size = MediaQuery.of(context).size;
+    if (!isLoaded) {
+      return const Scaffold(
+        body: Center(
+          child: Text("Model not loaded, waiting for it"),
+        ),
+      );
+    }
+    return Stack(
+      fit: StackFit.expand,
+      children: [
+        imageFile != null ? Image.file(imageFile!) : const SizedBox(),
+        Align(
+          alignment: Alignment.bottomCenter,
+          child: Row(
+            mainAxisAlignment: MainAxisAlignment.center,
+            children: [
+              TextButton(
+                onPressed: pickImage,
+                child: const Text("Pick image"),
+              ),
+              ElevatedButton(
+                onPressed: yoloOnImage,
+                child: const Text("Detect"),
+              )
+            ],
+          ),
+        ),
+        ...displayBoxesAroundRecognizedObjects(size),
+      ],
+    );
+  }
+
+  Future<void> loadYoloModel() async {
+    await widget.vision.loadYoloModel(
+        labels: 'assets/labels.txt',
+        modelPath: 'assets/yolov8n-seg.tflite',
+        modelVersion: "yolov8seg",
+        quantization: false,
+        numThreads: 2,
+        useGpu: true);
+    setState(() {
+      isLoaded = true;
+    });
+  }
+
+  Future<void> pickImage() async {
+    final ImagePicker picker = ImagePicker();
+    // Capture a photo
+    final XFile? photo = await picker.pickImage(source: ImageSource.gallery);
+    if (photo != null) {
+      setState(() {
+        imageFile = File(photo.path);
+      });
+    }
+  }
+
+  yoloOnImage() async {
+    yoloResults.clear();
+    Uint8List byte = await imageFile!.readAsBytes();
+    final image = await decodeImageFromList(byte);
+    imageHeight = image.height;
+    imageWidth = image.width;
+    final result = await widget.vision.yoloOnImage(
+        bytesList: byte,
+        imageHeight: image.height,
+        imageWidth: image.width,
+        iouThreshold: 0.8,
+        confThreshold: 0.4,
+        classThreshold: 0.5);
+    if (result.isNotEmpty) {
+      setState(() {
+        yoloResults = result;
+      });
+    }
+  }
+
+  List<Widget> displayBoxesAroundRecognizedObjects(Size screen) {
+    if (yoloResults.isEmpty) return [];
+
+    double factorX = screen.width / (imageWidth);
+    double imgRatio = imageWidth / imageHeight;
+    double newWidth = imageWidth * factorX;
+    double newHeight = newWidth / imgRatio;
+    double factorY = newHeight / (imageHeight);
+
+    double pady = (screen.height - newHeight) / 2;
+
+    Color colorPick = const Color.fromARGB(255, 50, 233, 30);
+    return yoloResults.map((result) {
+      return Stack(children: [
+        Positioned(
+          left: result["box"][0] * factorX,
+          top: result["box"][1] * factorY + pady,
+          width: (result["box"][2] - result["box"][0]) * factorX,
+          height: (result["box"][3] - result["box"][1]) * factorY,
+          child: Container(
+            decoration: BoxDecoration(
+              borderRadius: const BorderRadius.all(Radius.circular(10.0)),
+              border: Border.all(color: Colors.pink, width: 2.0),
+            ),
+            child: Text(
+              "${result['tag']} ${(result['box'][4] * 100).toStringAsFixed(0)}%",
+              style: TextStyle(
+                background: Paint()..color = colorPick,
+                color: Colors.white,
+                fontSize: 18.0,
+              ),
+            ),
+          ),
+        ),
+        Positioned(
+            left: result["box"][0] * factorX,
+            top: result["box"][1] * factorY + pady,
+            width: (result["box"][2] - result["box"][0]) * factorX,
+            height: (result["box"][3] - result["box"][1]) * factorY,
+            child: CustomPaint(
+              painter: PolygonPainter(
+                  points: (result["polygons"] as List<dynamic>).map((e) {
+                    Map<String, double> xy = Map<String, double>.from(e);
+                    xy['x'] = (xy['x'] as double) * factorX;
+                    xy['y'] = (xy['y'] as double) * factorY;
+                    return xy;
+                  }).toList()),
+            )),
+      ]);
+    }).toList();
+  }
+}
+
+class PolygonPainter extends CustomPainter {
+  final List<Map<String, double>> points;
+
+  PolygonPainter({required this.points});
+
+  @override
+  void paint(Canvas canvas, Size size) {
+    final paint = Paint()
+      ..color = const Color.fromARGB(129, 255, 2, 124)
+      ..strokeWidth = 2
+      ..style = PaintingStyle.fill;
+
+    final path = Path();
+    if (points.isNotEmpty) {
+      path.moveTo(points[0]['x']!, points[0]['y']!);
+      for (var i = 1; i < points.length; i++) {
+        path.lineTo(points[i]['x']!, points[i]['y']!);
+      }
+      path.close();
+    }
+
+    canvas.drawPath(path, paint);
+  }
+
+  @override
+  bool shouldRepaint(CustomPainter oldDelegate) {
+    return false;
+  }
 }
\ No newline at end of file