import 'package:camera/camera.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; // import 'package:google_mlkit_object_detection/google_mlkit_object_detection.dart'; // import 'package:flutter_tflite/flutter_tflite.dart'; // import 'package:flutter_speed_dial/flutter_speed_dial.dart'; import 'package:flutter_tts/flutter_tts.dart'; import 'package:flutter_vision/flutter_vision.dart'; import 'package:image_picker/image_picker.dart'; // import 'package:yolo/result_screen.dart'; import 'package:flutter/material.dart'; import 'dart:typed_data'; import 'dart:async'; import 'dart:io'; import 'dart:ui'; late List cameras; void main() { WidgetsFlutterBinding.ensureInitialized(); DartPluginRegistrant.ensureInitialized(); runApp(const App()); } class App extends StatelessWidget { const App({super.key}); @override Widget build(BuildContext context) { return MaterialApp( title: 'MegaView', theme: ThemeData( colorScheme: ColorScheme.fromSeed(seedColor: Colors.deepPurpleAccent), useMaterial3: true, ), debugShowCheckedModeBanner: false, // home: const MainScreen(), home: const MainScreen(), ); } } class MainScreen extends StatefulWidget { const MainScreen({super.key}); @override State createState() => _MainScreenState(); } class _MainScreenState extends State with WidgetsBindingObserver { bool _isPermissionGranted = false; late final Future _future; CameraController? _cameraController; late FlutterVision vision; // YOLO final textRecognizer = TextRecognizer(); // OCR FlutterTts flutterTts = FlutterTts(); // TTS @override void initState() { super.initState(); vision = FlutterVision(); // YOLO initTTS(); // TTS WidgetsBinding.instance.addObserver(this); _future = _requestCameraPermission(); } Future initTTS() async { // TTS await flutterTts.setLanguage("en-US"); // Set the language you want await flutterTts.setSpeechRate(0.5); // Adjust speech rate (1.0 is normal but too fast for my liking) await flutterTts.setVolume(1.0); // Adjust volume (0.0 to 1.0) await flutterTts.setPitch(1.0); // Adjust pitch (1.0 is normal) // You can set other configurations as well // Check if TTS is available // bool isAvailable = await flutterTts.isLanguageAvailable("en-US"); // print("TTS is available: $isAvailable"); } Future speak(String text) async { await flutterTts.speak(text); // TTS } @override void dispose() { WidgetsBinding.instance.removeObserver(this); _stopCamera(); textRecognizer.close(); // OCR Stop flutterTts.stop(); // TTS Stop vision.closeYoloModel(); // YOLO Stop super.dispose(); } @override void didChangeAppLifecycleState(AppLifecycleState state) { if (_cameraController == null || !_cameraController!.value.isInitialized) { return; } if (state == AppLifecycleState.inactive) { _stopCamera(); } else if (state == AppLifecycleState.resumed && _cameraController != null && _cameraController!.value.isInitialized) { _startCamera(); } } @override Widget build(BuildContext context) { return FutureBuilder( future: _future, builder: (context, snapshot) { return Stack( children: [ if (_isPermissionGranted) FutureBuilder>( future: availableCameras(), builder: (context, snapshot) { if (snapshot.hasData) { _initCameraController(snapshot.data!); return Center(child: CameraPreview(_cameraController!)); } else { return const LinearProgressIndicator(); } }, ), Scaffold( appBar: AppBar( backgroundColor: Theme.of(context).colorScheme.inversePrimary, title: const Text('MegaView'), centerTitle: true, ), backgroundColor: _isPermissionGranted ? Colors.transparent : null, body: _isPermissionGranted ? Column( mainAxisAlignment: MainAxisAlignment.end, children: [ Container(), Container( color: Theme.of(context).colorScheme.inversePrimary, alignment: Alignment.center, child: Padding( padding: const EdgeInsets.symmetric(vertical: 8.0), child: ElevatedButton( onPressed: _scanImage, style: ButtonStyle( minimumSize: MaterialStateProperty.all( const Size(256, 64), // Set the desired width and height ), ), child: const Text('Scan'), ), ), ), ], ) : Center( child: Container( padding: const EdgeInsets.only(left: 24.0, right: 24.0), child: const Text( 'Camera permission denied', textAlign: TextAlign.center, ), ), ), ), ], ); }, ); } Future _requestCameraPermission() async { final status = await Permission.camera.request(); _isPermissionGranted = status == PermissionStatus.granted; } void _startCamera() { if (_cameraController != null) { _cameraSelected(_cameraController!.description); } } void _stopCamera() { if (_cameraController != null) { _cameraController?.dispose(); } } void _initCameraController(List cameras) { if (_cameraController != null) { return; } // Select the first rear camera. CameraDescription? camera; for (var i = 0; i < cameras.length; i++) { final CameraDescription current = cameras[i]; if (current.lensDirection == CameraLensDirection.back) { camera = current; break; } } if (camera != null) { _cameraSelected(camera); } } Future _cameraSelected(CameraDescription camera) async { _cameraController = CameraController( camera, ResolutionPreset.high, enableAudio: false, ); await _cameraController!.initialize(); await _cameraController!.setFlashMode(FlashMode.off); if (!mounted) { return; } setState(() {}); } Future _scanImage() async { if (_cameraController == null) return; final navigator = Navigator.of(context); final pictureFile = await _cameraController!.takePicture(); final file = File(pictureFile.path); final inputImage = InputImage.fromFile(file); final recognizedText = await textRecognizer.processImage(inputImage); speak(recognizedText.text); await navigator.push( MaterialPageRoute( builder: (BuildContext context) => //ResultScreen(text: recognizedText.text) YoloVideo(vision: vision) ), ); } } // YOLO V5 REAL-TIME OBJECT DETECTION class YoloVideo extends StatefulWidget { final FlutterVision vision; const YoloVideo({Key? key, required this.vision}) : super(key: key); @override State createState() => _YoloVideoState(); } class _YoloVideoState extends State { late CameraController controller; late List> yoloResults; CameraImage? cameraImage; bool isLoaded = false; bool isDetecting = false; @override void initState() { super.initState(); init(); } init() async { cameras = await availableCameras(); controller = CameraController(cameras[0], ResolutionPreset.high); controller.initialize().then((value) { loadYoloModel().then((value) { setState(() { isLoaded = true; isDetecting = false; yoloResults = []; }); }); }); } @override void dispose() async { super.dispose(); controller.dispose(); } @override Widget build(BuildContext context) { final Size size = MediaQuery.of(context).size; if (!isLoaded) { return const Scaffold( body: Center( child: Text("Model not loaded, waiting for it"), ), ); } return Stack( fit: StackFit.expand, children: [ AspectRatio( aspectRatio: controller.value.aspectRatio, child: CameraPreview( controller, ), ), ...displayBoxesAroundRecognizedObjects(size), Positioned( bottom: 75, width: MediaQuery.of(context).size.width, child: Container( height: 80, width: 80, decoration: BoxDecoration( shape: BoxShape.circle, border: Border.all( width: 5, color: Colors.white, style: BorderStyle.solid), ), child: isDetecting ? IconButton( onPressed: () async { stopDetection(); }, icon: const Icon( Icons.stop, color: Colors.red, ), iconSize: 50, ) : IconButton( onPressed: () async { await startDetection(); }, icon: const Icon( Icons.play_arrow, color: Colors.white, ), iconSize: 50, ), ), ), ], ); } Future loadYoloModel() async { await widget.vision.loadYoloModel( labels: 'assets/labels.txt', modelPath: 'assets/yolov5n.tflite', modelVersion: "yolov5", numThreads: 2, useGpu: true); setState(() { isLoaded = true; }); } Future yoloOnFrame(CameraImage cameraImage) async { final result = await widget.vision.yoloOnFrame( bytesList: cameraImage.planes.map((plane) => plane.bytes).toList(), imageHeight: cameraImage.height, imageWidth: cameraImage.width, iouThreshold: 0.4, confThreshold: 0.4, classThreshold: 0.5); if (result.isNotEmpty) { setState(() { yoloResults = result; }); } } Future startDetection() async { setState(() { isDetecting = true; }); if (controller.value.isStreamingImages) { return; } await controller.startImageStream((image) async { if (isDetecting) { cameraImage = image; yoloOnFrame(image); } }); } Future stopDetection() async { setState(() { isDetecting = false; yoloResults.clear(); }); } List displayBoxesAroundRecognizedObjects(Size screen) { if (yoloResults.isEmpty) return []; double factorX = screen.width / (cameraImage?.height ?? 1); double factorY = screen.height / (cameraImage?.width ?? 1); Color colorPick = const Color.fromARGB(255, 50, 233, 30); return yoloResults.map((result) { return Positioned( left: result["box"][0] * factorX, top: result["box"][1] * factorY, width: (result["box"][2] - result["box"][0]) * factorX, height: (result["box"][3] - result["box"][1]) * factorY, child: Container( decoration: BoxDecoration( borderRadius: const BorderRadius.all(Radius.circular(10.0)), border: Border.all(color: Colors.pink, width: 2.0), ), child: Text( "${result['tag']} ${(result['box'][4] * 100).toStringAsFixed(0)}%", style: TextStyle( background: Paint()..color = colorPick, color: Colors.white, fontSize: 18.0, ), ), ), ); }).toList(); } } // YOLO V5 OBJECT DETECTION class YoloImageV5 extends StatefulWidget { final FlutterVision vision; const YoloImageV5({Key? key, required this.vision}) : super(key: key); @override State createState() => _YoloImageV5State(); } class _YoloImageV5State extends State { late List> yoloResults; File? imageFile; int imageHeight = 1; int imageWidth = 1; bool isLoaded = false; @override void initState() { super.initState(); loadYoloModel().then((value) { setState(() { yoloResults = []; isLoaded = true; }); }); } @override void dispose() async { super.dispose(); } @override Widget build(BuildContext context) { final Size size = MediaQuery.of(context).size; if (!isLoaded) { return const Scaffold( body: Center( child: Text("Model not loaded, waiting for it"), ), ); } return Stack( fit: StackFit.expand, children: [ imageFile != null ? Image.file(imageFile!) : const SizedBox(), Align( alignment: Alignment.bottomCenter, child: Row( mainAxisAlignment: MainAxisAlignment.center, children: [ TextButton( onPressed: pickImage, child: const Text("Pick image"), ), ElevatedButton( onPressed: yoloOnImage, child: const Text("Detect"), ) ], ), ), ...displayBoxesAroundRecognizedObjects(size), ], ); } Future loadYoloModel() async { await widget.vision.loadYoloModel( labels: 'assets/labels.txt', modelPath: 'assets/yolov5n.tflite', modelVersion: "yolov5", quantization: false, numThreads: 2, useGpu: true); setState(() { isLoaded = true; }); } Future pickImage() async { final ImagePicker picker = ImagePicker(); // Capture a photo final XFile? photo = await picker.pickImage(source: ImageSource.gallery); if (photo != null) { setState(() { imageFile = File(photo.path); }); } } yoloOnImage() async { yoloResults.clear(); Uint8List byte = await imageFile!.readAsBytes(); final image = await decodeImageFromList(byte); imageHeight = image.height; imageWidth = image.width; final result = await widget.vision.yoloOnImage( bytesList: byte, imageHeight: image.height, imageWidth: image.width, iouThreshold: 0.8, confThreshold: 0.4, classThreshold: 0.5); if (result.isNotEmpty) { setState(() { yoloResults = result; }); } } List displayBoxesAroundRecognizedObjects(Size screen) { if (yoloResults.isEmpty) return []; double factorX = screen.width / (imageWidth); double imgRatio = imageWidth / imageHeight; double newWidth = imageWidth * factorX; double newHeight = newWidth / imgRatio; double factorY = newHeight / (imageHeight); double pady = (screen.height - newHeight) / 2; Color colorPick = const Color.fromARGB(255, 50, 233, 30); return yoloResults.map((result) { return Positioned( left: result["box"][0] * factorX, top: result["box"][1] * factorY + pady, width: (result["box"][2] - result["box"][0]) * factorX, height: (result["box"][3] - result["box"][1]) * factorY, child: Container( decoration: BoxDecoration( borderRadius: const BorderRadius.all(Radius.circular(10.0)), border: Border.all(color: Colors.pink, width: 2.0), ), child: Text( "${result['tag']} ${(result['box'][4] * 100).toStringAsFixed(0)}%", style: TextStyle( background: Paint()..color = colorPick, color: Colors.white, fontSize: 18.0, ), ), ), ); }).toList(); } } // YOLO V8 SEMANTIC SEGMENTATION class YoloImageV8Seg extends StatefulWidget { final FlutterVision vision; const YoloImageV8Seg({Key? key, required this.vision}) : super(key: key); @override State createState() => _YoloImageV8SegState(); } class _YoloImageV8SegState extends State { late List> yoloResults; File? imageFile; int imageHeight = 1; int imageWidth = 1; bool isLoaded = false; @override void initState() { super.initState(); loadYoloModel().then((value) { setState(() { yoloResults = []; isLoaded = true; }); }); } @override void dispose() async { super.dispose(); } @override Widget build(BuildContext context) { final Size size = MediaQuery.of(context).size; if (!isLoaded) { return const Scaffold( body: Center( child: Text("Model not loaded, waiting for it"), ), ); } return Stack( fit: StackFit.expand, children: [ imageFile != null ? Image.file(imageFile!) : const SizedBox(), Align( alignment: Alignment.bottomCenter, child: Row( mainAxisAlignment: MainAxisAlignment.center, children: [ TextButton( onPressed: pickImage, child: const Text("Pick image"), ), ElevatedButton( onPressed: yoloOnImage, child: const Text("Detect"), ) ], ), ), ...displayBoxesAroundRecognizedObjects(size), ], ); } Future loadYoloModel() async { await widget.vision.loadYoloModel( labels: 'assets/labels.txt', modelPath: 'assets/yolov8n-seg.tflite', modelVersion: "yolov8seg", quantization: false, numThreads: 2, useGpu: true); setState(() { isLoaded = true; }); } Future pickImage() async { final ImagePicker picker = ImagePicker(); // Capture a photo final XFile? photo = await picker.pickImage(source: ImageSource.gallery); if (photo != null) { setState(() { imageFile = File(photo.path); }); } } yoloOnImage() async { yoloResults.clear(); Uint8List byte = await imageFile!.readAsBytes(); final image = await decodeImageFromList(byte); imageHeight = image.height; imageWidth = image.width; final result = await widget.vision.yoloOnImage( bytesList: byte, imageHeight: image.height, imageWidth: image.width, iouThreshold: 0.8, confThreshold: 0.4, classThreshold: 0.5); if (result.isNotEmpty) { setState(() { yoloResults = result; }); } } List displayBoxesAroundRecognizedObjects(Size screen) { if (yoloResults.isEmpty) return []; double factorX = screen.width / (imageWidth); double imgRatio = imageWidth / imageHeight; double newWidth = imageWidth * factorX; double newHeight = newWidth / imgRatio; double factorY = newHeight / (imageHeight); double pady = (screen.height - newHeight) / 2; Color colorPick = const Color.fromARGB(255, 50, 233, 30); return yoloResults.map((result) { return Stack(children: [ Positioned( left: result["box"][0] * factorX, top: result["box"][1] * factorY + pady, width: (result["box"][2] - result["box"][0]) * factorX, height: (result["box"][3] - result["box"][1]) * factorY, child: Container( decoration: BoxDecoration( borderRadius: const BorderRadius.all(Radius.circular(10.0)), border: Border.all(color: Colors.pink, width: 2.0), ), child: Text( "${result['tag']} ${(result['box'][4] * 100).toStringAsFixed(0)}%", style: TextStyle( background: Paint()..color = colorPick, color: Colors.white, fontSize: 18.0, ), ), ), ), Positioned( left: result["box"][0] * factorX, top: result["box"][1] * factorY + pady, width: (result["box"][2] - result["box"][0]) * factorX, height: (result["box"][3] - result["box"][1]) * factorY, child: CustomPaint( painter: PolygonPainter( points: (result["polygons"] as List).map((e) { Map xy = Map.from(e); xy['x'] = (xy['x'] as double) * factorX; xy['y'] = (xy['y'] as double) * factorY; return xy; }).toList()), )), ]); }).toList(); } } class PolygonPainter extends CustomPainter { final List> points; PolygonPainter({required this.points}); @override void paint(Canvas canvas, Size size) { final paint = Paint() ..color = const Color.fromARGB(129, 255, 2, 124) ..strokeWidth = 2 ..style = PaintingStyle.fill; final path = Path(); if (points.isNotEmpty) { path.moveTo(points[0]['x']!, points[0]['y']!); for (var i = 1; i < points.length; i++) { path.lineTo(points[i]['x']!, points[i]['y']!); } path.close(); } canvas.drawPath(path, paint); } @override bool shouldRepaint(CustomPainter oldDelegate) { return false; } }