From 67fa8fde7fe7ad5319c282753a884b2834dc1ca5 Mon Sep 17 00:00:00 2001 From: dustin Date: Sat, 7 Jul 2018 08:29:48 -0700 Subject: [PATCH 01/34] Add a placeholder for the live preview screen. --- .../example/lib/live_preview.dart | 15 +++++ .../firebase_ml_vision/example/lib/main.dart | 61 +++++++++++++++---- 2 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 packages/firebase_ml_vision/example/lib/live_preview.dart diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart new file mode 100644 index 000000000000..1db2d082af31 --- /dev/null +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -0,0 +1,15 @@ +import 'package:firebase_ml_vision_example/detector_painters.dart'; +import 'package:flutter/material.dart'; + +class LivePreview extends StatelessWidget { + final Detector detector; + + const LivePreview(this.detector, {Key key,}) : super(key: key); + + @override + Widget build(BuildContext context) { + return Center( + child: Text("Current detector: $detector"), + ); + } +} diff --git a/packages/firebase_ml_vision/example/lib/main.dart b/packages/firebase_ml_vision/example/lib/main.dart index fea6ac0dc7fb..59d562b54d12 100644 --- a/packages/firebase_ml_vision/example/lib/main.dart +++ b/packages/firebase_ml_vision/example/lib/main.dart @@ -5,10 +5,11 @@ import 'dart:async'; import 'dart:io'; +import 'package:firebase_ml_vision/firebase_ml_vision.dart'; import 'package:firebase_ml_vision_example/detector_painters.dart'; +import 'package:firebase_ml_vision_example/live_preview.dart'; import 'package:flutter/material.dart'; import 'package:image_picker/image_picker.dart'; -import 'package:firebase_ml_vision/firebase_ml_vision.dart'; void main() => runApp(new MaterialApp(home: _MyHomePage())); @@ -17,11 +18,28 @@ class _MyHomePage extends StatefulWidget { _MyHomePageState createState() => new _MyHomePageState(); } -class _MyHomePageState extends State<_MyHomePage> { +class _MyHomePageState extends State<_MyHomePage> + with SingleTickerProviderStateMixin { File _imageFile; Size _imageSize; List _scanResults; Detector _currentDetector = Detector.text; + TabController _tabController; + int _selectedPageIndex = 0; + + @override + void initState() { + super.initState(); + _tabController = new TabController(vsync: this, length: 2); + _tabController.addListener(_handleTabSelection); + _selectedPageIndex = 0; + } + + void _handleTabSelection() { + setState(() { + _selectedPageIndex = _tabController.index; + }); + } Future _getAndScanImage() async { setState(() { @@ -151,8 +169,10 @@ class _MyHomePageState extends State<_MyHomePage> { actions: [ new PopupMenuButton( onSelected: (Detector result) { - _currentDetector = result; - if (_imageFile != null) _scanImage(_imageFile); + setState(() { + _currentDetector = result; + if (_imageFile != null) _scanImage(_imageFile); + }); }, itemBuilder: (BuildContext context) => >[ const PopupMenuItem( @@ -174,15 +194,34 @@ class _MyHomePageState extends State<_MyHomePage> { ], ), ], + bottom: TabBar( + controller: _tabController, + tabs: [ + const Tab( + icon: const Icon(Icons.photo), + ), + const Tab( + icon: const Icon(Icons.camera), + ) + ], + ), ), - body: _imageFile == null - ? const Center(child: const Text('No image selected.')) - : _buildImage(), - floatingActionButton: new FloatingActionButton( - onPressed: _getAndScanImage, - tooltip: 'Pick Image', - child: const Icon(Icons.add_a_photo), + body: TabBarView( + controller: _tabController, + children: [ + _imageFile == null + ? const Center(child: const Text('No image selected.')) + : _buildImage(), + LivePreview(_currentDetector), + ], ), + floatingActionButton: _selectedPageIndex == 0 + ? new FloatingActionButton( + onPressed: _getAndScanImage, + tooltip: 'Pick Image', + child: const Icon(Icons.add_a_photo), + ) + : null, ); } } From c8649749a12f34af931a15ec4719c01adbc29ab0 Mon Sep 17 00:00:00 2001 From: dustin Date: Sat, 7 Jul 2018 12:13:01 -0700 Subject: [PATCH 02/34] WIP. basic barcode detection impl. --- .../flutter/plugins/camera/CameraPlugin.java | 2 + packages/camera/example/android/build.gradle | 2 +- .../firebasemlvision/BarcodeDetector.java | 67 ++++++++++++- .../FirebaseMlVisionPlugin.java | 5 +- .../firebasemlvision/TextDetector.java | 94 ++++++++++--------- .../constants/VisionBarcodeConstants.java | 7 ++ .../constants/VisionBaseConstants.java | 8 ++ .../util/DetectedItemUtils.java | 20 ++++ .../example/android/build.gradle | 2 +- .../example/lib/detector_painters.dart | 29 +++++- .../lib/firebase_ml_vision.dart | 5 + .../lib/src/barcode_detector.dart | 26 ++++- .../lib/src/vision_model_utils.dart | 16 ++++ 13 files changed, 224 insertions(+), 59 deletions(-) create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/constants/VisionBarcodeConstants.java create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/constants/VisionBaseConstants.java create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/util/DetectedItemUtils.java create mode 100644 packages/firebase_ml_vision/lib/src/vision_model_utils.dart diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 1d4bedefbbf4..6cf38c5f6239 100644 --- a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -131,6 +131,7 @@ public static void registerWith(Registrar registrar) { cameraManager = (CameraManager) registrar.activity().getSystemService(Context.CAMERA_SERVICE); + channel.setMethodCallHandler( new CameraPlugin(registrar, registrar.view(), registrar.activity())); } @@ -735,5 +736,6 @@ private void dispose() { close(); textureEntry.release(); } + } } diff --git a/packages/camera/example/android/build.gradle b/packages/camera/example/android/build.gradle index d4225c7905bc..23d41d039885 100644 --- a/packages/camera/example/android/build.gradle +++ b/packages/camera/example/android/build.gradle @@ -5,7 +5,7 @@ buildscript { } dependencies { - classpath 'com.android.tools.build:gradle:3.1.2' + classpath 'com.android.tools.build:gradle:3.1.3' } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java index ea574c67be7d..f7515bcae318 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java @@ -1,12 +1,75 @@ package io.flutter.plugins.firebasemlvision; +import android.graphics.Rect; +import android.support.annotation.NonNull; + +import com.google.android.gms.tasks.OnFailureListener; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.firebase.ml.vision.FirebaseVision; +import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode; +import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector; import com.google.firebase.ml.vision.common.FirebaseVisionImage; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; + +import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.*; class BarcodeDetector implements Detector { + public static final BarcodeDetector instance = new BarcodeDetector(); + private static FirebaseVisionBarcodeDetector barcodeDetector; + @Override - public void handleDetection(FirebaseVisionImage image, MethodChannel.Result result) {} + public void handleDetection(FirebaseVisionImage image, final MethodChannel.Result result) { + if (barcodeDetector == null) barcodeDetector = FirebaseVision.getInstance().getVisionBarcodeDetector(); + barcodeDetector + .detectInImage(image) + .addOnSuccessListener(new OnSuccessListener>() { + @Override + public void onSuccess(List firebaseVisionBarcodes) { + List> barcodes = new ArrayList<>(); + for (FirebaseVisionBarcode barcode : firebaseVisionBarcodes) { + Map barcodeData = new HashMap<>(); + addBarcodeData(barcodeData, barcode); + barcodes.add(barcodeData); + } + result.success(barcodes); + } + }) + .addOnFailureListener(new OnFailureListener() { + @Override + public void onFailure(@NonNull Exception e) { + result.error("barcodeDetectorError", e.getLocalizedMessage(), null); + } + }); + } @Override - public void close(MethodChannel.Result result) {} + public void close(MethodChannel.Result result) { + if (barcodeDetector != null) { + try { + barcodeDetector.close(); + result.success(null); + } catch (IOException e) { + result.error("barcodeDetectorError", e.getLocalizedMessage(), null); + } + } + barcodeDetector = null; + } + + private void addBarcodeData(Map addTo, FirebaseVisionBarcode barcode) { + Rect boundingBox = barcode.getBoundingBox(); + if (boundingBox != null) { + addTo.putAll(DetectedItemUtils.rectToFlutterMap(boundingBox)); + } + addTo.put(BARCODE_VALUE_TYPE, barcode.getValueType()); + addTo.put(BARCODE_DISPLAY_VALUE, barcode.getDisplayValue()); + addTo.put(BARCODE_RAW_VALUE, barcode.getRawValue()); + } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 454e02b04adf..cde6fdb24d2f 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -29,8 +29,11 @@ public static void registerWith(Registrar registrar) { public void onMethodCall(MethodCall call, Result result) { switch (call.method) { case "BarcodeDetector#detectInImage": + FirebaseVisionImage image = filePathToVisionImage((String) call.arguments, result); + if (image != null) BarcodeDetector.instance.handleDetection(image, result); break; case "BarcodeDetector#close": + BarcodeDetector.instance.close(result); break; case "FaceDetector#detectInImage": break; @@ -41,7 +44,7 @@ public void onMethodCall(MethodCall call, Result result) { case "LabelDetector#close": break; case "TextDetector#detectInImage": - FirebaseVisionImage image = filePathToVisionImage((String) call.arguments, result); + image = filePathToVisionImage((String) call.arguments, result); if (image != null) TextDetector.instance.handleDetection(image, result); break; case "TextDetector#close": diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java index 27e2b9723dc7..cf8780962615 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java @@ -3,13 +3,17 @@ import android.graphics.Point; import android.graphics.Rect; import android.support.annotation.NonNull; + import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; import com.google.firebase.ml.vision.common.FirebaseVisionImage; import com.google.firebase.ml.vision.text.FirebaseVisionText; import com.google.firebase.ml.vision.text.FirebaseVisionTextDetector; + import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; + import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -20,54 +24,55 @@ public class TextDetector implements Detector { public static final TextDetector instance = new TextDetector(); private static FirebaseVisionTextDetector textDetector; - private TextDetector() {} + private TextDetector() { + } public void handleDetection(FirebaseVisionImage image, final MethodChannel.Result result) { if (textDetector == null) textDetector = FirebaseVision.getInstance().getVisionTextDetector(); textDetector - .detectInImage(image) - .addOnSuccessListener( - new OnSuccessListener() { - @Override - public void onSuccess(FirebaseVisionText firebaseVisionText) { - List> blocks = new ArrayList<>(); - for (FirebaseVisionText.Block block : firebaseVisionText.getBlocks()) { - Map blockData = new HashMap<>(); - addTextData( - blockData, block.getBoundingBox(), block.getCornerPoints(), block.getText()); + .detectInImage(image) + .addOnSuccessListener( + new OnSuccessListener() { + @Override + public void onSuccess(FirebaseVisionText firebaseVisionText) { + List> blocks = new ArrayList<>(); + for (FirebaseVisionText.Block block : firebaseVisionText.getBlocks()) { + Map blockData = new HashMap<>(); + addTextData( + blockData, block.getBoundingBox(), block.getCornerPoints(), block.getText()); - List> lines = new ArrayList<>(); - for (FirebaseVisionText.Line line : block.getLines()) { - Map lineData = new HashMap<>(); - addTextData( - lineData, line.getBoundingBox(), line.getCornerPoints(), line.getText()); + List> lines = new ArrayList<>(); + for (FirebaseVisionText.Line line : block.getLines()) { + Map lineData = new HashMap<>(); + addTextData( + lineData, line.getBoundingBox(), line.getCornerPoints(), line.getText()); - List> elements = new ArrayList<>(); - for (FirebaseVisionText.Element element : line.getElements()) { - Map elementData = new HashMap<>(); - addTextData( - elementData, - element.getBoundingBox(), - element.getCornerPoints(), - element.getText()); - elements.add(elementData); - } - lineData.put("elements", elements); - lines.add(lineData); - } - blockData.put("lines", lines); - blocks.add(blockData); + List> elements = new ArrayList<>(); + for (FirebaseVisionText.Element element : line.getElements()) { + Map elementData = new HashMap<>(); + addTextData( + elementData, + element.getBoundingBox(), + element.getCornerPoints(), + element.getText()); + elements.add(elementData); } - result.success(blocks); - } - }) - .addOnFailureListener( - new OnFailureListener() { - @Override - public void onFailure(@NonNull Exception exception) { - result.error("textDetectorError", exception.getLocalizedMessage(), null); + lineData.put("elements", elements); + lines.add(lineData); } - }); + blockData.put("lines", lines); + blocks.add(blockData); + } + result.success(blocks); + } + }) + .addOnFailureListener( + new OnFailureListener() { + @Override + public void onFailure(@NonNull Exception exception) { + result.error("textDetectorError", exception.getLocalizedMessage(), null); + } + }); } public void close(MethodChannel.Result result) { @@ -84,20 +89,17 @@ public void close(MethodChannel.Result result) { } private void addTextData( - Map addTo, Rect boundingBox, Point[] cornerPoints, String text) { + Map addTo, Rect boundingBox, Point[] cornerPoints, String text) { addTo.put("text", text); if (boundingBox != null) { - addTo.put("left", boundingBox.left); - addTo.put("top", boundingBox.top); - addTo.put("width", boundingBox.width()); - addTo.put("height", boundingBox.height()); + addTo.putAll(DetectedItemUtils.rectToFlutterMap(boundingBox)); } List points = new ArrayList<>(); if (cornerPoints != null) { for (Point point : cornerPoints) { - points.add(new int[] {point.x, point.y}); + points.add(new int[]{point.x, point.y}); } } addTo.put("points", points); diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/constants/VisionBarcodeConstants.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/constants/VisionBarcodeConstants.java new file mode 100644 index 000000000000..805a2b25348e --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/constants/VisionBarcodeConstants.java @@ -0,0 +1,7 @@ +package io.flutter.plugins.firebasemlvision.constants; + +public class VisionBarcodeConstants { + public static String BARCODE_VALUE_TYPE = "barcode_value_type"; + public static String BARCODE_DISPLAY_VALUE = "barcode_display_value"; + public static String BARCODE_RAW_VALUE = "barcode_raw_value"; +} diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/constants/VisionBaseConstants.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/constants/VisionBaseConstants.java new file mode 100644 index 000000000000..34622fa23445 --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/constants/VisionBaseConstants.java @@ -0,0 +1,8 @@ +package io.flutter.plugins.firebasemlvision.constants; + +public class VisionBaseConstants { + public static final String LEFT = "left"; + public static final String TOP = "top"; + public static final String WIDTH = "width"; + public static final String HEIGHT = "height"; +} diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/util/DetectedItemUtils.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/util/DetectedItemUtils.java new file mode 100644 index 000000000000..e0dec74695ca --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/util/DetectedItemUtils.java @@ -0,0 +1,20 @@ +package io.flutter.plugins.firebasemlvision.util; + +import android.graphics.Rect; + +import java.util.HashMap; +import java.util.Map; +import static io.flutter.plugins.firebasemlvision.constants.VisionBaseConstants.*; + +public class DetectedItemUtils { + + public static Map rectToFlutterMap(Rect boundingBox) { + Map out = new HashMap<>(); + out.put(LEFT, boundingBox.left); + out.put(TOP, boundingBox.top); + out.put(WIDTH, boundingBox.width()); + out.put(HEIGHT, boundingBox.height()); + return out; + } + +} diff --git a/packages/firebase_ml_vision/example/android/build.gradle b/packages/firebase_ml_vision/example/android/build.gradle index aa8e5258c61e..1229169caed2 100644 --- a/packages/firebase_ml_vision/example/android/build.gradle +++ b/packages/firebase_ml_vision/example/android/build.gradle @@ -5,7 +5,7 @@ buildscript { } dependencies { - classpath 'com.android.tools.build:gradle:3.1.2' + classpath 'com.android.tools.build:gradle:3.1.3' classpath 'com.google.gms:google-services:3.1.2' } } diff --git a/packages/firebase_ml_vision/example/lib/detector_painters.dart b/packages/firebase_ml_vision/example/lib/detector_painters.dart index bffd4cc53a7a..fb2d734b9650 100644 --- a/packages/firebase_ml_vision/example/lib/detector_painters.dart +++ b/packages/firebase_ml_vision/example/lib/detector_painters.dart @@ -11,17 +11,36 @@ class BarcodeDetectorPainter extends CustomPainter { BarcodeDetectorPainter(this.absoluteImageSize, this.results); final Size absoluteImageSize; - final List results; + final List results; @override void paint(Canvas canvas, Size size) { - // TODO: implement paint + final double scaleX = size.width / absoluteImageSize.width; + final double scaleY = size.height / absoluteImageSize.height; + + Rect scaleRect(BarcodeContainer container) { + return new Rect.fromLTRB( + container.boundingBox.left * scaleX, + container.boundingBox.top * scaleY, + container.boundingBox.right * scaleX, + container.boundingBox.bottom * scaleY, + ); + } + + final Paint paint = new Paint() + ..style = PaintingStyle.stroke + ..strokeWidth = 2.0; + + for (BarcodeContainer barcode in results) { + paint.color = Colors.red; + canvas.drawRect(scaleRect(barcode), paint); + } } @override - bool shouldRepaint(CustomPainter oldDelegate) { - // TODO: implement shouldRepaint - return false; + bool shouldRepaint(BarcodeDetectorPainter oldDelegate) { + return oldDelegate.absoluteImageSize != absoluteImageSize || + oldDelegate.results != results; } } diff --git a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart index 664f6eb18f22..3d7cb91d2c84 100644 --- a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart +++ b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart @@ -8,6 +8,7 @@ import 'dart:async'; import 'dart:io'; import 'dart:math'; +import 'package:firebase_ml_vision/src/vision_model_utils.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; @@ -16,3 +17,7 @@ part 'src/face_detector.dart'; part 'src/firebase_vision.dart'; part 'src/label_detector.dart'; part 'src/text_detector.dart'; + +const String barcodeValueType = "barcode_value_type"; +const String barcodeDisplayValue = "barcode_display_value"; +const String barcodeRawValue = "barcode_raw_value"; \ No newline at end of file diff --git a/packages/firebase_ml_vision/lib/src/barcode_detector.dart b/packages/firebase_ml_vision/lib/src/barcode_detector.dart index d2768046a147..1dc44be096aa 100644 --- a/packages/firebase_ml_vision/lib/src/barcode_detector.dart +++ b/packages/firebase_ml_vision/lib/src/barcode_detector.dart @@ -9,13 +9,33 @@ class BarcodeDetector extends FirebaseVisionDetector { @override Future close() async { - // TODO: implement close + return FirebaseVision.channel.invokeMethod('BarcodeDetector#close'); } @override - Future detectInImage(FirebaseVisionImage visionImage) async { - // TODO: implement detectInImage + Future> detectInImage( + FirebaseVisionImage visionImage) async { + final List reply = await FirebaseVision.channel.invokeMethod( + 'BarcodeDetector#detectInImage', visionImage.imageFile.path); + final List barcodes = []; + reply.forEach((dynamic barcodeMap) { + barcodes.add(new BarcodeContainer._(barcodeMap)); + }); + return barcodes; } } +class BarcodeContainer { + final Rectangle boundingBox; + final int valueType; + final String displayValue; + final String rawValue; + + BarcodeContainer._(Map data) + : boundingBox = VisionModelUtils.mlRectToRectangle(data), + valueType = data[barcodeValueType], + displayValue = data[barcodeDisplayValue], + rawValue = data[barcodeRawValue]; +} + class BarcodeDetectorOptions {} diff --git a/packages/firebase_ml_vision/lib/src/vision_model_utils.dart b/packages/firebase_ml_vision/lib/src/vision_model_utils.dart new file mode 100644 index 000000000000..d6a4cd9baa0e --- /dev/null +++ b/packages/firebase_ml_vision/lib/src/vision_model_utils.dart @@ -0,0 +1,16 @@ +import 'dart:math'; + +class VisionModelUtils { + static const String rectLeft = "left"; + static const String rectTop = "top"; + static const String rectWidth = "width"; + static const String rectHeight = "height"; + + static Rectangle mlRectToRectangle(Map data) { + if (data != null) { + return Rectangle(data[rectLeft], data[rectTop], data[rectWidth], data[rectHeight]); + } else { + return null; + } + } +} \ No newline at end of file From 73f4fe2423896a917f19e55970aa84d27fc3620d Mon Sep 17 00:00:00 2001 From: dustin Date: Sat, 7 Jul 2018 15:43:19 -0700 Subject: [PATCH 03/34] WIP. --- packages/firebase_ml_vision/android/build.gradle | 4 ++-- .../plugins/firebasemlvision/BarcodeDetector.java | 1 + packages/firebase_ml_vision/example/lib/main.dart | 2 +- .../firebase_ml_vision/lib/src/vision_model_utils.dart | 9 +++++++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/firebase_ml_vision/android/build.gradle b/packages/firebase_ml_vision/android/build.gradle index 2d0d116e51ad..7672359f51b1 100644 --- a/packages/firebase_ml_vision/android/build.gradle +++ b/packages/firebase_ml_vision/android/build.gradle @@ -8,7 +8,7 @@ buildscript { } dependencies { - classpath 'com.android.tools.build:gradle:3.1.2' + classpath 'com.android.tools.build:gradle:3.1.3' } } @@ -32,6 +32,6 @@ android { disable 'InvalidPackage' } dependencies { - api 'com.google.firebase:firebase-ml-vision:15.+' + api 'com.google.firebase:firebase-ml-vision:16.0.0' } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java index f7515bcae318..7e42a127eed9 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java @@ -9,6 +9,7 @@ import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode; import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector; import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; import java.io.IOException; import java.util.ArrayList; diff --git a/packages/firebase_ml_vision/example/lib/main.dart b/packages/firebase_ml_vision/example/lib/main.dart index 59d562b54d12..8d715050b1a0 100644 --- a/packages/firebase_ml_vision/example/lib/main.dart +++ b/packages/firebase_ml_vision/example/lib/main.dart @@ -144,7 +144,7 @@ class _MyHomePageState extends State<_MyHomePage> decoration: new BoxDecoration( image: new DecorationImage( image: Image.file(_imageFile).image, - fit: BoxFit.fill, + fit: BoxFit.cover, ), ), child: _imageSize == null || _scanResults == null diff --git a/packages/firebase_ml_vision/lib/src/vision_model_utils.dart b/packages/firebase_ml_vision/lib/src/vision_model_utils.dart index d6a4cd9baa0e..a5af377ee4bf 100644 --- a/packages/firebase_ml_vision/lib/src/vision_model_utils.dart +++ b/packages/firebase_ml_vision/lib/src/vision_model_utils.dart @@ -8,9 +8,14 @@ class VisionModelUtils { static Rectangle mlRectToRectangle(Map data) { if (data != null) { - return Rectangle(data[rectLeft], data[rectTop], data[rectWidth], data[rectHeight]); + return Rectangle( + data[rectLeft], + data[rectTop], + data[rectWidth], + data[rectHeight], + ); } else { return null; } } -} \ No newline at end of file +} From 04395d80009f433de47689de8bb70ddc23e74c02 Mon Sep 17 00:00:00 2001 From: dustin Date: Mon, 9 Jul 2018 07:37:53 -0700 Subject: [PATCH 04/34] rotate image on Android prior to creating the FirebaseVisionImage resulting bounding box was inaccurate. --- .../firebase_ml_vision/android/build.gradle | 1 + .../FirebaseMlVisionPlugin.java | 71 ++++++++++++++++++- .../firebase_ml_vision/example/lib/main.dart | 2 +- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/packages/firebase_ml_vision/android/build.gradle b/packages/firebase_ml_vision/android/build.gradle index 7672359f51b1..9896a9287ae8 100644 --- a/packages/firebase_ml_vision/android/build.gradle +++ b/packages/firebase_ml_vision/android/build.gradle @@ -33,5 +33,6 @@ android { } dependencies { api 'com.google.firebase:firebase-ml-vision:16.0.0' + compile "com.android.support:exifinterface:27.1.1" } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index cde6fdb24d2f..336179c46da9 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -1,6 +1,14 @@ package io.flutter.plugins.firebasemlvision; +import android.content.ContentValues; +import android.content.Context; +import android.database.Cursor; +import android.graphics.Bitmap; +import android.graphics.Matrix; import android.net.Uri; +import android.provider.MediaStore; +import android.support.media.ExifInterface; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; @@ -59,11 +67,72 @@ private FirebaseVisionImage filePathToVisionImage(String path, Result result) { File file = new File(path); try { - return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file)); + Bitmap bitmap = MediaStore.Images.Media.getBitmap(registrar.context().getContentResolver(), Uri.fromFile(file)); + int rotation = 0; + int orientation = new ExifInterface(path).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); + + switch (orientation) { + case ExifInterface.ORIENTATION_ROTATE_90: + rotation = 90; + break; + case ExifInterface.ORIENTATION_ROTATE_180: + rotation = 180; + break; + case ExifInterface.ORIENTATION_ROTATE_270: + rotation = 270; + break; + } + Matrix matrix = new Matrix(); + matrix.postRotate(rotation); + Bitmap rotatedImg = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); + return FirebaseVisionImage.fromBitmap(rotatedImg); } catch (IOException exception) { result.error("textDetectorIOError", exception.getLocalizedMessage(), null); } return null; } + +// private static int getOrientationFromMediaStore(Context context, String imagePath) { +// Uri imageUri = getImageContentUri(context, imagePath); +// if(imageUri == null) { +// return -1; +// } +// +// String[] projection = {MediaStore.Images.ImageColumns.ORIENTATION}; +// Cursor cursor = context.getContentResolver().query(imageUri, projection, null, null, null); +// +// int orientation = -1; +// if (cursor != null && cursor.moveToFirst()) { +// orientation = cursor.getInt(0); +// cursor.close(); +// } +// +// return orientation; +// } +// +// private static Uri getImageContentUri(Context context, String imagePath) { +// String[] projection = new String[] {MediaStore.Images.Media._ID}; +// String selection = MediaStore.Images.Media.DATA + "=? "; +// String[] selectionArgs = new String[] {imagePath}; +// Cursor cursor = context.getContentResolver().query(IMAGE_PROVIDER_URI, projection, +// selection, selectionArgs, null); +// +// if (cursor != null && cursor.moveToFirst()) { +// int imageId = cursor.getInt(0); +// cursor.close(); +// +// return Uri.withAppendedPath(IMAGE_PROVIDER_URI, Integer.toString(imageId)); +// } +// +// if (new File(imagePath).exists()) { +// ContentValues values = new ContentValues(); +// values.put(MediaStore.Images.Media.DATA, imagePath); +// +// return context.getContentResolver().insert( +// MediaStore.Images.Media.EXTERNAL_CONTENT_URI, values); +// } +// +// return null; +// } } diff --git a/packages/firebase_ml_vision/example/lib/main.dart b/packages/firebase_ml_vision/example/lib/main.dart index 8d715050b1a0..59d562b54d12 100644 --- a/packages/firebase_ml_vision/example/lib/main.dart +++ b/packages/firebase_ml_vision/example/lib/main.dart @@ -144,7 +144,7 @@ class _MyHomePageState extends State<_MyHomePage> decoration: new BoxDecoration( image: new DecorationImage( image: Image.file(_imageFile).image, - fit: BoxFit.cover, + fit: BoxFit.fill, ), ), child: _imageSize == null || _scanResults == null From cb953e75e07425c0512528e24c2ab867f4ffad93 Mon Sep 17 00:00:00 2001 From: dustin Date: Mon, 9 Jul 2018 10:26:24 -0700 Subject: [PATCH 05/34] Add a basic camera preview implementation Android only for now. pulled impl from camera flutter plugin. --- .../android/src/main/AndroidManifest.xml | 2 + .../FirebaseMlVisionPlugin.java | 176 +++-- .../plugins/firebasemlvision/live/Camera.java | 599 ++++++++++++++++++ .../firebasemlvision/live/CameraInfo.java | 49 ++ .../live/CameraInfoException.java | 7 + .../example/lib/live_preview.dart | 127 +++- .../lib/firebase_ml_vision.dart | 1 + .../lib/src/firebase_vision.dart | 5 +- .../firebase_ml_vision/lib/src/live_view.dart | 249 ++++++++ 9 files changed, 1158 insertions(+), 57 deletions(-) create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraInfo.java create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraInfoException.java create mode 100644 packages/firebase_ml_vision/lib/src/live_view.dart diff --git a/packages/firebase_ml_vision/android/src/main/AndroidManifest.xml b/packages/firebase_ml_vision/android/src/main/AndroidManifest.xml index d43e3337d812..f5c4e1cb9f35 100644 --- a/packages/firebase_ml_vision/android/src/main/AndroidManifest.xml +++ b/packages/firebase_ml_vision/android/src/main/AndroidManifest.xml @@ -1,3 +1,5 @@ + + diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 336179c46da9..07296f492381 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -1,41 +1,146 @@ package io.flutter.plugins.firebasemlvision; -import android.content.ContentValues; -import android.content.Context; -import android.database.Cursor; +import android.app.Activity; +import android.app.Application; import android.graphics.Bitmap; import android.graphics.Matrix; +import android.hardware.camera2.CameraAccessException; import android.net.Uri; +import android.os.Bundle; import android.provider.MediaStore; +import android.support.annotation.Nullable; import android.support.media.ExifInterface; import com.google.firebase.ml.vision.common.FirebaseVisionImage; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugin.common.MethodChannel.MethodCallHandler; import io.flutter.plugin.common.MethodChannel.Result; +import io.flutter.plugin.common.PluginRegistry; import io.flutter.plugin.common.PluginRegistry.Registrar; -import java.io.File; -import java.io.IOException; +import io.flutter.plugins.firebasemlvision.live.Camera; +import io.flutter.plugins.firebasemlvision.live.CameraInfo; +import io.flutter.plugins.firebasemlvision.live.CameraInfoException; +import io.flutter.view.FlutterView; -/** FirebaseMlVisionPlugin */ +/** + * FirebaseMlVisionPlugin + */ public class FirebaseMlVisionPlugin implements MethodCallHandler { + public static final int CAMERA_REQUEST_ID = 928291720; private Registrar registrar; + private Activity activity; + + @Nullable + private Camera camera; private FirebaseMlVisionPlugin(Registrar registrar) { this.registrar = registrar; + this.activity = registrar.activity(); + + registrar.addRequestPermissionsResultListener(new CameraRequestPermissionsListener()); + + activity + .getApplication() + .registerActivityLifecycleCallbacks( + new Application.ActivityLifecycleCallbacks() { + @Override + public void onActivityCreated(Activity activity, Bundle savedInstanceState) { + } + + @Override + public void onActivityStarted(Activity activity) { + } + + @Override + public void onActivityResumed(Activity activity) { + if (camera != null && camera.getRequestingPermission()) { + camera.setRequestingPermission(false); + return; + } + // TODO: figure out how to open camera iff the app is presenting a live preview +// if (activity == FirebaseMlVisionPlugin.this.activity) { +// if (camera != null) { +// camera.open(null); +// } +// } + } + + @Override + public void onActivityPaused(Activity activity) { + if (activity == FirebaseMlVisionPlugin.this.activity) { + if (camera != null) { + camera.close(); + } + } + } + + @Override + public void onActivityStopped(Activity activity) { + if (activity == FirebaseMlVisionPlugin.this.activity) { + if (camera != null) { + camera.close(); + } + } + } + + @Override + public void onActivitySaveInstanceState(Activity activity, Bundle outState) { + } + + @Override + public void onActivityDestroyed(Activity activity) { + } + }); } - /** Plugin registration. */ + /** + * Plugin registration. + */ public static void registerWith(Registrar registrar) { final MethodChannel channel = - new MethodChannel(registrar.messenger(), "plugins.flutter.io/firebase_ml_vision"); + new MethodChannel(registrar.messenger(), "plugins.flutter.io/firebase_ml_vision"); channel.setMethodCallHandler(new FirebaseMlVisionPlugin(registrar)); } @Override public void onMethodCall(MethodCall call, Result result) { switch (call.method) { + case "init": + if (camera != null) { + camera.close(); + } + result.success(null); + break; + case "availableCameras": + try { + List> cameras = CameraInfo.getAvailableCameras(registrar.activeContext()); + result.success(cameras); + } catch (CameraInfoException e) { + result.error("cameraAccess", e.getMessage(), null); + } + break; + case "initialize": + String cameraName = call.argument("cameraName"); + String resolutionPreset = call.argument("resolutionPreset"); + if (camera != null) { + camera.close(); + } + camera = new Camera(registrar, cameraName, resolutionPreset, result); + break; + case "dispose": { + if (camera != null) { + camera.dispose(); + } + result.success(null); + break; + } case "BarcodeDetector#detectInImage": FirebaseVisionImage image = filePathToVisionImage((String) call.arguments, result); if (image != null) BarcodeDetector.instance.handleDetection(image, result); @@ -93,46 +198,17 @@ private FirebaseVisionImage filePathToVisionImage(String path, Result result) { return null; } -// private static int getOrientationFromMediaStore(Context context, String imagePath) { -// Uri imageUri = getImageContentUri(context, imagePath); -// if(imageUri == null) { -// return -1; -// } -// -// String[] projection = {MediaStore.Images.ImageColumns.ORIENTATION}; -// Cursor cursor = context.getContentResolver().query(imageUri, projection, null, null, null); -// -// int orientation = -1; -// if (cursor != null && cursor.moveToFirst()) { -// orientation = cursor.getInt(0); -// cursor.close(); -// } -// -// return orientation; -// } -// -// private static Uri getImageContentUri(Context context, String imagePath) { -// String[] projection = new String[] {MediaStore.Images.Media._ID}; -// String selection = MediaStore.Images.Media.DATA + "=? "; -// String[] selectionArgs = new String[] {imagePath}; -// Cursor cursor = context.getContentResolver().query(IMAGE_PROVIDER_URI, projection, -// selection, selectionArgs, null); -// -// if (cursor != null && cursor.moveToFirst()) { -// int imageId = cursor.getInt(0); -// cursor.close(); -// -// return Uri.withAppendedPath(IMAGE_PROVIDER_URI, Integer.toString(imageId)); -// } -// -// if (new File(imagePath).exists()) { -// ContentValues values = new ContentValues(); -// values.put(MediaStore.Images.Media.DATA, imagePath); -// -// return context.getContentResolver().insert( -// MediaStore.Images.Media.EXTERNAL_CONTENT_URI, values); -// } -// -// return null; -// } + private class CameraRequestPermissionsListener + implements PluginRegistry.RequestPermissionsResultListener { + @Override + public boolean onRequestPermissionsResult(int id, String[] permissions, int[] grantResults) { + if (id == CAMERA_REQUEST_ID) { + if (camera != null) { + camera.continueRequestingPermissions(); + } + return true; + } + return false; + } + } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java new file mode 100644 index 000000000000..42fed95badc9 --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -0,0 +1,599 @@ +package io.flutter.plugins.firebasemlvision.live; + +import android.Manifest; +import android.annotation.TargetApi; +import android.app.Activity; +import android.content.Context; +import android.content.pm.PackageManager; +import android.graphics.ImageFormat; +import android.graphics.SurfaceTexture; +import android.hardware.camera2.CameraAccessException; +import android.hardware.camera2.CameraCaptureSession; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraDevice; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.CameraMetadata; +import android.hardware.camera2.CaptureFailure; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.params.StreamConfigurationMap; +import android.media.Image; +import android.media.ImageReader; +import android.media.MediaRecorder; +import android.os.Build; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; +import android.support.annotation.RequiresApi; +import android.util.Log; +import android.util.Size; +import android.util.SparseIntArray; +import android.view.Surface; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugin.common.PluginRegistry; +import io.flutter.view.FlutterView; + +import static io.flutter.plugins.firebasemlvision.FirebaseMlVisionPlugin.CAMERA_REQUEST_ID; + +@TargetApi(Build.VERSION_CODES.LOLLIPOP) +public class Camera { + private static final SparseIntArray ORIENTATIONS = + new SparseIntArray() { + { + append(Surface.ROTATION_0, 0); + append(Surface.ROTATION_90, 90); + append(Surface.ROTATION_180, 180); + append(Surface.ROTATION_270, 270); + } + }; + + private final FlutterView.SurfaceTextureEntry textureEntry; + private CameraDevice cameraDevice; + private CameraCaptureSession cameraCaptureSession; + private EventChannel.EventSink eventSink; + private ImageReader imageReader; + private int sensorOrientation; + private boolean isFrontFacing; + private String cameraName; + private Size captureSize; + private Size previewSize; + private CaptureRequest.Builder captureRequestBuilder; + private Size videoSize; + private MediaRecorder mediaRecorder; + private boolean recordingVideo; + private Runnable cameraPermissionContinuation; + private boolean requestingPermission; + private PluginRegistry.Registrar registrar; + private Activity activity; + private CameraManager cameraManager; + + public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonNull String resolutionPreset, @NonNull final MethodChannel.Result result) { + + this.activity = registrar.activity(); + this.cameraManager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE); + this.registrar = registrar; + this.cameraName = cameraName; + textureEntry = registrar.view().createSurfaceTexture(); + + registerEventChannel(); + + if (resolutionPreset == null) { + Log.e("ML", "resolution preset is somehow null"); + resolutionPreset = "high"; + } + + try { + Size minPreviewSize; + switch (resolutionPreset) { + case "high": + minPreviewSize = new Size(1024, 768); + break; + case "medium": + minPreviewSize = new Size(640, 480); + break; + case "low": + minPreviewSize = new Size(320, 240); + break; + default: + throw new IllegalArgumentException("Unknown preset: " + resolutionPreset); + } + + CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraName); + StreamConfigurationMap streamConfigurationMap = + characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + //noinspection ConstantConditions + sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); + //noinspection ConstantConditions + isFrontFacing = + characteristics.get(CameraCharacteristics.LENS_FACING) + == CameraMetadata.LENS_FACING_FRONT; + computeBestCaptureSize(streamConfigurationMap); + computeBestPreviewAndRecordingSize(streamConfigurationMap, minPreviewSize, captureSize); + + if (cameraPermissionContinuation != null) { + result.error("cameraPermission", "Camera permission request ongoing", null); + } + cameraPermissionContinuation = + new Runnable() { + @Override + public void run() { + cameraPermissionContinuation = null; + if (!hasCameraPermission()) { + result.error( + "cameraPermission", "MediaRecorderCamera permission not granted", null); + return; + } +// if (!hasAudioPermission()) { +// result.error( +// "cameraPermission", "MediaRecorderAudio permission not granted", null); +// return; +// } + open(result); + } + }; + requestingPermission = false; + if (hasCameraPermission()/* && hasAudioPermission()*/) { + cameraPermissionContinuation.run(); + } else { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + requestingPermission = true; + registrar + .activity() + .requestPermissions( + new String[]{Manifest.permission.CAMERA}, + CAMERA_REQUEST_ID); + } + } + } catch (CameraAccessException e) { + result.error("CameraAccess", e.getMessage(), null); + } catch (IllegalArgumentException e) { + result.error("IllegalArgumentException", e.getMessage(), null); + } + } + + public void continueRequestingPermissions() { + cameraPermissionContinuation.run(); + } + + public boolean getRequestingPermission() { + return requestingPermission; + } + + public void setRequestingPermission(boolean isRequesting) { + requestingPermission = isRequesting; + } + + private void registerEventChannel() { + new EventChannel( + registrar.messenger(), "flutter.io/cameraPlugin/cameraEvents" + textureEntry.id()) + .setStreamHandler( + new EventChannel.StreamHandler() { + @Override + public void onListen(Object arguments, EventChannel.EventSink eventSink) { + Camera.this.eventSink = eventSink; + } + + @Override + public void onCancel(Object arguments) { + Camera.this.eventSink = null; + } + }); + } + + private boolean hasCameraPermission() { + return Build.VERSION.SDK_INT < Build.VERSION_CODES.M + || activity.checkSelfPermission(Manifest.permission.CAMERA) + == PackageManager.PERMISSION_GRANTED; + } +// +// private boolean hasAudioPermission() { +// return Build.VERSION.SDK_INT < Build.VERSION_CODES.M +// || registrar.activity().checkSelfPermission(Manifest.permission.RECORD_AUDIO) +// == PackageManager.PERMISSION_GRANTED; +// } + + @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) + private void computeBestPreviewAndRecordingSize( + StreamConfigurationMap streamConfigurationMap, Size minPreviewSize, Size captureSize) { + Size[] sizes = streamConfigurationMap.getOutputSizes(SurfaceTexture.class); + float captureSizeRatio = (float) captureSize.getWidth() / captureSize.getHeight(); + List goodEnough = new ArrayList<>(); + for (Size s : sizes) { + if ((float) s.getWidth() / s.getHeight() == captureSizeRatio + && minPreviewSize.getWidth() < s.getWidth() + && minPreviewSize.getHeight() < s.getHeight()) { + goodEnough.add(s); + } + } + + Collections.sort(goodEnough, new CompareSizesByArea()); + + if (goodEnough.isEmpty()) { + previewSize = sizes[0]; + videoSize = sizes[0]; + } else { + previewSize = goodEnough.get(0); + + // Video capture size should not be greater than 1080 because MediaRecorder cannot handle higher resolutions. + videoSize = goodEnough.get(0); + for (int i = goodEnough.size() - 1; i >= 0; i--) { + if (goodEnough.get(i).getHeight() <= 1080) { + videoSize = goodEnough.get(i); + break; + } + } + } + } + + @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) + private void computeBestCaptureSize(StreamConfigurationMap streamConfigurationMap) { + // For still image captures, we use the largest available size. + captureSize = + Collections.max( + Arrays.asList(streamConfigurationMap.getOutputSizes(ImageFormat.JPEG)), + new CompareSizesByArea()); + } + + @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) + private void prepareMediaRecorder(String outputFilePath) throws IOException { + if (mediaRecorder != null) { + mediaRecorder.release(); + } + mediaRecorder = new MediaRecorder(); + mediaRecorder.setAudioSource(MediaRecorder.AudioSource.MIC); + mediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); + mediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4); + mediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC); + mediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); + mediaRecorder.setVideoEncodingBitRate(1024 * 1000); + mediaRecorder.setAudioSamplingRate(16000); + mediaRecorder.setVideoFrameRate(27); + mediaRecorder.setVideoSize(videoSize.getWidth(), videoSize.getHeight()); + mediaRecorder.setOutputFile(outputFilePath); + + int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation(); + int displayOrientation = ORIENTATIONS.get(displayRotation); + if (isFrontFacing) displayOrientation = -displayOrientation; + mediaRecorder.setOrientationHint((displayOrientation + sensorOrientation) % 360); + + mediaRecorder.prepare(); + } + + private void open(@Nullable final MethodChannel.Result result) { + if (!hasCameraPermission()) { + if (result != null) result.error("cameraPermission", "Camera permission not granted", null); + } else { + try { + imageReader = + ImageReader.newInstance( + captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2); + cameraManager.openCamera( + cameraName, + new CameraDevice.StateCallback() { + @Override + public void onOpened(@NonNull CameraDevice cameraDevice) { + Camera.this.cameraDevice = cameraDevice; + try { + startPreview(); + } catch (CameraAccessException e) { + if (result != null) result.error("CameraAccess", e.getMessage(), null); + } + + if (result != null) { + Map reply = new HashMap<>(); + reply.put("textureId", textureEntry.id()); + reply.put("previewWidth", previewSize.getWidth()); + reply.put("previewHeight", previewSize.getHeight()); + result.success(reply); + } + } + + @Override + public void onClosed(@NonNull CameraDevice camera) { + if (eventSink != null) { + Map event = new HashMap<>(); + event.put("eventType", "cameraClosing"); + eventSink.success(event); + } + super.onClosed(camera); + } + + @Override + public void onDisconnected(@NonNull CameraDevice cameraDevice) { + cameraDevice.close(); + Camera.this.cameraDevice = null; + sendErrorEvent("The camera was disconnected."); + } + + @Override + public void onError(@NonNull CameraDevice cameraDevice, int errorCode) { + cameraDevice.close(); + Camera.this.cameraDevice = null; + String errorDescription; + switch (errorCode) { + case ERROR_CAMERA_IN_USE: + errorDescription = "The camera device is in use already."; + break; + case ERROR_MAX_CAMERAS_IN_USE: + errorDescription = "Max cameras in use"; + break; + case ERROR_CAMERA_DISABLED: + errorDescription = + "The camera device could not be opened due to a device policy."; + break; + case ERROR_CAMERA_DEVICE: + errorDescription = "The camera device has encountered a fatal error"; + break; + case ERROR_CAMERA_SERVICE: + errorDescription = "The camera service has encountered a fatal error."; + break; + default: + errorDescription = "Unknown camera error"; + } + sendErrorEvent(errorDescription); + } + }, + null); + } catch (CameraAccessException e) { + if (result != null) result.error("cameraAccess", e.getMessage(), null); + } + } + } + + private void writeToFile(ByteBuffer buffer, File file) throws IOException { + try (FileOutputStream outputStream = new FileOutputStream(file)) { + while (0 < buffer.remaining()) { + outputStream.getChannel().write(buffer); + } + } + } + + private void takePicture(String filePath, @NonNull final MethodChannel.Result result) { + final File file = new File(filePath); + + if (file.exists()) { + result.error( + "fileExists", + "File at path '" + filePath + "' already exists. Cannot overwrite.", + null); + return; + } + + imageReader.setOnImageAvailableListener( + new ImageReader.OnImageAvailableListener() { + @Override + public void onImageAvailable(ImageReader reader) { + try (Image image = reader.acquireLatestImage()) { + ByteBuffer buffer = image.getPlanes()[0].getBuffer(); + writeToFile(buffer, file); + result.success(null); + } catch (IOException e) { + result.error("IOError", "Failed saving image", null); + } + } + }, + null); + + try { + final CaptureRequest.Builder captureBuilder = + cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE); + captureBuilder.addTarget(imageReader.getSurface()); + int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation(); + int displayOrientation = ORIENTATIONS.get(displayRotation); + if (isFrontFacing) displayOrientation = -displayOrientation; + captureBuilder.set( + CaptureRequest.JPEG_ORIENTATION, (-displayOrientation + sensorOrientation) % 360); + + cameraCaptureSession.capture( + captureBuilder.build(), + new CameraCaptureSession.CaptureCallback() { + @Override + public void onCaptureFailed( + @NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + @NonNull CaptureFailure failure) { + String reason; + switch (failure.getReason()) { + case CaptureFailure.REASON_ERROR: + reason = "An error happened in the framework"; + break; + case CaptureFailure.REASON_FLUSHED: + reason = "The capture has failed due to an abortCaptures() call"; + break; + default: + reason = "Unknown reason"; + } + result.error("captureFailure", reason, null); + } + }, + null); + } catch (CameraAccessException e) { + result.error("cameraAccess", e.getMessage(), null); + } + } + + private void startVideoRecording(String filePath, @NonNull final MethodChannel.Result result) { + if (cameraDevice == null) { + result.error("configureFailed", "Camera was closed during configuration.", null); + return; + } + if (new File(filePath).exists()) { + result.error( + "fileExists", + "File at path '" + filePath + "' already exists. Cannot overwrite.", + null); + return; + } + try { + closeCaptureSession(); + prepareMediaRecorder(filePath); + + recordingVideo = true; + + SurfaceTexture surfaceTexture = textureEntry.surfaceTexture(); + surfaceTexture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); + captureRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); + + List surfaces = new ArrayList<>(); + + Surface previewSurface = new Surface(surfaceTexture); + surfaces.add(previewSurface); + captureRequestBuilder.addTarget(previewSurface); + + Surface recorderSurface = mediaRecorder.getSurface(); + surfaces.add(recorderSurface); + captureRequestBuilder.addTarget(recorderSurface); + + cameraDevice.createCaptureSession( + surfaces, + new CameraCaptureSession.StateCallback() { + @Override + public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) { + try { + if (cameraDevice == null) { + result.error("configureFailed", "Camera was closed during configuration", null); + return; + } + Camera.this.cameraCaptureSession = cameraCaptureSession; + captureRequestBuilder.set( + CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO); + cameraCaptureSession.setRepeatingRequest( + captureRequestBuilder.build(), null, null); + mediaRecorder.start(); + result.success(null); + } catch (CameraAccessException e) { + result.error("cameraAccess", e.getMessage(), null); + } + } + + @Override + public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) { + result.error("configureFailed", "Failed to configure camera session", null); + } + }, + null); + } catch (CameraAccessException | IOException e) { + result.error("videoRecordingFailed", e.getMessage(), null); + } + } + + private void stopVideoRecording(@NonNull final MethodChannel.Result result) { + if (!recordingVideo) { + result.success(null); + return; + } + + try { + recordingVideo = false; + mediaRecorder.stop(); + mediaRecorder.reset(); + startPreview(); + result.success(null); + } catch (CameraAccessException | IllegalStateException e) { + result.error("videoRecordingFailed", e.getMessage(), null); + } + } + + private void startPreview() throws CameraAccessException { + closeCaptureSession(); + + SurfaceTexture surfaceTexture = textureEntry.surfaceTexture(); + surfaceTexture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); + captureRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW); + + List surfaces = new ArrayList<>(); + + Surface previewSurface = new Surface(surfaceTexture); + surfaces.add(previewSurface); + captureRequestBuilder.addTarget(previewSurface); + + surfaces.add(imageReader.getSurface()); + + cameraDevice.createCaptureSession( + surfaces, + new CameraCaptureSession.StateCallback() { + + @Override + public void onConfigured(@NonNull CameraCaptureSession session) { + if (cameraDevice == null) { + sendErrorEvent("The camera was closed during configuration."); + return; + } + try { + cameraCaptureSession = session; + captureRequestBuilder.set( + CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO); + cameraCaptureSession.setRepeatingRequest(captureRequestBuilder.build(), null, null); + } catch (CameraAccessException e) { + sendErrorEvent(e.getMessage()); + } + } + + @Override + public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) { + sendErrorEvent("Failed to configure the camera for preview."); + } + }, + null); + } + + private void sendErrorEvent(String errorDescription) { + if (eventSink != null) { + Map event = new HashMap<>(); + event.put("eventType", "error"); + event.put("errorDescription", errorDescription); + eventSink.success(event); + } + } + + private void closeCaptureSession() { + if (cameraCaptureSession != null) { + cameraCaptureSession.close(); + cameraCaptureSession = null; + } + } + + public void close() { + closeCaptureSession(); + + if (cameraDevice != null) { + cameraDevice.close(); + cameraDevice = null; + } + if (imageReader != null) { + imageReader.close(); + imageReader = null; + } + if (mediaRecorder != null) { + mediaRecorder.reset(); + mediaRecorder.release(); + mediaRecorder = null; + } + } + + public void dispose() { + close(); + textureEntry.release(); + } + + private static class CompareSizesByArea implements Comparator { + @Override + public int compare(Size lhs, Size rhs) { + // We cast here to ensure the multiplications won't overflow. + return Long.signum( + (long) lhs.getWidth() * lhs.getHeight() - (long) rhs.getWidth() * rhs.getHeight()); + } + } +} diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraInfo.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraInfo.java new file mode 100644 index 000000000000..ef2e3871387e --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraInfo.java @@ -0,0 +1,49 @@ +package io.flutter.plugins.firebasemlvision.live; + +import android.annotation.TargetApi; +import android.content.Context; +import android.hardware.camera2.CameraAccessException; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.CameraMetadata; +import android.os.Build; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@TargetApi(Build.VERSION_CODES.LOLLIPOP) +public class CameraInfo { + public static List> getAvailableCameras(Context context) throws CameraInfoException { + try { + CameraManager cameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE); + String[] cameraNames = cameraManager.getCameraIdList(); + List> cameras = new ArrayList<>(); + for (String cameraName : cameraNames) { + HashMap details = new HashMap<>(); + CameraCharacteristics characteristics = + cameraManager.getCameraCharacteristics(cameraName); + details.put("name", cameraName); + @SuppressWarnings("ConstantConditions") + int lensFacing = characteristics.get(CameraCharacteristics.LENS_FACING); + switch (lensFacing) { + case CameraMetadata.LENS_FACING_FRONT: + details.put("lensFacing", "front"); + break; + case CameraMetadata.LENS_FACING_BACK: + details.put("lensFacing", "back"); + break; + case CameraMetadata.LENS_FACING_EXTERNAL: + details.put("lensFacing", "external"); + break; + } + cameras.add(details); + } + return cameras; + } catch (CameraAccessException e) { + throw new CameraInfoException(e.getMessage()); + } + } +} + diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraInfoException.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraInfoException.java new file mode 100644 index 000000000000..abf1ab44fbb5 --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraInfoException.java @@ -0,0 +1,7 @@ +package io.flutter.plugins.firebasemlvision.live; + +public class CameraInfoException extends Exception { + public CameraInfoException(String message) { + super(message); + } +} diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 1db2d082af31..cf54ba9e725f 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -1,15 +1,132 @@ +import 'dart:async'; + +import 'package:firebase_ml_vision/src/live_view.dart'; import 'package:firebase_ml_vision_example/detector_painters.dart'; import 'package:flutter/material.dart'; -class LivePreview extends StatelessWidget { +class LivePreview extends StatefulWidget { final Detector detector; - const LivePreview(this.detector, {Key key,}) : super(key: key); + const LivePreview( + this.detector, { + Key key, + }) : super(key: key); + + @override + LivePreviewState createState() { + return new LivePreviewState(); + } +} + +class LivePreviewState extends State { + bool _isShowingPreview = false; + LiveViewCameraLoadStateReady _readyLoadState; + + Stream _prepareCameraPreview() async* { + if (_readyLoadState != null) { + yield _readyLoadState; + } else { + yield new LiveViewCameraLoadStateLoading(); + final List cameras = await availableCameras(); + final backCamera = cameras.firstWhere((cameraDescription) => + cameraDescription.lensDirection == LiveViewCameraLensDirection.back); + if (backCamera != null) { + yield new LiveViewCameraLoadStateLoaded(backCamera); + try { + final LiveViewCameraController controller = + new LiveViewCameraController( + backCamera, LiveViewResolutionPreset.high); + await controller.initialize(); + yield new LiveViewCameraLoadStateReady(controller); + } on LiveViewCameraException catch (e) { + yield new LiveViewCameraLoadStateFailed( + "error initializing camera controller: ${e.toString()}"); + } + } else { + yield new LiveViewCameraLoadStateFailed("Could not find device camera"); + } + } + } @override Widget build(BuildContext context) { - return Center( - child: Text("Current detector: $detector"), - ); + if (_isShowingPreview) { + return new StreamBuilder( + stream: _prepareCameraPreview(), + builder: (BuildContext context, + AsyncSnapshot snapshot) { + print("snapshot data: ${snapshot.data}"); + final loadState = snapshot.data; + if (loadState != null) { + if (loadState is LiveViewCameraLoadStateLoading) { + return const Text("loading camera preview…"); + } else if (loadState is LiveViewCameraLoadStateLoaded) { + // get rid of previous controller if there is one + return new Text("loaded camera name: ${loadState + .cameraDescription + .name}"); + } else if (loadState is LiveViewCameraLoadStateReady) { + ////// BINGO!!!, the camera is ready to present + if (_readyLoadState != loadState) { + _readyLoadState?.dispose(); + _readyLoadState = loadState; + } + return new AspectRatio( + aspectRatio: _readyLoadState.controller.value.aspectRatio, + child: new LiveView(_readyLoadState.controller), + ); + } else if (loadState is LiveViewCameraLoadStateFailed) { + return new Text("error loading camera ${loadState + .errorMessage}"); + } else { + return const Text("Unknown Camera error"); + } + } else { + return new Text("Camera error: ${snapshot.error.toString()}"); + } + }, + ); + } else { + return Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Text("Current detector: ${widget.detector}"), + RaisedButton( + onPressed: () { + setState(() { + _isShowingPreview = true; + }); + }, + child: new Text("Start Live View"), + ), + ], + ); + } } } + +abstract class LiveViewCameraLoadState {} + +class LiveViewCameraLoadStateLoading extends LiveViewCameraLoadState {} + +class LiveViewCameraLoadStateLoaded extends LiveViewCameraLoadState { + final LiveViewCameraDescription cameraDescription; + + LiveViewCameraLoadStateLoaded(this.cameraDescription); +} + +class LiveViewCameraLoadStateReady extends LiveViewCameraLoadState { + final LiveViewCameraController controller; + + LiveViewCameraLoadStateReady(this.controller); + + void dispose() { + controller.dispose(); + } +} + +class LiveViewCameraLoadStateFailed extends LiveViewCameraLoadState { + final String errorMessage; + + LiveViewCameraLoadStateFailed(this.errorMessage); +} diff --git a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart index 3d7cb91d2c84..52db1e55ffca 100644 --- a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart +++ b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart @@ -9,6 +9,7 @@ import 'dart:io'; import 'dart:math'; import 'package:firebase_ml_vision/src/vision_model_utils.dart'; +import 'package:firebase_ml_vision/src/live_view.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 791ad20eed47..16d848131a1a 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -16,8 +16,9 @@ class FirebaseVision { FirebaseVision._(); @visibleForTesting - static const MethodChannel channel = - const MethodChannel('plugins.flutter.io/firebase_ml_vision'); + static final MethodChannel channel = + const MethodChannel('plugins.flutter.io/firebase_ml_vision') + ..invokeMethod('init'); /// Singleton of [FirebaseVision]. /// diff --git a/packages/firebase_ml_vision/lib/src/live_view.dart b/packages/firebase_ml_vision/lib/src/live_view.dart new file mode 100644 index 000000000000..eea74ce9b980 --- /dev/null +++ b/packages/firebase_ml_vision/lib/src/live_view.dart @@ -0,0 +1,249 @@ +import 'dart:async'; + +import 'package:firebase_ml_vision/firebase_ml_vision.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; + +enum LiveViewCameraLensDirection { front, back, external } + +enum LiveViewResolutionPreset { low, medium, high } + +/// Returns the resolution preset as a String. +String serializeResolutionPreset(LiveViewResolutionPreset resolutionPreset) { + switch (resolutionPreset) { + case LiveViewResolutionPreset.high: + return 'high'; + case LiveViewResolutionPreset.medium: + return 'medium'; + case LiveViewResolutionPreset.low: + return 'low'; + } + throw new ArgumentError('Unknown ResolutionPreset value'); +} + +LiveViewCameraLensDirection _parseCameraLensDirection(String string) { + switch (string) { + case 'front': + return LiveViewCameraLensDirection.front; + case 'back': + return LiveViewCameraLensDirection.back; + case 'external': + return LiveViewCameraLensDirection.external; + } + throw new ArgumentError('Unknown CameraLensDirection value'); +} + +/// Completes with a list of available cameras. +/// +/// May throw a [LiveViewCameraException]. +Future> availableCameras() async { + try { + final List cameras = + await FirebaseVision.channel.invokeMethod('availableCameras'); + return cameras.map((dynamic camera) { + return new LiveViewCameraDescription( + name: camera['name'], + lensDirection: _parseCameraLensDirection(camera['lensFacing']), + ); + }).toList(); + } on PlatformException catch (e) { + throw new LiveViewCameraException(e.code, e.message); + } +} + +class LiveViewCameraDescription { + final String name; + final LiveViewCameraLensDirection lensDirection; + + LiveViewCameraDescription({this.name, this.lensDirection}); + + @override + bool operator ==(Object o) { + return o is LiveViewCameraDescription && + o.name == name && + o.lensDirection == lensDirection; + } + + @override + int get hashCode { + return hashValues(name, lensDirection); + } + + @override + String toString() { + return '$runtimeType($name, $lensDirection)'; + } +} + +/// This is thrown when the plugin reports an error. +class LiveViewCameraException implements Exception { + String code; + String description; + + LiveViewCameraException(this.code, this.description); + + @override + String toString() => '$runtimeType($code, $description)'; +} + +// Build the UI texture view of the video data with textureId. +class LiveView extends StatelessWidget { + final LiveViewCameraController controller; + + const LiveView(this.controller); + + @override + Widget build(BuildContext context) { + return controller.value.isInitialized + ? new Texture(textureId: controller._textureId) + : new Container(); + } +} + +/// The state of a [LiveViewCameraController]. +class LiveViewCameraValue { + /// True after [LiveViewCameraController.initialize] has completed successfully. + final bool isInitialized; + + final String errorDescription; + + /// The size of the preview in pixels. + /// + /// Is `null` until [isInitialized] is `true`. + final Size previewSize; + + const LiveViewCameraValue({ + this.isInitialized, + this.errorDescription, + this.previewSize, + }); + + const LiveViewCameraValue.uninitialized() + : this( + isInitialized: false, + ); + + /// Convenience getter for `previewSize.height / previewSize.width`. + /// + /// Can only be called when [initialize] is done. + double get aspectRatio => previewSize.height / previewSize.width; + + bool get hasError => errorDescription != null; + + LiveViewCameraValue copyWith({ + bool isInitialized, + bool isRecordingVideo, + bool isTakingPicture, + String errorDescription, + Size previewSize, + }) { + return new LiveViewCameraValue( + isInitialized: isInitialized ?? this.isInitialized, + errorDescription: errorDescription, + previewSize: previewSize ?? this.previewSize, + ); + } + + @override + String toString() { + return '$runtimeType(' + 'isInitialized: $isInitialized, ' + 'errorDescription: $errorDescription, ' + 'previewSize: $previewSize)'; + } +} + +/// Controls a device camera live view. +/// +/// Use [availableCameras] to get a list of available cameras. +/// +/// Before using a [LiveViewCameraController] a call to [initialize] must complete. +/// +/// To show the camera preview on the screen use a [LiveView] widget. +class LiveViewCameraController extends ValueNotifier { + final LiveViewCameraDescription description; + final LiveViewResolutionPreset resolutionPreset; + + int _textureId; + bool _isDisposed = false; + StreamSubscription _eventSubscription; + Completer _creatingCompleter; + + LiveViewCameraController(this.description, this.resolutionPreset) + : super(const LiveViewCameraValue.uninitialized()); + + /// Initializes the camera on the device. + /// + /// Throws a [LiveViewCameraException] if the initialization fails. + Future initialize() async { + if (_isDisposed) { + return new Future.value(null); + } + try { + _creatingCompleter = new Completer(); + final Map reply = await FirebaseVision.channel.invokeMethod( + 'initialize', + { + 'cameraName': description.name, + 'resolutionPreset': serializeResolutionPreset(resolutionPreset), + }, + ); + _textureId = reply['textureId']; + value = value.copyWith( + isInitialized: true, + previewSize: new Size( + reply['previewWidth'].toDouble(), + reply['previewHeight'].toDouble(), + ), + ); + } on PlatformException catch (e) { + throw new LiveViewCameraException(e.code, e.message); + } + _eventSubscription = + new EventChannel('flutter.io/cameraPlugin/cameraEvents$_textureId') + .receiveBroadcastStream() + .listen(_listener); + _creatingCompleter.complete(null); + return _creatingCompleter.future; + } + + /// Listen to events from the native plugins. + /// + /// A "cameraClosing" event is sent when the camera is closed automatically by the system (for example when the app go to background). The plugin will try to reopen the camera automatically but any ongoing recording will end. + void _listener(dynamic event) { + final Map map = event; + if (_isDisposed) { + return; + } + + switch (map['eventType']) { + case 'error': + value = value.copyWith(errorDescription: event['errorDescription']); + break; + case 'cameraClosing': + value = value.copyWith(isRecordingVideo: false); + break; + } + } + + /// Releases the resources of this camera. + @override + Future dispose() async { + if (_isDisposed) { + return new Future.value(null); + } + _isDisposed = true; + super.dispose(); + if (_creatingCompleter == null) { + return new Future.value(null); + } else { + return _creatingCompleter.future.then((_) async { + await FirebaseVision.channel.invokeMethod( + 'dispose', + {'textureId': _textureId}, + ); + await _eventSubscription?.cancel(); + }); + } + } +} From 973b32c40ab76f7ccdacb7106d1ff8f63d4c449d Mon Sep 17 00:00:00 2001 From: dustin Date: Mon, 9 Jul 2018 10:47:03 -0700 Subject: [PATCH 06/34] resume camera preview make stream handling more appealing --- .../FirebaseMlVisionPlugin.java | 12 +-- .../plugins/firebasemlvision/live/Camera.java | 2 +- .../example/lib/live_preview.dart | 79 +++++++------------ 3 files changed, 37 insertions(+), 56 deletions(-) diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 07296f492381..14c6d71e20f1 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -64,12 +64,11 @@ public void onActivityResumed(Activity activity) { camera.setRequestingPermission(false); return; } - // TODO: figure out how to open camera iff the app is presenting a live preview -// if (activity == FirebaseMlVisionPlugin.this.activity) { -// if (camera != null) { -// camera.open(null); -// } -// } + if (activity == FirebaseMlVisionPlugin.this.activity) { + if (camera != null) { + camera.open(null); + } + } } @Override @@ -137,6 +136,7 @@ public void onMethodCall(MethodCall call, Result result) { case "dispose": { if (camera != null) { camera.dispose(); + camera = null; } result.success(null); break; diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index 42fed95badc9..77b89f50aafc 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -271,7 +271,7 @@ private void prepareMediaRecorder(String outputFilePath) throws IOException { mediaRecorder.prepare(); } - private void open(@Nullable final MethodChannel.Result result) { + public void open(@Nullable final MethodChannel.Result result) { if (!hasCameraPermission()) { if (result != null) result.error("cameraPermission", "Camera permission not granted", null); } else { diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index cf54ba9e725f..2b95bc597e36 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -39,6 +39,7 @@ class LivePreviewState extends State { await controller.initialize(); yield new LiveViewCameraLoadStateReady(controller); } on LiveViewCameraException catch (e) { + print("got an error: $e"); yield new LiveViewCameraLoadStateFailed( "error initializing camera controller: ${e.toString()}"); } @@ -50,58 +51,38 @@ class LivePreviewState extends State { @override Widget build(BuildContext context) { - if (_isShowingPreview) { - return new StreamBuilder( - stream: _prepareCameraPreview(), - builder: (BuildContext context, - AsyncSnapshot snapshot) { - print("snapshot data: ${snapshot.data}"); - final loadState = snapshot.data; - if (loadState != null) { - if (loadState is LiveViewCameraLoadStateLoading) { - return const Text("loading camera preview…"); - } else if (loadState is LiveViewCameraLoadStateLoaded) { - // get rid of previous controller if there is one - return new Text("loaded camera name: ${loadState - .cameraDescription - .name}"); - } else if (loadState is LiveViewCameraLoadStateReady) { - ////// BINGO!!!, the camera is ready to present - if (_readyLoadState != loadState) { - _readyLoadState?.dispose(); - _readyLoadState = loadState; - } - return new AspectRatio( - aspectRatio: _readyLoadState.controller.value.aspectRatio, - child: new LiveView(_readyLoadState.controller), - ); - } else if (loadState is LiveViewCameraLoadStateFailed) { - return new Text("error loading camera ${loadState - .errorMessage}"); - } else { - return const Text("Unknown Camera error"); + return new StreamBuilder( + stream: _prepareCameraPreview(), + initialData: new LiveViewCameraLoadStateLoading(), + builder: (BuildContext context, + AsyncSnapshot snapshot) { + final loadState = snapshot.data; + if (loadState != null) { + if (loadState is LiveViewCameraLoadStateLoading || + loadState is LiveViewCameraLoadStateLoaded) { + return const Text("loading camera preview…"); + } + if (loadState is LiveViewCameraLoadStateReady) { + ////// BINGO!!!, the camera is ready to present + if (_readyLoadState != loadState) { + _readyLoadState?.dispose(); + _readyLoadState = loadState; } + return new AspectRatio( + aspectRatio: _readyLoadState.controller.value.aspectRatio, + child: new LiveView(_readyLoadState.controller), + ); + } else if (loadState is LiveViewCameraLoadStateFailed) { + return new Text("error loading camera ${loadState + .errorMessage}"); } else { - return new Text("Camera error: ${snapshot.error.toString()}"); + return const Text("Unknown Camera error"); } - }, - ); - } else { - return Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Text("Current detector: ${widget.detector}"), - RaisedButton( - onPressed: () { - setState(() { - _isShowingPreview = true; - }); - }, - child: new Text("Start Live View"), - ), - ], - ); - } + } else { + return new Text("Camera error: ${snapshot.error.toString()}"); + } + }, + ); } } From 86899938a20856dbd8c7b52862798b1fb656d897 Mon Sep 17 00:00:00 2001 From: dustin Date: Mon, 9 Jul 2018 10:56:00 -0700 Subject: [PATCH 07/34] strip out picture taking and video recording from Android camera impl --- .../plugins/firebasemlvision/live/Camera.java | 236 +----------------- 1 file changed, 1 insertion(+), 235 deletions(-) diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index 77b89f50aafc..59843f813303 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -13,25 +13,17 @@ import android.hardware.camera2.CameraDevice; import android.hardware.camera2.CameraManager; import android.hardware.camera2.CameraMetadata; -import android.hardware.camera2.CaptureFailure; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.params.StreamConfigurationMap; -import android.media.Image; import android.media.ImageReader; import android.media.MediaRecorder; import android.os.Build; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.support.annotation.RequiresApi; -import android.util.Log; import android.util.Size; -import android.util.SparseIntArray; import android.view.Surface; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -49,37 +41,24 @@ @TargetApi(Build.VERSION_CODES.LOLLIPOP) public class Camera { - private static final SparseIntArray ORIENTATIONS = - new SparseIntArray() { - { - append(Surface.ROTATION_0, 0); - append(Surface.ROTATION_90, 90); - append(Surface.ROTATION_180, 180); - append(Surface.ROTATION_270, 270); - } - }; private final FlutterView.SurfaceTextureEntry textureEntry; private CameraDevice cameraDevice; private CameraCaptureSession cameraCaptureSession; private EventChannel.EventSink eventSink; private ImageReader imageReader; - private int sensorOrientation; - private boolean isFrontFacing; private String cameraName; private Size captureSize; private Size previewSize; private CaptureRequest.Builder captureRequestBuilder; - private Size videoSize; private MediaRecorder mediaRecorder; - private boolean recordingVideo; private Runnable cameraPermissionContinuation; private boolean requestingPermission; private PluginRegistry.Registrar registrar; private Activity activity; private CameraManager cameraManager; - public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonNull String resolutionPreset, @NonNull final MethodChannel.Result result) { + public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonNull final String resolutionPreset, @NonNull final MethodChannel.Result result) { this.activity = registrar.activity(); this.cameraManager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE); @@ -89,11 +68,6 @@ public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonN registerEventChannel(); - if (resolutionPreset == null) { - Log.e("ML", "resolution preset is somehow null"); - resolutionPreset = "high"; - } - try { Size minPreviewSize; switch (resolutionPreset) { @@ -113,12 +87,6 @@ public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonN CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraName); StreamConfigurationMap streamConfigurationMap = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); - //noinspection ConstantConditions - sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); - //noinspection ConstantConditions - isFrontFacing = - characteristics.get(CameraCharacteristics.LENS_FACING) - == CameraMetadata.LENS_FACING_FRONT; computeBestCaptureSize(streamConfigurationMap); computeBestPreviewAndRecordingSize(streamConfigurationMap, minPreviewSize, captureSize); @@ -135,11 +103,6 @@ public void run() { "cameraPermission", "MediaRecorderCamera permission not granted", null); return; } -// if (!hasAudioPermission()) { -// result.error( -// "cameraPermission", "MediaRecorderAudio permission not granted", null); -// return; -// } open(result); } }; @@ -197,12 +160,6 @@ private boolean hasCameraPermission() { || activity.checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; } -// -// private boolean hasAudioPermission() { -// return Build.VERSION.SDK_INT < Build.VERSION_CODES.M -// || registrar.activity().checkSelfPermission(Manifest.permission.RECORD_AUDIO) -// == PackageManager.PERMISSION_GRANTED; -// } @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) private void computeBestPreviewAndRecordingSize( @@ -222,15 +179,12 @@ private void computeBestPreviewAndRecordingSize( if (goodEnough.isEmpty()) { previewSize = sizes[0]; - videoSize = sizes[0]; } else { previewSize = goodEnough.get(0); // Video capture size should not be greater than 1080 because MediaRecorder cannot handle higher resolutions. - videoSize = goodEnough.get(0); for (int i = goodEnough.size() - 1; i >= 0; i--) { if (goodEnough.get(i).getHeight() <= 1080) { - videoSize = goodEnough.get(i); break; } } @@ -246,31 +200,6 @@ private void computeBestCaptureSize(StreamConfigurationMap streamConfigurationMa new CompareSizesByArea()); } - @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) - private void prepareMediaRecorder(String outputFilePath) throws IOException { - if (mediaRecorder != null) { - mediaRecorder.release(); - } - mediaRecorder = new MediaRecorder(); - mediaRecorder.setAudioSource(MediaRecorder.AudioSource.MIC); - mediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); - mediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4); - mediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC); - mediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); - mediaRecorder.setVideoEncodingBitRate(1024 * 1000); - mediaRecorder.setAudioSamplingRate(16000); - mediaRecorder.setVideoFrameRate(27); - mediaRecorder.setVideoSize(videoSize.getWidth(), videoSize.getHeight()); - mediaRecorder.setOutputFile(outputFilePath); - - int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation(); - int displayOrientation = ORIENTATIONS.get(displayRotation); - if (isFrontFacing) displayOrientation = -displayOrientation; - mediaRecorder.setOrientationHint((displayOrientation + sensorOrientation) % 360); - - mediaRecorder.prepare(); - } - public void open(@Nullable final MethodChannel.Result result) { if (!hasCameraPermission()) { if (result != null) result.error("cameraPermission", "Camera permission not granted", null); @@ -352,162 +281,7 @@ public void onError(@NonNull CameraDevice cameraDevice, int errorCode) { } } - private void writeToFile(ByteBuffer buffer, File file) throws IOException { - try (FileOutputStream outputStream = new FileOutputStream(file)) { - while (0 < buffer.remaining()) { - outputStream.getChannel().write(buffer); - } - } - } - - private void takePicture(String filePath, @NonNull final MethodChannel.Result result) { - final File file = new File(filePath); - - if (file.exists()) { - result.error( - "fileExists", - "File at path '" + filePath + "' already exists. Cannot overwrite.", - null); - return; - } - - imageReader.setOnImageAvailableListener( - new ImageReader.OnImageAvailableListener() { - @Override - public void onImageAvailable(ImageReader reader) { - try (Image image = reader.acquireLatestImage()) { - ByteBuffer buffer = image.getPlanes()[0].getBuffer(); - writeToFile(buffer, file); - result.success(null); - } catch (IOException e) { - result.error("IOError", "Failed saving image", null); - } - } - }, - null); - - try { - final CaptureRequest.Builder captureBuilder = - cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE); - captureBuilder.addTarget(imageReader.getSurface()); - int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation(); - int displayOrientation = ORIENTATIONS.get(displayRotation); - if (isFrontFacing) displayOrientation = -displayOrientation; - captureBuilder.set( - CaptureRequest.JPEG_ORIENTATION, (-displayOrientation + sensorOrientation) % 360); - - cameraCaptureSession.capture( - captureBuilder.build(), - new CameraCaptureSession.CaptureCallback() { - @Override - public void onCaptureFailed( - @NonNull CameraCaptureSession session, - @NonNull CaptureRequest request, - @NonNull CaptureFailure failure) { - String reason; - switch (failure.getReason()) { - case CaptureFailure.REASON_ERROR: - reason = "An error happened in the framework"; - break; - case CaptureFailure.REASON_FLUSHED: - reason = "The capture has failed due to an abortCaptures() call"; - break; - default: - reason = "Unknown reason"; - } - result.error("captureFailure", reason, null); - } - }, - null); - } catch (CameraAccessException e) { - result.error("cameraAccess", e.getMessage(), null); - } - } - - private void startVideoRecording(String filePath, @NonNull final MethodChannel.Result result) { - if (cameraDevice == null) { - result.error("configureFailed", "Camera was closed during configuration.", null); - return; - } - if (new File(filePath).exists()) { - result.error( - "fileExists", - "File at path '" + filePath + "' already exists. Cannot overwrite.", - null); - return; - } - try { - closeCaptureSession(); - prepareMediaRecorder(filePath); - - recordingVideo = true; - - SurfaceTexture surfaceTexture = textureEntry.surfaceTexture(); - surfaceTexture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); - captureRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); - - List surfaces = new ArrayList<>(); - - Surface previewSurface = new Surface(surfaceTexture); - surfaces.add(previewSurface); - captureRequestBuilder.addTarget(previewSurface); - - Surface recorderSurface = mediaRecorder.getSurface(); - surfaces.add(recorderSurface); - captureRequestBuilder.addTarget(recorderSurface); - - cameraDevice.createCaptureSession( - surfaces, - new CameraCaptureSession.StateCallback() { - @Override - public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) { - try { - if (cameraDevice == null) { - result.error("configureFailed", "Camera was closed during configuration", null); - return; - } - Camera.this.cameraCaptureSession = cameraCaptureSession; - captureRequestBuilder.set( - CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO); - cameraCaptureSession.setRepeatingRequest( - captureRequestBuilder.build(), null, null); - mediaRecorder.start(); - result.success(null); - } catch (CameraAccessException e) { - result.error("cameraAccess", e.getMessage(), null); - } - } - - @Override - public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) { - result.error("configureFailed", "Failed to configure camera session", null); - } - }, - null); - } catch (CameraAccessException | IOException e) { - result.error("videoRecordingFailed", e.getMessage(), null); - } - } - - private void stopVideoRecording(@NonNull final MethodChannel.Result result) { - if (!recordingVideo) { - result.success(null); - return; - } - - try { - recordingVideo = false; - mediaRecorder.stop(); - mediaRecorder.reset(); - startPreview(); - result.success(null); - } catch (CameraAccessException | IllegalStateException e) { - result.error("videoRecordingFailed", e.getMessage(), null); - } - } - private void startPreview() throws CameraAccessException { - closeCaptureSession(); SurfaceTexture surfaceTexture = textureEntry.surfaceTexture(); surfaceTexture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); @@ -558,15 +332,7 @@ private void sendErrorEvent(String errorDescription) { } } - private void closeCaptureSession() { - if (cameraCaptureSession != null) { - cameraCaptureSession.close(); - cameraCaptureSession = null; - } - } - public void close() { - closeCaptureSession(); if (cameraDevice != null) { cameraDevice.close(); From beaaa5fab7c71c0186888e6c526eff09ff07d2aa Mon Sep 17 00:00:00 2001 From: dustin Date: Mon, 9 Jul 2018 13:58:50 -0700 Subject: [PATCH 08/34] android: insert per-frame handling of preview images this is in preparation to integrate ML Kit recognition of frames. --- .../plugins/firebasemlvision/live/Camera.java | 57 ++++++++++++++++++- .../example/lib/live_preview.dart | 7 +++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index 59843f813303..1c6dea4aefca 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -14,13 +14,18 @@ import android.hardware.camera2.CameraManager; import android.hardware.camera2.CameraMetadata; import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.params.StreamConfigurationMap; +import android.media.Image; import android.media.ImageReader; import android.media.MediaRecorder; import android.os.Build; +import android.os.Handler; +import android.os.HandlerThread; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.support.annotation.RequiresApi; +import android.util.Log; import android.util.Size; import android.view.Surface; @@ -57,6 +62,9 @@ public class Camera { private PluginRegistry.Registrar registrar; private Activity activity; private CameraManager cameraManager; + private HandlerThread mBackgroundThread; + private Handler mBackgroundHandler; + private Surface imageReaderSurface; public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonNull final String resolutionPreset, @NonNull final MethodChannel.Result result) { @@ -200,14 +208,53 @@ private void computeBestCaptureSize(StreamConfigurationMap streamConfigurationMa new CompareSizesByArea()); } + /** + * Starts a background thread and its {@link Handler}. + */ + private void startBackgroundThread() { + mBackgroundThread = new HandlerThread("CameraBackground"); + mBackgroundThread.start(); + mBackgroundHandler = new Handler(mBackgroundThread.getLooper()); + } + + /** + * Stops the background thread and its {@link Handler}. + */ + private void stopBackgroundThread() { + mBackgroundThread.quitSafely(); + try { + mBackgroundThread.join(); + mBackgroundThread = null; + mBackgroundHandler = null; + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + ImageReader.OnImageAvailableListener imageAvailable = new ImageReader.OnImageAvailableListener() { + @Override + public void onImageAvailable(ImageReader reader) { + Image image = reader.acquireLatestImage(); + if (image == null) + return; + + Log.d("ML", "got an image from the image reader"); + + image.close(); + } + }; + public void open(@Nullable final MethodChannel.Result result) { if (!hasCameraPermission()) { if (result != null) result.error("cameraPermission", "Camera permission not granted", null); } else { try { + startBackgroundThread(); imageReader = ImageReader.newInstance( - captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2); + captureSize.getWidth(), captureSize.getHeight(), ImageFormat.YUV_420_888, 4); + imageReaderSurface = imageReader.getSurface(); + imageReader.setOnImageAvailableListener(imageAvailable, mBackgroundHandler); cameraManager.openCamera( cameraName, new CameraDevice.StateCallback() { @@ -293,7 +340,8 @@ private void startPreview() throws CameraAccessException { surfaces.add(previewSurface); captureRequestBuilder.addTarget(previewSurface); - surfaces.add(imageReader.getSurface()); + surfaces.add(imageReaderSurface); + captureRequestBuilder.addTarget(imageReaderSurface); cameraDevice.createCaptureSession( surfaces, @@ -333,7 +381,9 @@ private void sendErrorEvent(String errorDescription) { } public void close() { - + if (cameraCaptureSession != null) { + cameraCaptureSession.close(); + } if (cameraDevice != null) { cameraDevice.close(); cameraDevice = null; @@ -347,6 +397,7 @@ public void close() { mediaRecorder.release(); mediaRecorder = null; } + stopBackgroundThread(); } public void dispose() { diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 2b95bc597e36..7d117df81251 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -49,6 +49,13 @@ class LivePreviewState extends State { } } + + @override + void dispose() { + super.dispose(); + _readyLoadState?.controller.dispose(); + } + @override Widget build(BuildContext context) { return new StreamBuilder( From 1ad5ae7cdd1dd774d18603493b9909802f8fd538 Mon Sep 17 00:00:00 2001 From: dustin Date: Mon, 9 Jul 2018 15:57:52 -0700 Subject: [PATCH 09/34] android: pipe image frames to MLKit Barcode detector not yet returning any data to the Flutter app. --- .../plugins/firebasemlvision/live/Camera.java | 139 ++++++++++++++++-- 1 file changed, 130 insertions(+), 9 deletions(-) diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index 1c6dea4aefca..a563fd445098 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -14,7 +14,6 @@ import android.hardware.camera2.CameraManager; import android.hardware.camera2.CameraMetadata; import android.hardware.camera2.CaptureRequest; -import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.params.StreamConfigurationMap; import android.media.Image; import android.media.ImageReader; @@ -27,8 +26,19 @@ import android.support.annotation.RequiresApi; import android.util.Log; import android.util.Size; +import android.util.SparseIntArray; import android.view.Surface; +import android.view.WindowManager; +import com.google.android.gms.tasks.OnFailureListener; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.firebase.ml.vision.FirebaseVision; +import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode; +import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector; +import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; + +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -36,6 +46,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; @@ -46,6 +57,14 @@ @TargetApi(Build.VERSION_CODES.LOLLIPOP) public class Camera { + private static final SparseIntArray ORIENTATIONS = new SparseIntArray(4); + + static { + ORIENTATIONS.append(Surface.ROTATION_0, 90); + ORIENTATIONS.append(Surface.ROTATION_90, 0); + ORIENTATIONS.append(Surface.ROTATION_180, 270); + ORIENTATIONS.append(Surface.ROTATION_270, 180); + } private final FlutterView.SurfaceTextureEntry textureEntry; private CameraDevice cameraDevice; @@ -65,6 +84,8 @@ public class Camera { private HandlerThread mBackgroundThread; private Handler mBackgroundHandler; private Surface imageReaderSurface; + private CameraCharacteristics cameraCharacteristics; + private WindowManager windowManager; public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonNull final String resolutionPreset, @NonNull final MethodChannel.Result result) { @@ -92,9 +113,10 @@ public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonN throw new IllegalArgumentException("Unknown preset: " + resolutionPreset); } - CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraName); + cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraName); StreamConfigurationMap streamConfigurationMap = - characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + cameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + computeBestCaptureSize(streamConfigurationMap); computeBestPreviewAndRecordingSize(streamConfigurationMap, minPreviewSize, captureSize); @@ -231,16 +253,115 @@ private void stopBackgroundThread() { } } + private static ByteBuffer YUV_420_888toNV21(Image image) { + byte[] nv21; + ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); + ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); + ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); + + int ySize = yBuffer.remaining(); + int uSize = uBuffer.remaining(); + int vSize = vBuffer.remaining(); + + ByteBuffer output = ByteBuffer.allocate(ySize + uSize + vSize) + .put(yBuffer) + .put(vBuffer) + .put(uBuffer); + return output; + +// nv21 = new byte[ySize + uSize + vSize]; +// +// //U and V are swapped +// yBuffer.get(nv21, 0, ySize); +// vBuffer.get(nv21, ySize, vSize); +// uBuffer.get(nv21, ySize + vSize, uSize); +// +// return nv21; + } + + private int getRotation() { + if (windowManager == null) { + windowManager = (WindowManager) activity.getSystemService(Context.WINDOW_SERVICE); + } + int degrees = 0; + int rotation = windowManager.getDefaultDisplay().getRotation(); + switch (rotation) { + case Surface.ROTATION_0: + degrees = 0; + break; + case Surface.ROTATION_90: + degrees = 90; + break; + case Surface.ROTATION_180: + degrees = 180; + break; + case Surface.ROTATION_270: + degrees = 270; + break; + default: + Log.e("ML", "Bad rotation value: $rotation"); + } + + try { + int angle; + int displayAngle; // TODO? setDisplayOrientation? + CameraCharacteristics cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraName); + Integer orientation = cameraCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); + // back-facing + angle = (orientation - degrees + 360) % 360; + displayAngle = angle; + int translatedAngle = angle / 90; + Log.d("ML", "Translated angle: " + translatedAngle); + return translatedAngle; // this corresponds to the rotation constants + } catch (CameraAccessException e) { + return 0; + } + } + + private AtomicBoolean shouldThrottle = new AtomicBoolean(false); + + private void processImage(Image image) { + if (shouldThrottle.get()) { +// Log.d("ML", "should throttle"); + return; + } + Log.d("ML", "about to process a vision frame"); + shouldThrottle.set(true); + ByteBuffer imageBuffer = YUV_420_888toNV21(image); + FirebaseVisionImageMetadata metadata = new FirebaseVisionImageMetadata.Builder() + .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(getRotation()) + .build(); + FirebaseVisionImage firebaseVisionImage = FirebaseVisionImage.fromByteBuffer(imageBuffer, metadata); + + FirebaseVisionBarcodeDetector visionBarcodeDetector = FirebaseVision.getInstance().getVisionBarcodeDetector(); + visionBarcodeDetector.detectInImage(firebaseVisionImage).addOnSuccessListener(new OnSuccessListener>() { + @Override + public void onSuccess(List firebaseVisionBarcodes) { + shouldThrottle.set(false); + Log.d("ML", "barcode scan success, got codes: " + firebaseVisionBarcodes.size()); + } + }).addOnFailureListener(new OnFailureListener() { + @Override + public void onFailure(@NonNull Exception e) { + shouldThrottle.set(false); + Log.d("ML", "barcode scan failure, message: " + e.getMessage()); + } + }); + Log.d("ML", "got an image from the image reader"); + } + ImageReader.OnImageAvailableListener imageAvailable = new ImageReader.OnImageAvailableListener() { @Override public void onImageAvailable(ImageReader reader) { Image image = reader.acquireLatestImage(); - if (image == null) - return; - - Log.d("ML", "got an image from the image reader"); - - image.close(); + if (image != null) { +// Log.d("ML", "image was not null"); + processImage(image); + image.close(); + } } }; From 76930fcd3b07d82a115528b771a8fd18c7336deb Mon Sep 17 00:00:00 2001 From: dustin Date: Tue, 10 Jul 2018 10:09:09 -0700 Subject: [PATCH 10/34] android: send count of recognized barcodes --- .../plugins/firebasemlvision/live/Camera.java | 31 +++++-- .../firebase_ml_vision/lib/src/live_view.dart | 87 +++++++++++++++++-- 2 files changed, 100 insertions(+), 18 deletions(-) diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index a563fd445098..559b3e95bac9 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -170,7 +170,7 @@ public void setRequestingPermission(boolean isRequesting) { private void registerEventChannel() { new EventChannel( - registrar.messenger(), "flutter.io/cameraPlugin/cameraEvents" + textureEntry.id()) + registrar.messenger(), "plugins.flutter.io/firebase_ml_vision/liveViewEvents" + textureEntry.id()) .setStreamHandler( new EventChannel.StreamHandler() { @Override @@ -243,13 +243,15 @@ private void startBackgroundThread() { * Stops the background thread and its {@link Handler}. */ private void stopBackgroundThread() { - mBackgroundThread.quitSafely(); - try { - mBackgroundThread.join(); - mBackgroundThread = null; - mBackgroundHandler = null; - } catch (InterruptedException e) { - e.printStackTrace(); + if (mBackgroundThread != null) { + mBackgroundThread.quitSafely(); + try { + mBackgroundThread.join(); + mBackgroundThread = null; + mBackgroundHandler = null; + } catch (InterruptedException e) { + e.printStackTrace(); + } } } @@ -341,12 +343,14 @@ private void processImage(Image image) { @Override public void onSuccess(List firebaseVisionBarcodes) { shouldThrottle.set(false); + sendRecognizedCount(firebaseVisionBarcodes.size()); Log.d("ML", "barcode scan success, got codes: " + firebaseVisionBarcodes.size()); } }).addOnFailureListener(new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { shouldThrottle.set(false); + sendErrorEvent(e.getLocalizedMessage()); Log.d("ML", "barcode scan failure, message: " + e.getMessage()); } }); @@ -373,7 +377,7 @@ public void open(@Nullable final MethodChannel.Result result) { startBackgroundThread(); imageReader = ImageReader.newInstance( - captureSize.getWidth(), captureSize.getHeight(), ImageFormat.YUV_420_888, 4); + previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 4); imageReaderSurface = imageReader.getSurface(); imageReader.setOnImageAvailableListener(imageAvailable, mBackgroundHandler); cameraManager.openCamera( @@ -492,6 +496,15 @@ public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession null); } + private void sendRecognizedCount(int count) { + if (eventSink != null) { + Map event = new HashMap<>(); + event.put("eventType", "recognized"); + event.put("count", String.valueOf(count)); + eventSink.success(event); + } + } + private void sendErrorEvent(String errorDescription) { if (eventSink != null) { Map event = new HashMap<>(); diff --git a/packages/firebase_ml_vision/lib/src/live_view.dart b/packages/firebase_ml_vision/lib/src/live_view.dart index eea74ce9b980..b402530aa346 100644 --- a/packages/firebase_ml_vision/lib/src/live_view.dart +++ b/packages/firebase_ml_vision/lib/src/live_view.dart @@ -87,15 +87,74 @@ class LiveViewCameraException implements Exception { } // Build the UI texture view of the video data with textureId. -class LiveView extends StatelessWidget { +class LiveView extends StatefulWidget { final LiveViewCameraController controller; const LiveView(this.controller); + @override + LiveViewState createState() { + return new LiveViewState(); + } +} + +class LiveViewState extends State { + List scannedCodeContainers = []; + int scannedCodes = 0; + StreamSubscription _eventSubscription; + + bool _isDisposed = false; + +// void _listener(dynamic event) { +// print("liveView state got an event: $event"); +// final Map map = event; +// if (_isDisposed) { +// return; +// } +// +// switch (map['eventType']) { +// case 'recognized': +// // TODO: parse barcode +// final int recognizedCount = int.parse(event['count']); +// print("got $recognizedCount codes"); +// scannedItemStreamController.add(recognizedCount); +// break; +// case 'cameraClosing': +// scannedItemStreamController.close(); +// break; +// } +// } + + @override + void initState() { + super.initState(); + widget.controller.addListener(() { + setState(() { + scannedCodes = widget.controller.value.recognizedCount; + }); + }); + } +// +// @override +// void dispose() { +// super.dispose(); +// _isDisposed = true; +// _eventSubscription?.cancel(); +// } + @override Widget build(BuildContext context) { - return controller.value.isInitialized - ? new Texture(textureId: controller._textureId) + return widget.controller.value.isInitialized + ? new Stack( + children: [ + new Texture(textureId: widget.controller._textureId), + new Center( + child: new Text(scannedCodes == 0 + ? "No codes" + : "Got ${scannedCodes} codes!!!"), + ), + ], + ) : new Container(); } } @@ -112,10 +171,13 @@ class LiveViewCameraValue { /// Is `null` until [isInitialized] is `true`. final Size previewSize; + final int recognizedCount; + const LiveViewCameraValue({ this.isInitialized, this.errorDescription, this.previewSize, + this.recognizedCount, }); const LiveViewCameraValue.uninitialized() @@ -136,11 +198,13 @@ class LiveViewCameraValue { bool isTakingPicture, String errorDescription, Size previewSize, + int recognizedCount, }) { return new LiveViewCameraValue( isInitialized: isInitialized ?? this.isInitialized, errorDescription: errorDescription, previewSize: previewSize ?? this.previewSize, + recognizedCount: recognizedCount ?? this.recognizedCount, ); } @@ -181,7 +245,8 @@ class LiveViewCameraController extends ValueNotifier { } try { _creatingCompleter = new Completer(); - final Map reply = await FirebaseVision.channel.invokeMethod( + final Map reply = + await FirebaseVision.channel.invokeMethod( 'initialize', { 'cameraName': description.name, @@ -199,10 +264,10 @@ class LiveViewCameraController extends ValueNotifier { } on PlatformException catch (e) { throw new LiveViewCameraException(e.code, e.message); } - _eventSubscription = - new EventChannel('flutter.io/cameraPlugin/cameraEvents$_textureId') - .receiveBroadcastStream() - .listen(_listener); + _eventSubscription = new EventChannel( + 'plugins.flutter.io/firebase_ml_vision/liveViewEvents$_textureId') + .receiveBroadcastStream() + .listen(_listener); _creatingCompleter.complete(null); return _creatingCompleter.future; } @@ -215,8 +280,12 @@ class LiveViewCameraController extends ValueNotifier { if (_isDisposed) { return; } - + print("got an event: $event"); switch (map['eventType']) { + case 'recognized': + final int recognizedCount = int.parse(event['count']); + value = value.copyWith(recognizedCount: recognizedCount); + break; case 'error': value = value.copyWith(errorDescription: event['errorDescription']); break; From a0a1b62eabbbdaf14080c7437e2ee27c446c2cdc Mon Sep 17 00:00:00 2001 From: dustin Date: Tue, 10 Jul 2018 11:23:58 -0700 Subject: [PATCH 11/34] android: get barcode bounding boxes displaying in Flutter --- .../plugins/firebasemlvision/live/Camera.java | 26 +++++- .../example/lib/detector_painters.dart | 26 ++++++ .../example/lib/live_preview.dart | 22 ++++- .../firebase_ml_vision/example/lib/main.dart | 27 +------ .../lib/src/barcode_detector.dart | 12 ++- .../firebase_ml_vision/lib/src/live_view.dart | 81 +++++++------------ 6 files changed, 109 insertions(+), 85 deletions(-) diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index 559b3e95bac9..ad5ba8fc2e6f 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -6,6 +6,7 @@ import android.content.Context; import android.content.pm.PackageManager; import android.graphics.ImageFormat; +import android.graphics.Rect; import android.graphics.SurfaceTexture; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; @@ -51,9 +52,13 @@ import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugin.common.PluginRegistry; +import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; import io.flutter.view.FlutterView; import static io.flutter.plugins.firebasemlvision.FirebaseMlVisionPlugin.CAMERA_REQUEST_ID; +import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.BARCODE_DISPLAY_VALUE; +import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.BARCODE_RAW_VALUE; +import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.BARCODE_VALUE_TYPE; @TargetApi(Build.VERSION_CODES.LOLLIPOP) public class Camera { @@ -343,7 +348,7 @@ private void processImage(Image image) { @Override public void onSuccess(List firebaseVisionBarcodes) { shouldThrottle.set(false); - sendRecognizedCount(firebaseVisionBarcodes.size()); + sendRecognizedBarcodes(firebaseVisionBarcodes); Log.d("ML", "barcode scan success, got codes: " + firebaseVisionBarcodes.size()); } }).addOnFailureListener(new OnFailureListener() { @@ -496,11 +501,24 @@ public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession null); } - private void sendRecognizedCount(int count) { + private void sendRecognizedBarcodes(List barcodes) { if (eventSink != null) { - Map event = new HashMap<>(); + List> outputMap = new ArrayList<>(); + for (FirebaseVisionBarcode barcode : barcodes) { + Map barcodeData = new HashMap<>(); + Rect boundingBox = barcode.getBoundingBox(); + if (boundingBox != null) { + barcodeData.putAll(DetectedItemUtils.rectToFlutterMap(boundingBox)); + } + barcodeData.put(BARCODE_VALUE_TYPE, barcode.getValueType()); + barcodeData.put(BARCODE_DISPLAY_VALUE, barcode.getDisplayValue()); + barcodeData.put(BARCODE_RAW_VALUE, barcode.getRawValue()); + outputMap.add(barcodeData); + } + Map event = new HashMap<>(); event.put("eventType", "recognized"); - event.put("count", String.valueOf(count)); + event.put("recognitionType", "barcode"); + event.put("barcodeData", outputMap); eventSink.success(event); } } diff --git a/packages/firebase_ml_vision/example/lib/detector_painters.dart b/packages/firebase_ml_vision/example/lib/detector_painters.dart index fb2d734b9650..bfc164d39c32 100644 --- a/packages/firebase_ml_vision/example/lib/detector_painters.dart +++ b/packages/firebase_ml_vision/example/lib/detector_painters.dart @@ -7,6 +7,32 @@ import 'package:flutter/material.dart'; enum Detector { barcode, face, label, text } +CustomPaint customPaintForResults( + Detector detector, Size imageSize, List results) { + CustomPainter painter; + + switch (detector) { + case Detector.barcode: + painter = new BarcodeDetectorPainter(imageSize, results); + break; + case Detector.face: + painter = new FaceDetectorPainter(imageSize, results); + break; + case Detector.label: + painter = new LabelDetectorPainter(imageSize, results); + break; + case Detector.text: + painter = new TextDetectorPainter(imageSize, results); + break; + default: + break; + } + + return new CustomPaint( + painter: painter, + ); +} + class BarcodeDetectorPainter extends CustomPainter { BarcodeDetectorPainter(this.absoluteImageSize, this.results); diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 7d117df81251..449cba454701 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:firebase_ml_vision/firebase_ml_vision.dart'; import 'package:firebase_ml_vision/src/live_view.dart'; import 'package:firebase_ml_vision_example/detector_painters.dart'; import 'package:flutter/material.dart'; @@ -49,7 +50,6 @@ class LivePreviewState extends State { } } - @override void dispose() { super.dispose(); @@ -70,14 +70,30 @@ class LivePreviewState extends State { return const Text("loading camera preview…"); } if (loadState is LiveViewCameraLoadStateReady) { - ////// BINGO!!!, the camera is ready to present if (_readyLoadState != loadState) { _readyLoadState?.dispose(); _readyLoadState = loadState; } return new AspectRatio( aspectRatio: _readyLoadState.controller.value.aspectRatio, - child: new LiveView(_readyLoadState.controller), + child: new LiveView( + controller: _readyLoadState.controller, + overlayBuilder: (BuildContext context, Size previewSize, + List barcodes) { + return barcodes == null + ? const Center( + child: const Text( + 'Scanning...', + style: const TextStyle( + color: Colors.green, + fontSize: 30.0, + ), + ), + ) + : customPaintForResults( + Detector.barcode, previewSize, barcodes); + }, + ), ); } else if (loadState is LiveViewCameraLoadStateFailed) { return new Text("error loading camera ${loadState diff --git a/packages/firebase_ml_vision/example/lib/main.dart b/packages/firebase_ml_vision/example/lib/main.dart index 59d562b54d12..d183331e6bd0 100644 --- a/packages/firebase_ml_vision/example/lib/main.dart +++ b/packages/firebase_ml_vision/example/lib/main.dart @@ -113,31 +113,6 @@ class _MyHomePageState extends State<_MyHomePage> }); } - CustomPaint _buildResults(Size imageSize, List results) { - CustomPainter painter; - - switch (_currentDetector) { - case Detector.barcode: - painter = new BarcodeDetectorPainter(_imageSize, results); - break; - case Detector.face: - painter = new FaceDetectorPainter(_imageSize, results); - break; - case Detector.label: - painter = new LabelDetectorPainter(_imageSize, results); - break; - case Detector.text: - painter = new TextDetectorPainter(_imageSize, results); - break; - default: - break; - } - - return new CustomPaint( - painter: painter, - ); - } - Widget _buildImage() { return new Container( constraints: const BoxConstraints.expand(), @@ -157,7 +132,7 @@ class _MyHomePageState extends State<_MyHomePage> ), ), ) - : _buildResults(_imageSize, _scanResults), + : customPaintForResults(_currentDetector, _imageSize, _scanResults), ); } diff --git a/packages/firebase_ml_vision/lib/src/barcode_detector.dart b/packages/firebase_ml_vision/lib/src/barcode_detector.dart index 1dc44be096aa..30c5d276cc74 100644 --- a/packages/firebase_ml_vision/lib/src/barcode_detector.dart +++ b/packages/firebase_ml_vision/lib/src/barcode_detector.dart @@ -19,7 +19,7 @@ class BarcodeDetector extends FirebaseVisionDetector { 'BarcodeDetector#detectInImage', visionImage.imageFile.path); final List barcodes = []; reply.forEach((dynamic barcodeMap) { - barcodes.add(new BarcodeContainer._(barcodeMap)); + barcodes.add(new BarcodeContainer(barcodeMap)); }); return barcodes; } @@ -31,11 +31,19 @@ class BarcodeContainer { final String displayValue; final String rawValue; - BarcodeContainer._(Map data) + BarcodeContainer(Map data) : boundingBox = VisionModelUtils.mlRectToRectangle(data), valueType = data[barcodeValueType], displayValue = data[barcodeDisplayValue], rawValue = data[barcodeRawValue]; + + @override + String toString() { + return 'BarcodeContainer{boundingBox: $boundingBox,' + ' valueType: $valueType,' + ' displayValue: $displayValue,' + ' rawValue: $rawValue}'; + } } class BarcodeDetectorOptions {} diff --git a/packages/firebase_ml_vision/lib/src/live_view.dart b/packages/firebase_ml_vision/lib/src/live_view.dart index b402530aa346..0b5bb0a84dd5 100644 --- a/packages/firebase_ml_vision/lib/src/live_view.dart +++ b/packages/firebase_ml_vision/lib/src/live_view.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:firebase_ml_vision/firebase_ml_vision.dart'; +import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; @@ -86,11 +87,15 @@ class LiveViewCameraException implements Exception { String toString() => '$runtimeType($code, $description)'; } +typedef Widget OverlayBuilder(BuildContext context, Size previewImageSize, + List barcodes); + // Build the UI texture view of the video data with textureId. class LiveView extends StatefulWidget { final LiveViewCameraController controller; + final OverlayBuilder overlayBuilder; - const LiveView(this.controller); + const LiveView({this.controller, this.overlayBuilder}); @override LiveViewState createState() { @@ -99,62 +104,31 @@ class LiveView extends StatefulWidget { } class LiveViewState extends State { - List scannedCodeContainers = []; - int scannedCodes = 0; - StreamSubscription _eventSubscription; - - bool _isDisposed = false; - -// void _listener(dynamic event) { -// print("liveView state got an event: $event"); -// final Map map = event; -// if (_isDisposed) { -// return; -// } -// -// switch (map['eventType']) { -// case 'recognized': -// // TODO: parse barcode -// final int recognizedCount = int.parse(event['count']); -// print("got $recognizedCount codes"); -// scannedItemStreamController.add(recognizedCount); -// break; -// case 'cameraClosing': -// scannedItemStreamController.close(); -// break; -// } -// } + List scannedCodes = []; @override void initState() { super.initState(); widget.controller.addListener(() { setState(() { - scannedCodes = widget.controller.value.recognizedCount; + scannedCodes = widget.controller.value.scannedBarcodes; }); }); } -// -// @override -// void dispose() { -// super.dispose(); -// _isDisposed = true; -// _eventSubscription?.cancel(); -// } @override Widget build(BuildContext context) { return widget.controller.value.isInitialized ? new Stack( - children: [ - new Texture(textureId: widget.controller._textureId), - new Center( - child: new Text(scannedCodes == 0 - ? "No codes" - : "Got ${scannedCodes} codes!!!"), - ), - ], + children: [ + new Texture(textureId: widget.controller._textureId), + new Container( + constraints: const BoxConstraints.expand(), + child: widget.overlayBuilder( + context, widget.controller.value.previewSize, scannedCodes), ) + ], + ) : new Container(); } } @@ -171,13 +145,13 @@ class LiveViewCameraValue { /// Is `null` until [isInitialized] is `true`. final Size previewSize; - final int recognizedCount; + final List scannedBarcodes; const LiveViewCameraValue({ this.isInitialized, this.errorDescription, this.previewSize, - this.recognizedCount, + this.scannedBarcodes, }); const LiveViewCameraValue.uninitialized() @@ -198,13 +172,13 @@ class LiveViewCameraValue { bool isTakingPicture, String errorDescription, Size previewSize, - int recognizedCount, + List scannedBarcodes, }) { return new LiveViewCameraValue( isInitialized: isInitialized ?? this.isInitialized, errorDescription: errorDescription, previewSize: previewSize ?? this.previewSize, - recognizedCount: recognizedCount ?? this.recognizedCount, + scannedBarcodes: scannedBarcodes ?? this.scannedBarcodes, ); } @@ -213,7 +187,8 @@ class LiveViewCameraValue { return '$runtimeType(' 'isInitialized: $isInitialized, ' 'errorDescription: $errorDescription, ' - 'previewSize: $previewSize)'; + 'previewSize: $previewSize, ' + 'scannedBarcodes: $scannedBarcodes)'; } } @@ -280,11 +255,17 @@ class LiveViewCameraController extends ValueNotifier { if (_isDisposed) { return; } - print("got an event: $event"); switch (map['eventType']) { case 'recognized': - final int recognizedCount = int.parse(event['count']); - value = value.copyWith(recognizedCount: recognizedCount); + String recognitionType = event['recognitionType']; + if (recognitionType == "barcode") { + final List reply = event['barcodeData']; + final List barcodes = []; + reply.forEach((dynamic barcodeMap) { + barcodes.add(new BarcodeContainer(barcodeMap)); + }); + value = value.copyWith(scannedBarcodes: barcodes); + } break; case 'error': value = value.copyWith(errorDescription: event['errorDescription']); From 56272e420a2f3be1469cf2fca4936d9563ca434c Mon Sep 17 00:00:00 2001 From: dustin Date: Tue, 10 Jul 2018 14:30:06 -0700 Subject: [PATCH 12/34] ios: Add basic barcode scanning. --- .../ios/Runner.xcodeproj/project.pbxproj | 7 ++- .../xcshareddata/xcschemes/Runner.xcscheme | 8 +++- .../example/ios/Runner/AppDelegate.m | 2 + .../ios/Runner/Base.lproj/Main.storyboard | 14 ++++-- .../ios/Classes/BarcodeDetector.m | 47 +++++++++++++++++++ .../ios/Classes/FirebaseMlVisionPlugin.m | 3 ++ 6 files changed, 73 insertions(+), 8 deletions(-) diff --git a/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj b/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj index bd47eafb95e9..11f2fd5defde 100644 --- a/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj +++ b/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj @@ -186,6 +186,7 @@ TargetAttributes = { 97C146ED1CF9000F007C117D = { CreatedOnToolsVersion = 7.3.1; + DevelopmentTeam = XPX9B2R9P6; }; }; }; @@ -247,7 +248,7 @@ ); inputPaths = ( "${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh", - "${PODS_ROOT}/../.symlinks/flutter/ios/Flutter.framework", + "${PODS_ROOT}/../.symlinks/flutter/ios-release/Flutter.framework", ); name = "[CP] Embed Pods Frameworks"; outputPaths = ( @@ -297,10 +298,12 @@ ); inputPaths = ( "${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-resources.sh", + "${PODS_CONFIGURATION_BUILD_DIR}/GoogleMobileVision/GoogleMVFaceDetectorResources.bundle", "${PODS_CONFIGURATION_BUILD_DIR}/GoogleMobileVision/GoogleMVTextDetectorResources.bundle", ); name = "[CP] Copy Pods Resources"; outputPaths = ( + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleMVFaceDetectorResources.bundle", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleMVTextDetectorResources.bundle", ); runOnlyForDeploymentPostprocessing = 0; @@ -451,6 +454,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; + DEVELOPMENT_TEAM = XPX9B2R9P6; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -474,6 +478,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; + DEVELOPMENT_TEAM = XPX9B2R9P6; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", diff --git a/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index 1263ac84b105..851d32ee7b53 100644 --- a/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -26,7 +26,6 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - language = "" shouldUseLaunchSchemeArgsEnv = "YES"> @@ -46,7 +45,6 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - language = "" launchStyle = "0" useCustomWorkingDirectory = "NO" ignoresPersistentStateOnLaunch = "NO" @@ -63,6 +61,12 @@ ReferencedContainer = "container:Runner.xcodeproj"> + + + + diff --git a/packages/firebase_ml_vision/example/ios/Runner/AppDelegate.m b/packages/firebase_ml_vision/example/ios/Runner/AppDelegate.m index 59a72e90be12..87b0634370a7 100644 --- a/packages/firebase_ml_vision/example/ios/Runner/AppDelegate.m +++ b/packages/firebase_ml_vision/example/ios/Runner/AppDelegate.m @@ -1,10 +1,12 @@ #include "AppDelegate.h" #include "GeneratedPluginRegistrant.h" +@import Firebase; @implementation AppDelegate - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + [FIRApp configure]; [GeneratedPluginRegistrant registerWithRegistry:self]; // Override point for customization after application launch. return [super application:application didFinishLaunchingWithOptions:launchOptions]; diff --git a/packages/firebase_ml_vision/example/ios/Runner/Base.lproj/Main.storyboard b/packages/firebase_ml_vision/example/ios/Runner/Base.lproj/Main.storyboard index f3c28516fb38..5e371aca7cc9 100644 --- a/packages/firebase_ml_vision/example/ios/Runner/Base.lproj/Main.storyboard +++ b/packages/firebase_ml_vision/example/ios/Runner/Base.lproj/Main.storyboard @@ -1,8 +1,12 @@ - - + + + + + - + + @@ -14,9 +18,9 @@ - + - + diff --git a/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m b/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m index a7794501d883..b48274365a99 100644 --- a/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m @@ -1,4 +1,51 @@ #import "FirebaseMlVisionPlugin.h" @implementation BarcodeDetector +static FIRVisionBarcodeDetector *barcodeDetector; + ++ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { + if (barcodeDetector == nil) { + FIRVision *vision = [FIRVision vision]; + barcodeDetector = [vision barcodeDetector]; + } + + [barcodeDetector + detectInImage:image + completion:^(NSArray * _Nullable barcodes, NSError * _Nullable error) { + if (error) { + [FLTFirebaseMlVisionPlugin handleError:error result:result]; + return; + } else if (!barcodes) { + result(@[]); + return; + } + + NSMutableArray *blocks = [NSMutableArray array]; + for (FIRVisionBarcode *barcode in barcodes) { + NSDictionary *barcodeData = [BarcodeDetector getBarcodeData:barcode]; + [blocks addObject:barcodeData]; + } + + result(blocks); + }]; +} + ++ (void)close { + barcodeDetector = nil; +} + + ++ (NSDictionary *)getBarcodeData:(FIRVisionBarcode *)barcode { + CGRect frame = barcode.frame; + NSString *displayValue = barcode.displayValue == nil ? @"" : barcode.displayValue; + NSString *rawValue = barcode.rawValue == nil ? @"" : barcode.rawValue; + return @{ + @"left" : @((int)frame.origin.x), + @"top" : @((int)frame.origin.y), + @"width" : @((int)frame.size.width), + @"height" : @((int)frame.size.height), + @"barcode_display_value" : displayValue, + @"barcode_raw_value" : rawValue + }; +} @end diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index 857968de6b26..a3f361136e11 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -37,7 +37,10 @@ - (instancetype)init { - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result { if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { + FIRVisionImage *image = [self filePathToVisionImage:call.arguments]; + [BarcodeDetector handleDetection:image result:result]; } else if ([@"BarcodeDetector#close" isEqualToString:call.method]) { + [BarcodeDetector close]; } else if ([@"FaceDetector#detectInImage" isEqualToString:call.method]) { } else if ([@"FaceDetector#close" isEqualToString:call.method]) { } else if ([@"LabelDetector#detectInImage" isEqualToString:call.method]) { From 4e10f4448f8786349f0942691b3d37605b41f9bf Mon Sep 17 00:00:00 2001 From: dustin Date: Wed, 11 Jul 2018 11:05:25 -0700 Subject: [PATCH 13/34] ios: Add live view barcode scanning --- .../ios/Runner.xcodeproj/project.pbxproj | 2 +- .../ios/Classes/BarcodeDetector.m | 10 +- .../ios/Classes/FirebaseMlVisionPlugin.h | 7 + .../ios/Classes/FirebaseMlVisionPlugin.m | 96 ++++- .../firebase_ml_vision/ios/Classes/LiveView.h | 37 ++ .../firebase_ml_vision/ios/Classes/LiveView.m | 341 ++++++++++++++++++ .../ios/Classes/TextDetector.m | 116 +++--- .../ios/Classes/UIUtilities.h | 28 ++ .../ios/Classes/UIUtilities.m | 97 +++++ 9 files changed, 669 insertions(+), 65 deletions(-) create mode 100644 packages/firebase_ml_vision/ios/Classes/LiveView.h create mode 100644 packages/firebase_ml_vision/ios/Classes/LiveView.m create mode 100644 packages/firebase_ml_vision/ios/Classes/UIUtilities.h create mode 100644 packages/firebase_ml_vision/ios/Classes/UIUtilities.m diff --git a/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj b/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj index 11f2fd5defde..0ae5c14c9cd2 100644 --- a/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj +++ b/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj @@ -248,7 +248,7 @@ ); inputPaths = ( "${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh", - "${PODS_ROOT}/../.symlinks/flutter/ios-release/Flutter.framework", + "${PODS_ROOT}/../.symlinks/flutter/ios/Flutter.framework", ); name = "[CP] Embed Pods Frameworks"; outputPaths = ( diff --git a/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m b/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m index b48274365a99..241f0f4173b5 100644 --- a/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m @@ -3,7 +3,7 @@ @implementation BarcodeDetector static FIRVisionBarcodeDetector *barcodeDetector; -+ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { ++ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result resultWrapper:(FlutterResultWrapper)wrapper { if (barcodeDetector == nil) { FIRVision *vision = [FIRVision vision]; barcodeDetector = [vision barcodeDetector]; @@ -26,10 +26,16 @@ + (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { [blocks addObject:barcodeData]; } - result(blocks); + result(wrapper(blocks)); }]; } ++ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { + [BarcodeDetector handleDetection:image result:result resultWrapper:^id(id _Nullable result) { + return result; + }]; +} + + (void)close { barcodeDetector = nil; } diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h index 1967dfafabf5..95002c1e411a 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h @@ -2,12 +2,19 @@ #import "Firebase/Firebase.h" +/* + A callback type to allow the caller to format the detected response + before it is sent back to Flutter + */ +typedef id (^FlutterResultWrapper)(id _Nullable result); + @interface FLTFirebaseMlVisionPlugin : NSObject + (void)handleError:(NSError *)error result:(FlutterResult)result; @end @protocol Detector @required ++ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result resultWrapper:(FlutterResultWrapper)wrapper; + (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result; + (void)close; @optional diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index a3f361136e11..634ff6e52b8d 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -1,4 +1,5 @@ #import "FirebaseMlVisionPlugin.h" +#import "LiveView.h" @interface NSError (FlutterError) @property(readonly, nonatomic) FlutterError *flutterError; @@ -12,6 +13,12 @@ - (FlutterError *)flutterError { } @end +@interface FLTFirebaseMlVisionPlugin() +@property(readonly, nonatomic) NSObject *registry; +@property(readonly, nonatomic) NSObject *messenger; +@property(readonly, nonatomic) FLTCam *camera; +@end + @implementation FLTFirebaseMlVisionPlugin + (void)handleError:(NSError *)error result:(FlutterResult)result { result([error flutterError]); @@ -21,22 +28,97 @@ + (void)registerWithRegistrar:(NSObject *)registrar { FlutterMethodChannel *channel = [FlutterMethodChannel methodChannelWithName:@"plugins.flutter.io/firebase_ml_vision" binaryMessenger:[registrar messenger]]; - FLTFirebaseMlVisionPlugin *instance = [[FLTFirebaseMlVisionPlugin alloc] init]; + FLTFirebaseMlVisionPlugin *instance = [[FLTFirebaseMlVisionPlugin alloc] initWithRegistry:[registrar textures] messenger:[registrar messenger]]; [registrar addMethodCallDelegate:instance channel:channel]; } -- (instancetype)init { +- (instancetype)initWithRegistry:(NSObject *)registry + messenger:(NSObject *)messenger { self = [super init]; - if (self) { - if (![FIRApp defaultApp]) { - [FIRApp configure]; - } + NSAssert(self, @"super init cannot be nil"); + if (![FIRApp defaultApp]) { + [FIRApp configure]; } + _registry = registry; + _messenger = messenger; return self; } - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result { - if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { + if ([@"init" isEqualToString:call.method]) { + if (_camera) { + [_camera close]; + } + result(nil); + } else if ([@"availableCameras" isEqualToString:call.method]) { + AVCaptureDeviceDiscoverySession *discoverySession = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeBuiltInWideAngleCamera ] + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionUnspecified]; + NSArray *devices = discoverySession.devices; + NSMutableArray *> *reply = + [[NSMutableArray alloc] initWithCapacity:devices.count]; + for (AVCaptureDevice *device in devices) { + NSString *lensFacing; + switch ([device position]) { + case AVCaptureDevicePositionBack: + lensFacing = @"back"; + break; + case AVCaptureDevicePositionFront: + lensFacing = @"front"; + break; + case AVCaptureDevicePositionUnspecified: + lensFacing = @"external"; + break; + } + [reply addObject:@{ + @"name" : [device uniqueID], + @"lensFacing" : lensFacing, + }]; + } + result(reply); + } else if ([@"initialize" isEqualToString:call.method]) { + NSString *cameraName = call.arguments[@"cameraName"]; + NSString *resolutionPreset = call.arguments[@"resolutionPreset"]; + NSError *error; + FLTCam *cam = [[FLTCam alloc] initWithCameraName:cameraName + resolutionPreset:resolutionPreset + error:&error]; + if (error) { + result([error flutterError]); + } else { + NSLog(@"initialize called"); + if (_camera) { + [_camera close]; + } + int64_t textureId = [_registry registerTexture:cam]; + _camera = cam; + cam.onFrameAvailable = ^{ + [_registry textureFrameAvailable:textureId]; + }; + FlutterEventChannel *eventChannel = [FlutterEventChannel + eventChannelWithName:[NSString + stringWithFormat:@"plugins.flutter.io/firebase_ml_vision/liveViewEvents%lld", + textureId] + binaryMessenger:_messenger]; + [eventChannel setStreamHandler:cam]; + cam.eventChannel = eventChannel; + result(@{ + @"textureId" : @(textureId), + @"previewWidth" : @(cam.previewSize.width), + @"previewHeight" : @(cam.previewSize.height), + @"captureWidth" : @(cam.captureSize.width), + @"captureHeight" : @(cam.captureSize.height), + }); + [cam start]; + } + } else if ([@"dispose" isEqualToString:call.method]) { + NSDictionary *argsMap = call.arguments; + NSUInteger textureId = ((NSNumber *)argsMap[@"textureId"]).unsignedIntegerValue; + [_registry unregisterTexture:textureId]; + [_camera close]; + result(nil); + } else if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { FIRVisionImage *image = [self filePathToVisionImage:call.arguments]; [BarcodeDetector handleDetection:image result:result]; } else if ([@"BarcodeDetector#close" isEqualToString:call.method]) { diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.h b/packages/firebase_ml_vision/ios/Classes/LiveView.h new file mode 100644 index 000000000000..ddc69a983474 --- /dev/null +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.h @@ -0,0 +1,37 @@ +#import +#import +#import + +@interface FLTCam : NSObject +@property(readonly, nonatomic) int64_t textureId; +@property (nonatomic) bool isUsingFrontCamera; +@property(nonatomic, copy) void (^onFrameAvailable)(); +@property(nonatomic) FlutterEventChannel *eventChannel; +@property(nonatomic) FlutterEventSink eventSink; +@property(readonly, nonatomic) AVCaptureSession *captureSession; +@property(readonly, nonatomic) AVCaptureDevice *captureDevice; +@property(readonly, nonatomic) AVCapturePhotoOutput *capturePhotoOutput; +@property(readonly, nonatomic) AVCaptureVideoDataOutput *captureVideoOutput; +@property(readonly, nonatomic) AVCaptureInput *captureVideoInput; +@property(readonly) CVPixelBufferRef volatile latestPixelBuffer; +@property(readonly, nonatomic) CGSize previewSize; +@property(readonly, nonatomic) CGSize captureSize; +@property(strong, nonatomic) AVAssetWriter *videoWriter; +@property(strong, nonatomic) AVAssetWriterInput *videoWriterInput; +@property(strong, nonatomic) AVAssetWriterInput *audioWriterInput; +@property(strong, nonatomic) AVAssetWriterInputPixelBufferAdaptor *assetWriterPixelBufferAdaptor; +@property(strong, nonatomic) AVCaptureVideoDataOutput *videoOutput; +@property(strong, nonatomic) AVCaptureAudioDataOutput *audioOutput; +@property(assign, nonatomic) BOOL isRecording; +@property(assign, nonatomic) BOOL isAudioSetup; +- (instancetype)initWithCameraName:(NSString *)cameraName + resolutionPreset:(NSString *)resolutionPreset + error:(NSError **)error; +- (void)start; +- (void)stop; +- (void)close; +- (void)startVideoRecordingAtPath:(NSString *)path result:(FlutterResult)result; +- (void)stopVideoRecordingWithResult:(FlutterResult)result; +- (void)captureToFile:(NSString *)filename result:(FlutterResult)result; +@end diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.m b/packages/firebase_ml_vision/ios/Classes/LiveView.m new file mode 100644 index 000000000000..e3ad1a76bfd5 --- /dev/null +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.m @@ -0,0 +1,341 @@ +#import "FirebaseMlVisionPlugin.h" +#import "LiveView.h" +#import "UIUtilities.h" +#import +#import + +@interface FLTCam () +@property (assign, atomic) BOOL isRecognizing; +@end + +@implementation FLTCam +- (instancetype)initWithCameraName:(NSString *)cameraName + resolutionPreset:(NSString *)resolutionPreset + error:(NSError **)error { + self = [super init]; + NSAssert(self, @"super init cannot be nil"); + _isUsingFrontCamera = NO; + _captureSession = [[AVCaptureSession alloc] init]; + AVCaptureSessionPreset preset; + if ([resolutionPreset isEqualToString:@"high"]) { + preset = AVCaptureSessionPresetHigh; + } else if ([resolutionPreset isEqualToString:@"medium"]) { + preset = AVCaptureSessionPresetMedium; + } else { + NSAssert([resolutionPreset isEqualToString:@"low"], @"Unknown resolution preset %@", + resolutionPreset); + preset = AVCaptureSessionPresetLow; + } + _captureSession.sessionPreset = preset; + _captureDevice = [AVCaptureDevice deviceWithUniqueID:cameraName]; + NSError *localError = nil; + _captureVideoInput = + [AVCaptureDeviceInput deviceInputWithDevice:_captureDevice error:&localError]; + if (localError) { + *error = localError; + return nil; + } + CMVideoDimensions dimensions = + CMVideoFormatDescriptionGetDimensions([[_captureDevice activeFormat] formatDescription]); + _previewSize = CGSizeMake(dimensions.width, dimensions.height); + + _captureVideoOutput = [AVCaptureVideoDataOutput new]; + _captureVideoOutput.videoSettings = + @{(NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA) }; + [_captureVideoOutput setAlwaysDiscardsLateVideoFrames:YES]; + [_captureVideoOutput setSampleBufferDelegate:self queue:dispatch_get_main_queue()]; + + AVCaptureConnection *connection = + [AVCaptureConnection connectionWithInputPorts:_captureVideoInput.ports + output:_captureVideoOutput]; + if ([_captureDevice position] == AVCaptureDevicePositionFront) { + connection.videoMirrored = YES; + } + connection.videoOrientation = AVCaptureVideoOrientationPortrait; + [_captureSession addInputWithNoConnections:_captureVideoInput]; + [_captureSession addOutputWithNoConnections:_captureVideoOutput]; + [_captureSession addConnection:connection]; + _capturePhotoOutput = [AVCapturePhotoOutput new]; + [_captureSession addOutput:_capturePhotoOutput]; + return self; +} + +- (void)start { + [_captureSession startRunning]; +} + +- (void)stop { + [_captureSession stopRunning]; +} + +- (void)captureToFile:(NSString *)path result:(FlutterResult)result { +// AVCapturePhotoSettings *settings = [AVCapturePhotoSettings photoSettings]; +// [_capturePhotoOutput +// capturePhotoWithSettings:settings +// delegate:[[FLTSavePhotoDelegate alloc] initWithPath:path result:result]]; +} + +- (void)captureOutput:(AVCaptureOutput *)output +didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { + if (output == _captureVideoOutput) { + CVPixelBufferRef newBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + CFRetain(newBuffer); + CVPixelBufferRef old = _latestPixelBuffer; + while (!OSAtomicCompareAndSwapPtrBarrier(old, newBuffer, (void **)&_latestPixelBuffer)) { + old = _latestPixelBuffer; + } + if (old != nil) { + CFRelease(old); + } + if (_onFrameAvailable) { + _onFrameAvailable(); + } + if (!_isRecognizing) { + _isRecognizing = YES; + FIRVisionImage *visionImage = [[FIRVisionImage alloc] initWithBuffer:sampleBuffer]; + FIRVisionImageMetadata *metadata = [[FIRVisionImageMetadata alloc] init]; + UIImageOrientation orientation = [UIUtilities imageOrientationFromDevicePosition:_isUsingFrontCamera ? AVCaptureDevicePositionFront : AVCaptureDevicePositionBack]; + FIRVisionDetectorImageOrientation visionOrientation = [UIUtilities visionImageOrientationFromImageOrientation:orientation]; + + metadata.orientation = visionOrientation; + visionImage.metadata = metadata; + CGFloat imageWidth = CVPixelBufferGetWidth(newBuffer); + CGFloat imageHeight = CVPixelBufferGetHeight(newBuffer); + [BarcodeDetector handleDetection:visionImage result:_eventSink resultWrapper:^id(id _Nullable result) { + _isRecognizing = NO; + return @{@"eventType": @"recognized", @"recognitionType": @"barcode", @"barcodeData": result}; + }]; + } +// switch (_currentDetector) { +// case DetectorOnDeviceFace: +// [self detectFacesOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; +// break; +// case DetectorOnDeviceText: +// [self detectTextOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; +// break; +// } + } + if (!CMSampleBufferDataIsReady(sampleBuffer)) { + _eventSink(@{ + @"event" : @"error", + @"errorDescription" : @"sample buffer is not ready. Skipping sample" + }); + return; + } + if (_isRecording) { + if (_videoWriter.status == AVAssetWriterStatusFailed) { + _eventSink(@{ + @"event" : @"error", + @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] + }); + return; + } + CMTime lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); + if (_videoWriter.status != AVAssetWriterStatusWriting) { + [_videoWriter startWriting]; + [_videoWriter startSessionAtSourceTime:lastSampleTime]; + } + if (output == _captureVideoOutput) { + [self newVideoSample:sampleBuffer]; + } else { + [self newAudioSample:sampleBuffer]; + } + } +} + +- (void)newVideoSample:(CMSampleBufferRef)sampleBuffer { + if (_videoWriter.status != AVAssetWriterStatusWriting) { + if (_videoWriter.status == AVAssetWriterStatusFailed) { + _eventSink(@{ + @"event" : @"error", + @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] + }); + } + return; + } + if (_videoWriterInput.readyForMoreMediaData) { + if (![_videoWriterInput appendSampleBuffer:sampleBuffer]) { + _eventSink(@{ + @"event" : @"error", + @"errorDescription" : + [NSString stringWithFormat:@"%@", @"Unable to write to video input"] + }); + } + } +} + +- (void)newAudioSample:(CMSampleBufferRef)sampleBuffer { + if (_videoWriter.status != AVAssetWriterStatusWriting) { + if (_videoWriter.status == AVAssetWriterStatusFailed) { + _eventSink(@{ + @"event" : @"error", + @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] + }); + } + return; + } + if (_audioWriterInput.readyForMoreMediaData) { + if (![_audioWriterInput appendSampleBuffer:sampleBuffer]) { + _eventSink(@{ + @"event" : @"error", + @"errorDescription" : + [NSString stringWithFormat:@"%@", @"Unable to write to audio input"] + }); + } + } +} + +- (void)close { + [_captureSession stopRunning]; + for (AVCaptureInput *input in [_captureSession inputs]) { + [_captureSession removeInput:input]; + } + for (AVCaptureOutput *output in [_captureSession outputs]) { + [_captureSession removeOutput:output]; + } +} + +- (void)dealloc { + if (_latestPixelBuffer) { + CFRelease(_latestPixelBuffer); + } +} + +- (CVPixelBufferRef)copyPixelBuffer { + CVPixelBufferRef pixelBuffer = _latestPixelBuffer; + while (!OSAtomicCompareAndSwapPtrBarrier(pixelBuffer, nil, (void **)&_latestPixelBuffer)) { + pixelBuffer = _latestPixelBuffer; + } + return pixelBuffer; +} + +- (FlutterError *_Nullable)onCancelWithArguments:(id _Nullable)arguments { + _eventSink = nil; + return nil; +} + +- (FlutterError *_Nullable)onListenWithArguments:(id _Nullable)arguments + eventSink:(nonnull FlutterEventSink)events { + _eventSink = events; + return nil; +} +- (void)startVideoRecordingAtPath:(NSString *)path result:(FlutterResult)result { + if (!_isRecording) { + if (![self setupWriterForPath:path]) { + _eventSink(@{@"event" : @"error", @"errorDescription" : @"Setup Writer Failed"}); + return; + } + [_captureSession stopRunning]; + _isRecording = YES; + [_captureSession startRunning]; + result(nil); + } else { + _eventSink(@{@"event" : @"error", @"errorDescription" : @"Video is already recording!"}); + } +} + +- (void)stopVideoRecordingWithResult:(FlutterResult)result { + if (_isRecording) { + _isRecording = NO; + if (_videoWriter.status != AVAssetWriterStatusUnknown) { + [_videoWriter finishWritingWithCompletionHandler:^{ + if (self->_videoWriter.status == AVAssetWriterStatusCompleted) { + result(nil); + } else { + self->_eventSink(@{ + @"event" : @"error", + @"errorDescription" : @"AVAssetWriter could not finish writing!" + }); + } + }]; + } + } else { +// NSError *error = +// [NSError errorWithDomain:NSCocoaErrorDomain +// code:NSURLErrorResourceUnavailable +// userInfo:@{NSLocalizedDescriptionKey : @"Video is not recording!"}]; +// result([error flutterError]); + } +} + +- (BOOL)setupWriterForPath:(NSString *)path { + NSError *error = nil; + NSURL *outputURL; + if (path != nil) { + outputURL = [NSURL fileURLWithPath:path]; + } else { + return NO; + } + if (!_isAudioSetup) { + [self setUpCaptureSessionForAudio]; + } + _videoWriter = + [[AVAssetWriter alloc] initWithURL:outputURL fileType:AVFileTypeQuickTimeMovie error:&error]; + NSParameterAssert(_videoWriter); + if (error) { + _eventSink(@{@"event" : @"error", @"errorDescription" : error.description}); + return NO; + } + NSDictionary *videoSettings = [NSDictionary + dictionaryWithObjectsAndKeys:AVVideoCodecH264, AVVideoCodecKey, + [NSNumber numberWithInt:_previewSize.height], AVVideoWidthKey, + [NSNumber numberWithInt:_previewSize.width], AVVideoHeightKey, + nil]; + _videoWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo + outputSettings:videoSettings]; + NSParameterAssert(_videoWriterInput); + _videoWriterInput.expectsMediaDataInRealTime = YES; + + // Add the audio input + AudioChannelLayout acl; + bzero(&acl, sizeof(acl)); + acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono; + NSDictionary *audioOutputSettings = nil; + // Both type of audio inputs causes output video file to be corrupted. + audioOutputSettings = [NSDictionary + dictionaryWithObjectsAndKeys:[NSNumber numberWithInt:kAudioFormatMPEG4AAC], AVFormatIDKey, + [NSNumber numberWithFloat:44100.0], AVSampleRateKey, + [NSNumber numberWithInt:1], AVNumberOfChannelsKey, + [NSData dataWithBytes:&acl length:sizeof(acl)], + AVChannelLayoutKey, nil]; + _audioWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio + outputSettings:audioOutputSettings]; + _audioWriterInput.expectsMediaDataInRealTime = YES; + [_videoWriter addInput:_videoWriterInput]; + [_videoWriter addInput:_audioWriterInput]; + dispatch_queue_t queue = dispatch_queue_create("MyQueue", NULL); + [_captureVideoOutput setSampleBufferDelegate:self queue:queue]; + [_audioOutput setSampleBufferDelegate:self queue:queue]; + + return YES; +} +- (void)setUpCaptureSessionForAudio { + NSError *error = nil; + // Create a device input with the device and add it to the session. + // Setup the audio input. + AVCaptureDevice *audioDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + AVCaptureDeviceInput *audioInput = + [AVCaptureDeviceInput deviceInputWithDevice:audioDevice error:&error]; + if (error) { + _eventSink(@{@"event" : @"error", @"errorDescription" : error.description}); + } + // Setup the audio output. + _audioOutput = [[AVCaptureAudioDataOutput alloc] init]; + + if ([_captureSession canAddInput:audioInput]) { + [_captureSession addInput:audioInput]; + + if ([_captureSession canAddOutput:_audioOutput]) { + [_captureSession addOutput:_audioOutput]; + _isAudioSetup = YES; + } else { + _eventSink(@{ + @"event" : @"error", + @"errorDescription" : @"Unable to add Audio input/output to session capture" + }); + _isAudioSetup = NO; + } + } +} +@end diff --git a/packages/firebase_ml_vision/ios/Classes/TextDetector.m b/packages/firebase_ml_vision/ios/Classes/TextDetector.m index 74102ce022aa..33d3d44914bb 100644 --- a/packages/firebase_ml_vision/ios/Classes/TextDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/TextDetector.m @@ -3,67 +3,73 @@ @implementation TextDetector static FIRVisionTextDetector *textDetector; -+ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { ++ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result resultWrapper:(FlutterResultWrapper)wrapper { if (textDetector == nil) { FIRVision *vision = [FIRVision vision]; textDetector = [vision textDetector]; } - + [textDetector - detectInImage:image - completion:^(NSArray> *_Nullable features, NSError *_Nullable error) { - if (error) { - [FLTFirebaseMlVisionPlugin handleError:error result:result]; - return; - } else if (!features) { - result(@[]); - return; - } - - NSMutableArray *blocks = [NSMutableArray array]; - for (id feature in features) { - NSMutableDictionary *blockData = [NSMutableDictionary dictionary]; - if ([feature isKindOfClass:[FIRVisionTextBlock class]]) { - FIRVisionTextBlock *block = (FIRVisionTextBlock *)feature; - - [blockData addEntriesFromDictionary:[self getTextData:block.frame - cornerPoints:block.cornerPoints - text:block.text]]; - blockData[@"lines"] = [self getLineData:block.lines]; - } else if ([feature isKindOfClass:[FIRVisionTextLine class]]) { - // We structure the return data to have the line be inside a FIRVisionTextBlock. - FIRVisionTextLine *line = (FIRVisionTextLine *)feature; - - [blockData addEntriesFromDictionary:[self getTextData:line.frame - cornerPoints:line.cornerPoints - text:line.text]]; - NSArray *lines = @[ line ]; - blockData[@"lines"] = [self getLineData:lines]; - } else if ([feature isKindOfClass:[FIRVisionTextElement class]]) { - // We structure the return data to have the element inside a FIRVisionTextLine - // that is inside a FIRVisionTextBlock. - FIRVisionTextElement *element = (FIRVisionTextElement *)feature; - - [blockData addEntriesFromDictionary:[self getTextData:element.frame - cornerPoints:element.cornerPoints - text:element.text]]; - - NSMutableDictionary *lineData = [NSMutableDictionary dictionary]; - [lineData addEntriesFromDictionary:[self getTextData:element.frame - cornerPoints:element.cornerPoints - text:element.text]]; - - NSArray *elements = @[ element ]; - lineData[@"elements"] = [self getElementData:elements]; - - blockData[@"lines"] = lineData; - } - - [blocks addObject:blockData]; - } + detectInImage:image + completion:^(NSArray> *_Nullable features, NSError *_Nullable error) { + if (error) { + [FLTFirebaseMlVisionPlugin handleError:error result:result]; + return; + } else if (!features) { + result(@[]); + return; + } + + NSMutableArray *blocks = [NSMutableArray array]; + for (id feature in features) { + NSMutableDictionary *blockData = [NSMutableDictionary dictionary]; + if ([feature isKindOfClass:[FIRVisionTextBlock class]]) { + FIRVisionTextBlock *block = (FIRVisionTextBlock *)feature; + + [blockData addEntriesFromDictionary:[self getTextData:block.frame + cornerPoints:block.cornerPoints + text:block.text]]; + blockData[@"lines"] = [self getLineData:block.lines]; + } else if ([feature isKindOfClass:[FIRVisionTextLine class]]) { + // We structure the return data to have the line be inside a FIRVisionTextBlock. + FIRVisionTextLine *line = (FIRVisionTextLine *)feature; + + [blockData addEntriesFromDictionary:[self getTextData:line.frame + cornerPoints:line.cornerPoints + text:line.text]]; + NSArray *lines = @[ line ]; + blockData[@"lines"] = [self getLineData:lines]; + } else if ([feature isKindOfClass:[FIRVisionTextElement class]]) { + // We structure the return data to have the element inside a FIRVisionTextLine + // that is inside a FIRVisionTextBlock. + FIRVisionTextElement *element = (FIRVisionTextElement *)feature; + + [blockData addEntriesFromDictionary:[self getTextData:element.frame + cornerPoints:element.cornerPoints + text:element.text]]; + + NSMutableDictionary *lineData = [NSMutableDictionary dictionary]; + [lineData addEntriesFromDictionary:[self getTextData:element.frame + cornerPoints:element.cornerPoints + text:element.text]]; + + NSArray *elements = @[ element ]; + lineData[@"elements"] = [self getElementData:elements]; + + blockData[@"lines"] = lineData; + } + + [blocks addObject:blockData]; + } + + result(wrapper(blocks)); + }]; +} - result(blocks); - }]; ++ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { + [TextDetector handleDetection:image result:result resultWrapper:^id(id _Nullable result) { + return result; + }]; } + (void)close { diff --git a/packages/firebase_ml_vision/ios/Classes/UIUtilities.h b/packages/firebase_ml_vision/ios/Classes/UIUtilities.h new file mode 100644 index 000000000000..e8e296fe0dcb --- /dev/null +++ b/packages/firebase_ml_vision/ios/Classes/UIUtilities.h @@ -0,0 +1,28 @@ +// +// Copyright (c) 2018 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#import +@import AVFoundation; +@import UIKit; +@import FirebaseMLVision; + +@interface UIUtilities : NSObject ++ (void)imageOrientation; ++ (UIImageOrientation)imageOrientationFromDevicePosition:(AVCaptureDevicePosition)devicePosition; ++ (FIRVisionDetectorImageOrientation)visionImageOrientationFromImageOrientation:(UIImageOrientation)imageOrientation; ++ (UIDeviceOrientation)currentUIOrientation; + +@end diff --git a/packages/firebase_ml_vision/ios/Classes/UIUtilities.m b/packages/firebase_ml_vision/ios/Classes/UIUtilities.m new file mode 100644 index 000000000000..f5f49c8825c8 --- /dev/null +++ b/packages/firebase_ml_vision/ios/Classes/UIUtilities.m @@ -0,0 +1,97 @@ +// +// Copyright (c) 2018 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#import "UIUtilities.h" + +static CGFloat const circleViewAlpha = 0.7; +static CGFloat const rectangleViewAlpha = 0.3; +static CGFloat const shapeViewAlpha = 0.3; +static CGFloat const rectangleViewCornerRadius = 10.0; + +@implementation UIUtilities + ++ (void)imageOrientation { + [self imageOrientationFromDevicePosition:AVCaptureDevicePositionBack]; +} + ++ (UIImageOrientation) imageOrientationFromDevicePosition:(AVCaptureDevicePosition)devicePosition { + UIDeviceOrientation deviceOrientation = UIDevice.currentDevice.orientation; + if (deviceOrientation == UIDeviceOrientationFaceDown || deviceOrientation == UIDeviceOrientationFaceUp || deviceOrientation == UIDeviceOrientationUnknown) { + deviceOrientation = [self currentUIOrientation]; + } + switch (deviceOrientation) { + case UIDeviceOrientationPortrait: + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationLeftMirrored : UIImageOrientationRight; + case UIDeviceOrientationLandscapeLeft: + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationDownMirrored : UIImageOrientationUp; + case UIDeviceOrientationPortraitUpsideDown: + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationRightMirrored : UIImageOrientationLeft; + case UIDeviceOrientationLandscapeRight: + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationUpMirrored : UIImageOrientationDown; + case UIDeviceOrientationFaceDown: + case UIDeviceOrientationFaceUp: + case UIDeviceOrientationUnknown: + return UIImageOrientationUp; + } +} + ++ (FIRVisionDetectorImageOrientation) visionImageOrientationFromImageOrientation:(UIImageOrientation)imageOrientation { + switch (imageOrientation) { + case UIImageOrientationUp: + return FIRVisionDetectorImageOrientationTopLeft; + case UIImageOrientationDown: + return FIRVisionDetectorImageOrientationBottomRight; + case UIImageOrientationLeft: + return FIRVisionDetectorImageOrientationLeftBottom; + case UIImageOrientationRight: + return FIRVisionDetectorImageOrientationRightTop; + case UIImageOrientationUpMirrored: + return FIRVisionDetectorImageOrientationTopRight; + case UIImageOrientationDownMirrored: + return FIRVisionDetectorImageOrientationBottomLeft; + case UIImageOrientationLeftMirrored: + return FIRVisionDetectorImageOrientationLeftTop; + case UIImageOrientationRightMirrored: + return FIRVisionDetectorImageOrientationRightBottom; + } +} + ++ (UIDeviceOrientation) currentUIOrientation { + UIDeviceOrientation (^deviceOrientation)(void) = ^UIDeviceOrientation(void) { + switch (UIApplication.sharedApplication.statusBarOrientation) { + case UIInterfaceOrientationLandscapeLeft: + return UIDeviceOrientationLandscapeRight; + case UIInterfaceOrientationLandscapeRight: + return UIDeviceOrientationLandscapeLeft; + case UIInterfaceOrientationPortraitUpsideDown: + return UIDeviceOrientationPortraitUpsideDown; + case UIInterfaceOrientationPortrait: + case UIInterfaceOrientationUnknown: + return UIDeviceOrientationPortrait; + } + }; + + if (NSThread.isMainThread) { + return deviceOrientation(); + } else { + __block UIDeviceOrientation currentOrientation = UIDeviceOrientationPortrait; + dispatch_sync(dispatch_get_main_queue(), ^{ + currentOrientation = deviceOrientation(); + }); + return currentOrientation; + } +} +@end From 131356409f8117280a9ca8b2c10116e2920d026d Mon Sep 17 00:00:00 2001 From: dustin Date: Fri, 13 Jul 2018 14:14:18 -0700 Subject: [PATCH 14/34] WIP: live text detection defect: preview image is not oriented correctly. detector does not work properly when the preview image is oriented correctly. --- .../example/lib/detector_painters.dart | 25 ++- .../example/lib/live_preview.dart | 33 ++-- .../firebase_ml_vision/example/lib/main.dart | 32 ++-- .../ios/Classes/FirebaseMlVisionPlugin.m | 26 ++- .../firebase_ml_vision/ios/Classes/LiveView.h | 2 + .../firebase_ml_vision/ios/Classes/LiveView.m | 177 ++++++++++++------ .../ios/Classes/TextDetector.m | 2 +- .../lib/src/firebase_vision.dart | 30 +++ .../firebase_ml_vision/lib/src/live_view.dart | 37 +++- .../lib/src/text_detector.dart | 4 + 10 files changed, 254 insertions(+), 114 deletions(-) diff --git a/packages/firebase_ml_vision/example/lib/detector_painters.dart b/packages/firebase_ml_vision/example/lib/detector_painters.dart index bfc164d39c32..552c0b0aeaf5 100644 --- a/packages/firebase_ml_vision/example/lib/detector_painters.dart +++ b/packages/firebase_ml_vision/example/lib/detector_painters.dart @@ -5,24 +5,29 @@ import 'package:firebase_ml_vision/firebase_ml_vision.dart'; import 'package:flutter/material.dart'; -enum Detector { barcode, face, label, text } - CustomPaint customPaintForResults( - Detector detector, Size imageSize, List results) { + FirebaseVisionDetectorType detector, Size imageSize, List results) { CustomPainter painter; - switch (detector) { - case Detector.barcode: - painter = new BarcodeDetectorPainter(imageSize, results); + case FirebaseVisionDetectorType.barcode: + try { + painter = new BarcodeDetectorPainter(imageSize, results.cast()); + } on CastError { + painter = null; + } break; - case Detector.face: + case FirebaseVisionDetectorType.face: painter = new FaceDetectorPainter(imageSize, results); break; - case Detector.label: + case FirebaseVisionDetectorType.label: painter = new LabelDetectorPainter(imageSize, results); break; - case Detector.text: - painter = new TextDetectorPainter(imageSize, results); + case FirebaseVisionDetectorType.text: + try { + painter = new TextDetectorPainter(imageSize, results.cast()); + } on CastError { + painter = null; + } break; default: break; diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 449cba454701..c3b0c64d80ab 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -6,7 +6,7 @@ import 'package:firebase_ml_vision_example/detector_painters.dart'; import 'package:flutter/material.dart'; class LivePreview extends StatefulWidget { - final Detector detector; + final FirebaseVisionDetectorType detector; const LivePreview( this.detector, { @@ -38,9 +38,9 @@ class LivePreviewState extends State { new LiveViewCameraController( backCamera, LiveViewResolutionPreset.high); await controller.initialize(); + setLiveViewDetector(); yield new LiveViewCameraLoadStateReady(controller); } on LiveViewCameraException catch (e) { - print("got an error: $e"); yield new LiveViewCameraLoadStateFailed( "error initializing camera controller: ${e.toString()}"); } @@ -50,6 +50,17 @@ class LivePreviewState extends State { } } + @override + void initState() { + super.initState(); + setLiveViewDetector(); + } + + void setLiveViewDetector() async { + // set the initial recognizer + await FirebaseVision.instance.setLiveViewRecognizer(widget.detector); + } + @override void dispose() { super.dispose(); @@ -78,20 +89,12 @@ class LivePreviewState extends State { aspectRatio: _readyLoadState.controller.value.aspectRatio, child: new LiveView( controller: _readyLoadState.controller, - overlayBuilder: (BuildContext context, Size previewSize, - List barcodes) { - return barcodes == null - ? const Center( - child: const Text( - 'Scanning...', - style: const TextStyle( - color: Colors.green, - fontSize: 30.0, - ), - ), - ) + overlayBuilder: + (BuildContext context, Size previewSize, dynamic data) { + return data == null + ? new Container() : customPaintForResults( - Detector.barcode, previewSize, barcodes); + widget.detector, previewSize, data); }, ), ); diff --git a/packages/firebase_ml_vision/example/lib/main.dart b/packages/firebase_ml_vision/example/lib/main.dart index d183331e6bd0..40cb7979e7c1 100644 --- a/packages/firebase_ml_vision/example/lib/main.dart +++ b/packages/firebase_ml_vision/example/lib/main.dart @@ -23,7 +23,7 @@ class _MyHomePageState extends State<_MyHomePage> File _imageFile; Size _imageSize; List _scanResults; - Detector _currentDetector = Detector.text; + FirebaseVisionDetectorType _currentDetector = FirebaseVisionDetectorType.text; TabController _tabController; int _selectedPageIndex = 0; @@ -89,16 +89,16 @@ class _MyHomePageState extends State<_MyHomePage> FirebaseVisionDetector detector; switch (_currentDetector) { - case Detector.barcode: + case FirebaseVisionDetectorType.barcode: detector = FirebaseVision.instance.barcodeDetector(null); break; - case Detector.face: + case FirebaseVisionDetectorType.face: detector = FirebaseVision.instance.faceDetector(null); break; - case Detector.label: + case FirebaseVisionDetectorType.label: detector = FirebaseVision.instance.labelDetector(null); break; - case Detector.text: + case FirebaseVisionDetectorType.text: detector = FirebaseVision.instance.textDetector(); break; default: @@ -142,29 +142,29 @@ class _MyHomePageState extends State<_MyHomePage> appBar: new AppBar( title: const Text('ML Vision Example'), actions: [ - new PopupMenuButton( - onSelected: (Detector result) { + new PopupMenuButton( + onSelected: (FirebaseVisionDetectorType result) { setState(() { _currentDetector = result; if (_imageFile != null) _scanImage(_imageFile); }); }, - itemBuilder: (BuildContext context) => >[ - const PopupMenuItem( + itemBuilder: (BuildContext context) => >[ + const PopupMenuItem( child: const Text('Detect Barcode'), - value: Detector.barcode, + value: FirebaseVisionDetectorType.barcode, ), - const PopupMenuItem( + const PopupMenuItem( child: const Text('Detect Face'), - value: Detector.face, + value: FirebaseVisionDetectorType.face, ), - const PopupMenuItem( + const PopupMenuItem( child: const Text('Detect Label'), - value: Detector.label, + value: FirebaseVisionDetectorType.label, ), - const PopupMenuItem( + const PopupMenuItem( child: const Text('Detect Text'), - value: Detector.text, + value: FirebaseVisionDetectorType.text, ), ], ), diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index 634ff6e52b8d..be1c277a4404 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -103,13 +103,15 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result binaryMessenger:_messenger]; [eventChannel setStreamHandler:cam]; cam.eventChannel = eventChannel; - result(@{ - @"textureId" : @(textureId), - @"previewWidth" : @(cam.previewSize.width), - @"previewHeight" : @(cam.previewSize.height), - @"captureWidth" : @(cam.captureSize.width), - @"captureHeight" : @(cam.captureSize.height), - }); + cam.onSizeAvailable = ^{ + result(@{ + @"textureId" : @(textureId), + @"previewWidth" : @(cam.previewSize.width), + @"previewHeight" : @(cam.previewSize.height), + @"captureWidth" : @(cam.captureSize.width), + @"captureHeight" : @(cam.captureSize.height), + }); + }; [cam start]; } } else if ([@"dispose" isEqualToString:call.method]) { @@ -118,6 +120,16 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result [_registry unregisterTexture:textureId]; [_camera close]; result(nil); + } else if ([@"LiveView#setRecognizer" isEqualToString:call.method]) { + NSLog(@"setRecognizer called"); + NSDictionary *argsMap = call.arguments; + NSString *recognizerType = ((NSString *)argsMap[@"recognizerType"]); + NSLog(recognizerType); + if (_camera) { + NSLog(@"got a camera, setting the recognizer"); +// [_camera setRecognizerType:recognizerType]; + } + result(nil); } else if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { FIRVisionImage *image = [self filePathToVisionImage:call.arguments]; [BarcodeDetector handleDetection:image result:result]; diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.h b/packages/firebase_ml_vision/ios/Classes/LiveView.h index ddc69a983474..e9e9de5f4931 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.h +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.h @@ -7,6 +7,7 @@ AVCaptureAudioDataOutputSampleBufferDelegate, FlutterStreamHandler> @property(readonly, nonatomic) int64_t textureId; @property (nonatomic) bool isUsingFrontCamera; @property(nonatomic, copy) void (^onFrameAvailable)(); +@property(nonatomic, copy) void (^onSizeAvailable)(); @property(nonatomic) FlutterEventChannel *eventChannel; @property(nonatomic) FlutterEventSink eventSink; @property(readonly, nonatomic) AVCaptureSession *captureSession; @@ -34,4 +35,5 @@ AVCaptureAudioDataOutputSampleBufferDelegate, FlutterStreamHandler> - (void)startVideoRecordingAtPath:(NSString *)path result:(FlutterResult)result; - (void)stopVideoRecordingWithResult:(FlutterResult)result; - (void)captureToFile:(NSString *)filename result:(FlutterResult)result; +//- (void)setRecognizerType:(NSString *)recognizerType; @end diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.m b/packages/firebase_ml_vision/ios/Classes/LiveView.m index e3ad1a76bfd5..c15920551c09 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.m +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.m @@ -4,8 +4,12 @@ #import #import +static NSString *const sessionQueueLabel = @"io.flutter.plugins.firebaseml.visiondetector.SessionQueue"; +static NSString *const videoDataOutputQueueLabel = @"io.flutter.plugins.firebaseml.visiondetector.VideoDataOutputQueue"; + @interface FLTCam () @property (assign, atomic) BOOL isRecognizing; +@property (nonatomic) dispatch_queue_t sessionQueue; @end @implementation FLTCam @@ -14,58 +18,113 @@ - (instancetype)initWithCameraName:(NSString *)cameraName error:(NSError **)error { self = [super init]; NSAssert(self, @"super init cannot be nil"); + + // Configure Captgure Session + _isUsingFrontCamera = NO; _captureSession = [[AVCaptureSession alloc] init]; + _sessionQueue = dispatch_queue_create(sessionQueueLabel.UTF8String, nil); + + // base example uses AVCaptureVideoPreviewLayer here and the layer is added to a view, Flutter Texture works differently here + [self setUpCaptureSessionOutputWithResolutionPreset:resolutionPreset]; + [self setUpCaptureSessionInputWithCameraName:cameraName]; + + + // Probably unnecessary +// CMVideoDimensions dimensions = +// CMVideoFormatDescriptionGetDimensions([[_captureDevice activeFormat] formatDescription]); +// _previewSize = CGSizeMake(dimensions.width, dimensions.height); + +// _capturePhotoOutput = [AVCapturePhotoOutput new]; +// [_captureSession addOutput:_capturePhotoOutput]; + return self; +} + +- (AVCaptureSessionPreset) resolutionPresetForPreference:(NSString *)preference { AVCaptureSessionPreset preset; - if ([resolutionPreset isEqualToString:@"high"]) { + if ([preference isEqualToString:@"high"]) { preset = AVCaptureSessionPresetHigh; - } else if ([resolutionPreset isEqualToString:@"medium"]) { + } else if ([preference isEqualToString:@"medium"]) { preset = AVCaptureSessionPresetMedium; } else { - NSAssert([resolutionPreset isEqualToString:@"low"], @"Unknown resolution preset %@", - resolutionPreset); + NSAssert([preference isEqualToString:@"low"], @"Unknown resolution preset %@", + preference); preset = AVCaptureSessionPresetLow; } - _captureSession.sessionPreset = preset; - _captureDevice = [AVCaptureDevice deviceWithUniqueID:cameraName]; - NSError *localError = nil; - _captureVideoInput = - [AVCaptureDeviceInput deviceInputWithDevice:_captureDevice error:&localError]; - if (localError) { - *error = localError; - return nil; - } - CMVideoDimensions dimensions = - CMVideoFormatDescriptionGetDimensions([[_captureDevice activeFormat] formatDescription]); - _previewSize = CGSizeMake(dimensions.width, dimensions.height); - - _captureVideoOutput = [AVCaptureVideoDataOutput new]; - _captureVideoOutput.videoSettings = - @{(NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA) }; - [_captureVideoOutput setAlwaysDiscardsLateVideoFrames:YES]; - [_captureVideoOutput setSampleBufferDelegate:self queue:dispatch_get_main_queue()]; - - AVCaptureConnection *connection = - [AVCaptureConnection connectionWithInputPorts:_captureVideoInput.ports - output:_captureVideoOutput]; - if ([_captureDevice position] == AVCaptureDevicePositionFront) { - connection.videoMirrored = YES; - } - connection.videoOrientation = AVCaptureVideoOrientationPortrait; - [_captureSession addInputWithNoConnections:_captureVideoInput]; - [_captureSession addOutputWithNoConnections:_captureVideoOutput]; - [_captureSession addConnection:connection]; - _capturePhotoOutput = [AVCapturePhotoOutput new]; - [_captureSession addOutput:_capturePhotoOutput]; - return self; + return preset; +} + +- (void)setUpCaptureSessionOutputWithResolutionPreset:(NSString *)resolutionPreset { + dispatch_async(_sessionQueue, ^{ + [self->_captureSession beginConfiguration]; + self->_captureSession.sessionPreset = [self resolutionPresetForPreference:resolutionPreset]; + + _captureVideoOutput = [[AVCaptureVideoDataOutput alloc] init]; + _captureVideoOutput.videoSettings = @{(id)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA]}; + dispatch_queue_t outputQueue = dispatch_queue_create(videoDataOutputQueueLabel.UTF8String, nil); + [_captureVideoOutput setSampleBufferDelegate:self queue:outputQueue]; + if ([self.captureSession canAddOutput:_captureVideoOutput]) { + [self.captureSession addOutputWithNoConnections:_captureVideoOutput]; + [self.captureSession commitConfiguration]; + } else { + NSLog(@"%@", @"Failed to add capture session output."); + } + }); +} + +- (void)setUpCaptureSessionInputWithCameraName:(NSString *)cameraName { + dispatch_async(_sessionQueue, ^{ + AVCaptureDevice *device = [AVCaptureDevice deviceWithUniqueID:cameraName]; + CMVideoDimensions dimensions = + CMVideoFormatDescriptionGetDimensions([[device activeFormat] formatDescription]); + _previewSize = CGSizeMake(dimensions.width, dimensions.height); + if (_onSizeAvailable) { + _onSizeAvailable(); + } + if (device) { + NSArray *currentInputs = self.captureSession.inputs; + for (AVCaptureInput *input in currentInputs) { + [self.captureSession removeInput:input]; + } + NSError *error; + _captureVideoInput = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error]; + + if (error) { + NSLog(@"Failed to create capture device input: %@", error.localizedDescription); + return; + } else { + // TODO? ACaptureConnection? + AVCaptureConnection *connection = + [AVCaptureConnection connectionWithInputPorts:_captureVideoInput.ports + output:_captureVideoOutput]; + if ([_captureDevice position] == AVCaptureDevicePositionFront) { + connection.videoMirrored = YES; + } +// connection.videoOrientation = AVCaptureVideoOrientationPortrait; + [_captureSession addInputWithNoConnections:_captureVideoInput]; + [_captureSession addConnection:connection]; +// if ([self.captureSession canAddInput:_captureVideoInput]) { +// [self.captureSession addInput:_captureVideoInput]; +// } else { +// NSLog(@"%@", @"Failed to add capture session input."); +// } + } + } else { + NSLog(@"Failed to get capture device for camera position: %ld", cameraName); + } + }); } - (void)start { - [_captureSession startRunning]; + dispatch_async(_sessionQueue, ^{ + [self->_captureSession startRunning]; + }); } - (void)stop { - [_captureSession stopRunning]; + dispatch_async(_sessionQueue, ^{ + [self->_captureSession stopRunning]; + }); } - (void)captureToFile:(NSString *)path result:(FlutterResult)result { @@ -76,21 +135,14 @@ - (void)captureToFile:(NSString *)path result:(FlutterResult)result { } - (void)captureOutput:(AVCaptureOutput *)output -didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + didOutput:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { - if (output == _captureVideoOutput) { - CVPixelBufferRef newBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); - CFRetain(newBuffer); - CVPixelBufferRef old = _latestPixelBuffer; - while (!OSAtomicCompareAndSwapPtrBarrier(old, newBuffer, (void **)&_latestPixelBuffer)) { - old = _latestPixelBuffer; - } - if (old != nil) { - CFRelease(old); - } - if (_onFrameAvailable) { - _onFrameAvailable(); - } + NSLog(@"Got Here!!!!"); +} + +- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { + CVImageBufferRef newBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (newBuffer) { if (!_isRecognizing) { _isRecognizing = YES; FIRVisionImage *visionImage = [[FIRVisionImage alloc] initWithBuffer:sampleBuffer]; @@ -102,11 +154,27 @@ - (void)captureOutput:(AVCaptureOutput *)output visionImage.metadata = metadata; CGFloat imageWidth = CVPixelBufferGetWidth(newBuffer); CGFloat imageHeight = CVPixelBufferGetHeight(newBuffer); - [BarcodeDetector handleDetection:visionImage result:_eventSink resultWrapper:^id(id _Nullable result) { + [TextDetector handleDetection:visionImage result:_eventSink resultWrapper:^id(id _Nullable result) { _isRecognizing = NO; - return @{@"eventType": @"recognized", @"recognitionType": @"barcode", @"barcodeData": result}; + return @{@"eventType": @"recognized", @"recognitionType": @"text", @"textData": result}; }]; + // [BarcodeDetector handleDetection:visionImage result:_eventSink resultWrapper:^id(id _Nullable result) { + // _isRecognizing = NO; + // return @{@"eventType": @"recognized", @"recognitionType": @"barcode", @"barcodeData": result}; + // }]; } + CFRetain(newBuffer); + CVPixelBufferRef old = _latestPixelBuffer; + while (!OSAtomicCompareAndSwapPtrBarrier(old, newBuffer, (void **)&_latestPixelBuffer)) { + old = _latestPixelBuffer; + } + if (old != nil) { + CFRelease(old); + } + if (_onFrameAvailable) { + _onFrameAvailable(); + } + } // switch (_currentDetector) { // case DetectorOnDeviceFace: // [self detectFacesOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; @@ -115,7 +183,6 @@ - (void)captureOutput:(AVCaptureOutput *)output // [self detectTextOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; // break; // } - } if (!CMSampleBufferDataIsReady(sampleBuffer)) { _eventSink(@{ @"event" : @"error", diff --git a/packages/firebase_ml_vision/ios/Classes/TextDetector.m b/packages/firebase_ml_vision/ios/Classes/TextDetector.m index 33d3d44914bb..526e347f3716 100644 --- a/packages/firebase_ml_vision/ios/Classes/TextDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/TextDetector.m @@ -8,7 +8,7 @@ + (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result res FIRVision *vision = [FIRVision vision]; textDetector = [vision textDetector]; } - + NSLog(@"handling text detection"); [textDetector detectInImage:image completion:^(NSArray> *_Nullable features, NSError *_Nullable error) { diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 16d848131a1a..ff48ec0a04d5 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -43,6 +43,29 @@ class FirebaseVision { /// Creates an instance of [TextDetector]. TextDetector textDetector() => new TextDetector._(); + + Future setLiveViewRecognizer(FirebaseVisionDetectorType type) async { + print("sending setLiveVieRecognizer to device"); + final String typeMessage = recognizerMessageType(type); + if (typeMessage == null) return; + await FirebaseVision.channel.invokeMethod("LiveView#setRecognizer", + {"recognizerType": typeMessage}); + } + + String recognizerMessageType(FirebaseVisionDetectorType type) { + switch (type) { + case FirebaseVisionDetectorType.barcode: + return "barcode"; + case FirebaseVisionDetectorType.face: + return "face"; + case FirebaseVisionDetectorType.label: + return "label"; + case FirebaseVisionDetectorType.text: + return "text"; + default: + return null; + } + } } /// Represents an image object used for both on-device and cloud API detectors. @@ -75,3 +98,10 @@ abstract class FirebaseVisionDetector { /// Release model resources for the detector. Future close(); } + +enum FirebaseVisionDetectorType { + barcode, + face, + label, + text, +} diff --git a/packages/firebase_ml_vision/lib/src/live_view.dart b/packages/firebase_ml_vision/lib/src/live_view.dart index 0b5bb0a84dd5..9d95e3a8ddb4 100644 --- a/packages/firebase_ml_vision/lib/src/live_view.dart +++ b/packages/firebase_ml_vision/lib/src/live_view.dart @@ -87,8 +87,8 @@ class LiveViewCameraException implements Exception { String toString() => '$runtimeType($code, $description)'; } -typedef Widget OverlayBuilder(BuildContext context, Size previewImageSize, - List barcodes); +typedef Widget OverlayBuilder( + BuildContext context, Size previewImageSize, dynamic data); // Build the UI texture view of the video data with textureId. class LiveView extends StatefulWidget { @@ -104,14 +104,14 @@ class LiveView extends StatefulWidget { } class LiveViewState extends State { - List scannedCodes = []; + List scannedCodes = []; @override void initState() { super.initState(); widget.controller.addListener(() { setState(() { - scannedCodes = widget.controller.value.scannedBarcodes; + scannedCodes = widget.controller.value.detectedData; }); }); } @@ -145,13 +145,16 @@ class LiveViewCameraValue { /// Is `null` until [isInitialized] is `true`. final Size previewSize; - final List scannedBarcodes; + final List detectedData; + + final FirebaseVisionDetectorType recognizerType; const LiveViewCameraValue({ this.isInitialized, this.errorDescription, this.previewSize, - this.scannedBarcodes, + this.detectedData, + this.recognizerType, }); const LiveViewCameraValue.uninitialized() @@ -172,13 +175,15 @@ class LiveViewCameraValue { bool isTakingPicture, String errorDescription, Size previewSize, - List scannedBarcodes, + List detectedData, + FirebaseVisionDetectorType recognizerType, }) { return new LiveViewCameraValue( isInitialized: isInitialized ?? this.isInitialized, errorDescription: errorDescription, previewSize: previewSize ?? this.previewSize, - scannedBarcodes: scannedBarcodes ?? this.scannedBarcodes, + detectedData: detectedData ?? this.detectedData, + recognizerType: recognizerType ?? this.recognizerType, ); } @@ -188,7 +193,7 @@ class LiveViewCameraValue { 'isInitialized: $isInitialized, ' 'errorDescription: $errorDescription, ' 'previewSize: $previewSize, ' - 'scannedBarcodes: $scannedBarcodes)'; + 'scannedBarcodes: $detectedData)'; } } @@ -247,6 +252,12 @@ class LiveViewCameraController extends ValueNotifier { return _creatingCompleter.future; } + Future setRecognizer( + FirebaseVisionDetectorType recognizerType) async { + await FirebaseVision.instance.setLiveViewRecognizer(recognizerType); + value = value.copyWith(recognizerType: recognizerType); + } + /// Listen to events from the native plugins. /// /// A "cameraClosing" event is sent when the camera is closed automatically by the system (for example when the app go to background). The plugin will try to reopen the camera automatically but any ongoing recording will end. @@ -264,7 +275,13 @@ class LiveViewCameraController extends ValueNotifier { reply.forEach((dynamic barcodeMap) { barcodes.add(new BarcodeContainer(barcodeMap)); }); - value = value.copyWith(scannedBarcodes: barcodes); + value = value.copyWith(detectedData: barcodes); + } else if (recognitionType == "text") { + final List reply = event['textData']; + final detectedData = reply.map((dynamic block) { + return TextBlock.fromBlockData(block); + }).toList(); + value = value.copyWith(detectedData: detectedData); } break; case 'error': diff --git a/packages/firebase_ml_vision/lib/src/text_detector.dart b/packages/firebase_ml_vision/lib/src/text_detector.dart index c6b37d53dc23..05b06798612b 100644 --- a/packages/firebase_ml_vision/lib/src/text_detector.dart +++ b/packages/firebase_ml_vision/lib/src/text_detector.dart @@ -88,6 +88,10 @@ class TextBlock extends TextContainer { .toList(), super._(block); + factory TextBlock.fromBlockData(Map block) { + return new TextBlock._(block); + } + final List _lines; /// The contents of the text block, broken down into individual lines. From 99196c3ccf98b616e3f34dcfd364a46e81ffd85d Mon Sep 17 00:00:00 2001 From: dustin Date: Sat, 14 Jul 2018 13:18:11 -0600 Subject: [PATCH 15/34] WIP: android live text detection. screen coordinates are still not translating properly --- .../firebasemlvision/BarcodeDetector.java | 9 +- .../plugins/firebasemlvision/Detector.java | 28 +++- .../firebasemlvision/DetectorException.java | 25 +++ .../firebasemlvision/FaceDetector.java | 12 +- .../FirebaseMlVisionPlugin.java | 28 +++- .../FlutterResultWrapper.java | 5 + .../firebasemlvision/LabelDetector.java | 9 +- .../firebasemlvision/TextDetector.java | 13 +- .../plugins/firebasemlvision/live/Camera.java | 152 ++++++++++++------ 9 files changed, 211 insertions(+), 70 deletions(-) create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/DetectorException.java create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FlutterResultWrapper.java diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java index 7e42a127eed9..af556227c602 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java @@ -17,17 +17,18 @@ import java.util.List; import java.util.Map; +import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.*; -class BarcodeDetector implements Detector { +public class BarcodeDetector extends Detector { public static final BarcodeDetector instance = new BarcodeDetector(); private static FirebaseVisionBarcodeDetector barcodeDetector; @Override - public void handleDetection(FirebaseVisionImage image, final MethodChannel.Result result) { + public void handleDetection(FirebaseVisionImage image, final OnDetectionFinishedCallback finishedCallback) { if (barcodeDetector == null) barcodeDetector = FirebaseVision.getInstance().getVisionBarcodeDetector(); barcodeDetector .detectInImage(image) @@ -40,13 +41,13 @@ public void onSuccess(List firebaseVisionBarcodes) { addBarcodeData(barcodeData, barcode); barcodes.add(barcodeData); } - result.success(barcodes); + finishedCallback.dataReady(BarcodeDetector.this, barcodes); } }) .addOnFailureListener(new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { - result.error("barcodeDetectorError", e.getLocalizedMessage(), null); + finishedCallback.detectionError(new DetectorException("barcodeDetectorError", e.getLocalizedMessage(), null)); } }); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java index 138092af8845..aec07cd58db5 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java @@ -1,10 +1,32 @@ package io.flutter.plugins.firebasemlvision; import com.google.firebase.ml.vision.common.FirebaseVisionImage; + import io.flutter.plugin.common.MethodChannel; -interface Detector { - void close(MethodChannel.Result result); +public abstract class Detector { + + public interface OnDetectionFinishedCallback { + void dataReady(Detector detector, Object data); + + void detectionError(DetectorException e); + } + + static FlutterResultWrapper defaultFlutterResultWrapper = new FlutterResultWrapper() { + @Override + public Object wrapFlutterResultData(Detector detector, Object data) { + return data; + } + }; + + public abstract void close(MethodChannel.Result result); + +// protected abstract void processImage( +// FirebaseVisionImage image, +// final FlutterResultWrapper resultWrapper, +// final OnDetectionFinishedCallback callback); - void handleDetection(FirebaseVisionImage image, final MethodChannel.Result result); + public abstract void handleDetection( + FirebaseVisionImage image, + OnDetectionFinishedCallback finishedCallback); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/DetectorException.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/DetectorException.java new file mode 100644 index 000000000000..162d137f2eed --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/DetectorException.java @@ -0,0 +1,25 @@ +package io.flutter.plugins.firebasemlvision; + +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugin.common.MethodChannel; + +public class DetectorException extends Exception { + private String detectorExceptionType; + private String detectorExceptionDescription; + private Object exceptionData; + + public DetectorException(String detectorExceptionType, String detectorExceptionDescription, Object exceptionData) { + super(detectorExceptionType + ": " + detectorExceptionDescription); + this.detectorExceptionType = detectorExceptionType; + this.detectorExceptionDescription = detectorExceptionDescription; + this.exceptionData = exceptionData; + } + + public void sendError(EventChannel.EventSink eventSink) { + eventSink.error(detectorExceptionType, detectorExceptionDescription, exceptionData); + } + + public void sendError(MethodChannel.Result result) { + result.error(detectorExceptionType, detectorExceptionDescription, exceptionData); + } +} diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java index 384e9f91257d..8d600f5aeb58 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java @@ -1,12 +1,18 @@ package io.flutter.plugins.firebasemlvision; import com.google.firebase.ml.vision.common.FirebaseVisionImage; + +import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; -class FaceDetector implements Detector { +public class FaceDetector extends Detector { + @Override - public void handleDetection(FirebaseVisionImage image, MethodChannel.Result result) {} + public void handleDetection(FirebaseVisionImage image, OnDetectionFinishedCallback finishedCallback) { + + } @Override - public void close(MethodChannel.Result result) {} + public void close(MethodChannel.Result result) { + } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 14c6d71e20f1..25215c8943b7 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -109,7 +109,7 @@ public static void registerWith(Registrar registrar) { } @Override - public void onMethodCall(MethodCall call, Result result) { + public void onMethodCall(MethodCall call, final Result result) { switch (call.method) { case "init": if (camera != null) { @@ -141,9 +141,21 @@ public void onMethodCall(MethodCall call, Result result) { result.success(null); break; } + case "LiveView#setRecognizer": + break; case "BarcodeDetector#detectInImage": FirebaseVisionImage image = filePathToVisionImage((String) call.arguments, result); - if (image != null) BarcodeDetector.instance.handleDetection(image, result); + if (image != null) BarcodeDetector.instance.handleDetection(image, new Detector.OnDetectionFinishedCallback() { + @Override + public void dataReady(Detector detector, Object data) { + result.success(data); + } + + @Override + public void detectionError(DetectorException e) { + e.sendError(result); + } + }); break; case "BarcodeDetector#close": BarcodeDetector.instance.close(result); @@ -158,7 +170,17 @@ public void onMethodCall(MethodCall call, Result result) { break; case "TextDetector#detectInImage": image = filePathToVisionImage((String) call.arguments, result); - if (image != null) TextDetector.instance.handleDetection(image, result); + if (image != null) TextDetector.instance.handleDetection(image, new Detector.OnDetectionFinishedCallback() { + @Override + public void dataReady(Detector detector, Object data) { + result.success(data); + } + + @Override + public void detectionError(DetectorException e) { + e.sendError(result); + } + }); break; case "TextDetector#close": TextDetector.instance.close(result); diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FlutterResultWrapper.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FlutterResultWrapper.java new file mode 100644 index 000000000000..15d7fa4c2481 --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FlutterResultWrapper.java @@ -0,0 +1,5 @@ +package io.flutter.plugins.firebasemlvision; + +public interface FlutterResultWrapper { + Object wrapFlutterResultData(Detector detector, Object data); +} diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java index fd7f21154a57..c4c04fce7281 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java @@ -1,11 +1,16 @@ package io.flutter.plugins.firebasemlvision; import com.google.firebase.ml.vision.common.FirebaseVisionImage; + +import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; -class LabelDetector implements Detector { +public class LabelDetector extends Detector { + @Override - public void handleDetection(FirebaseVisionImage image, MethodChannel.Result result) {} + public void handleDetection(FirebaseVisionImage image, OnDetectionFinishedCallback finishedCallback) { + + } @Override public void close(MethodChannel.Result result) {} diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java index cf8780962615..dfe26756ebf3 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java @@ -11,6 +11,9 @@ import com.google.firebase.ml.vision.text.FirebaseVisionText; import com.google.firebase.ml.vision.text.FirebaseVisionTextDetector; +import org.w3c.dom.Text; + +import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; @@ -20,14 +23,15 @@ import java.util.List; import java.util.Map; -public class TextDetector implements Detector { +public class TextDetector extends Detector { public static final TextDetector instance = new TextDetector(); private static FirebaseVisionTextDetector textDetector; private TextDetector() { } - public void handleDetection(FirebaseVisionImage image, final MethodChannel.Result result) { + @Override + public void handleDetection(FirebaseVisionImage image, final OnDetectionFinishedCallback finishedCallback) { if (textDetector == null) textDetector = FirebaseVision.getInstance().getVisionTextDetector(); textDetector .detectInImage(image) @@ -63,16 +67,17 @@ public void onSuccess(FirebaseVisionText firebaseVisionText) { blockData.put("lines", lines); blocks.add(blockData); } - result.success(blocks); + finishedCallback.dataReady(TextDetector.this, blocks); } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception exception) { - result.error("textDetectorError", exception.getLocalizedMessage(), null); + finishedCallback.detectionError(new DetectorException("textDetectorError", exception.getLocalizedMessage(), null)); } }); + } public void close(MethodChannel.Result result) { diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index ad5ba8fc2e6f..f3f73c81ef0d 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -52,9 +52,15 @@ import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugin.common.PluginRegistry; +import io.flutter.plugins.firebasemlvision.BarcodeDetector; +import io.flutter.plugins.firebasemlvision.DetectorException; +import io.flutter.plugins.firebasemlvision.FlutterResultWrapper; +import io.flutter.plugins.firebasemlvision.TextDetector; import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; import io.flutter.view.FlutterView; +import io.flutter.plugins.firebasemlvision.Detector; + import static io.flutter.plugins.firebasemlvision.FirebaseMlVisionPlugin.CAMERA_REQUEST_ID; import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.BARCODE_DISPLAY_VALUE; import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.BARCODE_RAW_VALUE; @@ -89,8 +95,62 @@ public class Camera { private HandlerThread mBackgroundThread; private Handler mBackgroundHandler; private Surface imageReaderSurface; - private CameraCharacteristics cameraCharacteristics; private WindowManager windowManager; + private Detector currentDetector = TextDetector.instance; + + private FlutterResultWrapper liveDetectorResultWrapper = new FlutterResultWrapper() { + @SuppressWarnings("unchecked") + @Override + public Object wrapFlutterResultData(Detector detector, Object data) { + Map event = new HashMap<>(); + event.put("eventType", "recognized"); + String dataType; + String dataLabel; + if (detector instanceof BarcodeDetector) { + dataType = "barcode"; + dataLabel = "barcodeData"; + } else if (detector instanceof TextDetector) { + dataType = "text"; + dataLabel = "textData"; + } else { + // unsupported live detector + return data; + } + event.put("recognitionType", dataType); + event.put(dataLabel, data); + return event; + } + }; + + private Detector.OnDetectionFinishedCallback liveDetectorFinishedCallback = new Detector.OnDetectionFinishedCallback() { + @Override + public void dataReady(Detector detector, Object data) { + shouldThrottle.set(false); + Map event = new HashMap<>(); + event.put("eventType", "recognized"); + String dataType; + String dataLabel; + if (detector instanceof BarcodeDetector) { + dataType = "barcode"; + dataLabel = "barcodeData"; + } else if (detector instanceof TextDetector) { + dataType = "text"; + dataLabel = "textData"; + } else { + // unsupported live detector + return; + } + event.put("recognitionType", dataType); + event.put(dataLabel, data); + eventSink.success(event); + } + + @Override + public void detectionError(DetectorException e) { + shouldThrottle.set(false); + e.sendError(eventSink); + } + }; public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonNull final String resolutionPreset, @NonNull final MethodChannel.Result result) { @@ -118,7 +178,7 @@ public Camera(PluginRegistry.Registrar registrar, final String cameraName, @NonN throw new IllegalArgumentException("Unknown preset: " + resolutionPreset); } - cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraName); + CameraCharacteristics cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraName); StreamConfigurationMap streamConfigurationMap = cameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); @@ -276,14 +336,6 @@ private static ByteBuffer YUV_420_888toNV21(Image image) { .put(uBuffer); return output; -// nv21 = new byte[ySize + uSize + vSize]; -// -// //U and V are swapped -// yBuffer.get(nv21, 0, ySize); -// vBuffer.get(nv21, ySize, vSize); -// uBuffer.get(nv21, ySize + vSize, uSize); -// -// return nv21; } private int getRotation() { @@ -328,11 +380,10 @@ private int getRotation() { private AtomicBoolean shouldThrottle = new AtomicBoolean(false); private void processImage(Image image) { + if (eventSink == null) return; if (shouldThrottle.get()) { -// Log.d("ML", "should throttle"); return; } - Log.d("ML", "about to process a vision frame"); shouldThrottle.set(true); ByteBuffer imageBuffer = YUV_420_888toNV21(image); FirebaseVisionImageMetadata metadata = new FirebaseVisionImageMetadata.Builder() @@ -343,26 +394,25 @@ private void processImage(Image image) { .build(); FirebaseVisionImage firebaseVisionImage = FirebaseVisionImage.fromByteBuffer(imageBuffer, metadata); - FirebaseVisionBarcodeDetector visionBarcodeDetector = FirebaseVision.getInstance().getVisionBarcodeDetector(); - visionBarcodeDetector.detectInImage(firebaseVisionImage).addOnSuccessListener(new OnSuccessListener>() { - @Override - public void onSuccess(List firebaseVisionBarcodes) { - shouldThrottle.set(false); - sendRecognizedBarcodes(firebaseVisionBarcodes); - Log.d("ML", "barcode scan success, got codes: " + firebaseVisionBarcodes.size()); - } - }).addOnFailureListener(new OnFailureListener() { - @Override - public void onFailure(@NonNull Exception e) { - shouldThrottle.set(false); - sendErrorEvent(e.getLocalizedMessage()); - Log.d("ML", "barcode scan failure, message: " + e.getMessage()); - } - }); - Log.d("ML", "got an image from the image reader"); + currentDetector.handleDetection(firebaseVisionImage, liveDetectorFinishedCallback); + +// FirebaseVisionBarcodeDetector visionBarcodeDetector = FirebaseVision.getInstance().getVisionBarcodeDetector(); +// visionBarcodeDetector.detectInImage(firebaseVisionImage).addOnSuccessListener(new OnSuccessListener>() { +// @Override +// public void onSuccess(List firebaseVisionBarcodes) { +// shouldThrottle.set(false); +// sendRecognizedBarcodes(firebaseVisionBarcodes); +// } +// }).addOnFailureListener(new OnFailureListener() { +// @Override +// public void onFailure(@NonNull Exception e) { +// shouldThrottle.set(false); +// sendErrorEvent(e.getLocalizedMessage()); +// } +// }); } - ImageReader.OnImageAvailableListener imageAvailable = new ImageReader.OnImageAvailableListener() { + private ImageReader.OnImageAvailableListener imageAvailable = new ImageReader.OnImageAvailableListener() { @Override public void onImageAvailable(ImageReader reader) { Image image = reader.acquireLatestImage(); @@ -501,27 +551,27 @@ public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession null); } - private void sendRecognizedBarcodes(List barcodes) { - if (eventSink != null) { - List> outputMap = new ArrayList<>(); - for (FirebaseVisionBarcode barcode : barcodes) { - Map barcodeData = new HashMap<>(); - Rect boundingBox = barcode.getBoundingBox(); - if (boundingBox != null) { - barcodeData.putAll(DetectedItemUtils.rectToFlutterMap(boundingBox)); - } - barcodeData.put(BARCODE_VALUE_TYPE, barcode.getValueType()); - barcodeData.put(BARCODE_DISPLAY_VALUE, barcode.getDisplayValue()); - barcodeData.put(BARCODE_RAW_VALUE, barcode.getRawValue()); - outputMap.add(barcodeData); - } - Map event = new HashMap<>(); - event.put("eventType", "recognized"); - event.put("recognitionType", "barcode"); - event.put("barcodeData", outputMap); - eventSink.success(event); - } - } +// private void sendRecognizedBarcodes(List barcodes) { +// if (eventSink != null) { +// List> outputMap = new ArrayList<>(); +// for (FirebaseVisionBarcode barcode : barcodes) { +// Map barcodeData = new HashMap<>(); +// Rect boundingBox = barcode.getBoundingBox(); +// if (boundingBox != null) { +// barcodeData.putAll(DetectedItemUtils.rectToFlutterMap(boundingBox)); +// } +// barcodeData.put(BARCODE_VALUE_TYPE, barcode.getValueType()); +// barcodeData.put(BARCODE_DISPLAY_VALUE, barcode.getDisplayValue()); +// barcodeData.put(BARCODE_RAW_VALUE, barcode.getRawValue()); +// outputMap.add(barcodeData); +// } +// Map event = new HashMap<>(); +// event.put("eventType", "recognized"); +// event.put("recognitionType", "barcode"); +// event.put("barcodeData", outputMap); +// eventSink.success(event); +// } +// } private void sendErrorEvent(String errorDescription) { if (eventSink != null) { From dacaccfba74b9ab9ddb6a40dcee433f0001139d0 Mon Sep 17 00:00:00 2001 From: dustin Date: Wed, 18 Jul 2018 22:43:24 -0700 Subject: [PATCH 16/34] WIP: Android legacy camera detector impl. Min SDK on this lib is 16. refactor detectors so that we can send messages back to flutter on Results and Sinks --- .../firebasemlvision/BarcodeDetector.java | 20 +- .../plugins/firebasemlvision/Detector.java | 52 +- .../firebasemlvision/FaceDetector.java | 7 +- .../FirebaseMlVisionPlugin.java | 107 ++- .../firebasemlvision/LabelDetector.java | 7 +- .../firebasemlvision/TextDetector.java | 26 +- .../plugins/firebasemlvision/live/Camera.java | 41 +- .../firebasemlvision/live/LegacyCamera.java | 829 ++++++++++++++++++ .../example/lib/live_preview.dart | 2 +- .../firebase_ml_vision/lib/src/live_view.dart | 2 +- 10 files changed, 980 insertions(+), 113 deletions(-) create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java index af556227c602..79ab1a67944e 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java @@ -2,6 +2,7 @@ import android.graphics.Rect; import android.support.annotation.NonNull; +import android.support.annotation.Nullable; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; @@ -9,15 +10,14 @@ import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode; import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector; import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; -import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; @@ -28,7 +28,7 @@ public class BarcodeDetector extends Detector { private static FirebaseVisionBarcodeDetector barcodeDetector; @Override - public void handleDetection(FirebaseVisionImage image, final OnDetectionFinishedCallback finishedCallback) { + void processImage(FirebaseVisionImage image, final OperationFinishedCallback finishedCallback) { if (barcodeDetector == null) barcodeDetector = FirebaseVision.getInstance().getVisionBarcodeDetector(); barcodeDetector .detectInImage(image) @@ -41,25 +41,29 @@ public void onSuccess(List firebaseVisionBarcodes) { addBarcodeData(barcodeData, barcode); barcodes.add(barcodeData); } - finishedCallback.dataReady(BarcodeDetector.this, barcodes); + finishedCallback.success(BarcodeDetector.this, barcodes); } }) .addOnFailureListener(new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { - finishedCallback.detectionError(new DetectorException("barcodeDetectorError", e.getLocalizedMessage(), null)); + finishedCallback.error(new DetectorException("barcodeDetectorError", e.getLocalizedMessage(), null)); } }); } @Override - public void close(MethodChannel.Result result) { + public void close(@Nullable OperationFinishedCallback callback) { if (barcodeDetector != null) { try { barcodeDetector.close(); - result.success(null); + if (callback != null) { + callback.success(this, null); + } } catch (IOException e) { - result.error("barcodeDetectorError", e.getLocalizedMessage(), null); + if (callback != null) { + callback.error(new DetectorException("barcodeDetectorError", e.getLocalizedMessage(), null)); + } } } barcodeDetector = null; diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java index aec07cd58db5..1347df83988a 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java @@ -1,32 +1,48 @@ package io.flutter.plugins.firebasemlvision; +import android.support.annotation.Nullable; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import io.flutter.plugin.common.MethodChannel; +import java.util.concurrent.atomic.AtomicBoolean; public abstract class Detector { - public interface OnDetectionFinishedCallback { - void dataReady(Detector detector, Object data); + public interface OperationFinishedCallback { + void success(Detector detector, Object data); - void detectionError(DetectorException e); + void error(DetectorException e); } - static FlutterResultWrapper defaultFlutterResultWrapper = new FlutterResultWrapper() { - @Override - public Object wrapFlutterResultData(Detector detector, Object data) { - return data; - } - }; - - public abstract void close(MethodChannel.Result result); + private final AtomicBoolean shouldThrottle = new AtomicBoolean(false); -// protected abstract void processImage( -// FirebaseVisionImage image, -// final FlutterResultWrapper resultWrapper, -// final OnDetectionFinishedCallback callback); + public abstract void close(@Nullable OperationFinishedCallback callback); - public abstract void handleDetection( + public void handleDetection( FirebaseVisionImage image, - OnDetectionFinishedCallback finishedCallback); + final OperationFinishedCallback finishedCallback) { + if (shouldThrottle.get()) { + return; + } + processImage(image, new OperationFinishedCallback() { + @Override + public void success(Detector detector, Object data) { + shouldThrottle.set(false); + finishedCallback.success(detector, data); + } + + @Override + public void error(DetectorException e) { + shouldThrottle.set(false); + finishedCallback.error(e); + } + }); + + // Begin throttling until this frame of input has been processed, either in onSuccess or + // onFailure. + shouldThrottle.set(true); + } + + abstract void processImage( + FirebaseVisionImage image, OperationFinishedCallback finishedCallback); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java index 8d600f5aeb58..764490dda1f9 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java @@ -1,18 +1,19 @@ package io.flutter.plugins.firebasemlvision; +import android.support.annotation.Nullable; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; public class FaceDetector extends Detector { @Override - public void handleDetection(FirebaseVisionImage image, OnDetectionFinishedCallback finishedCallback) { + void processImage(FirebaseVisionImage image, OperationFinishedCallback finishedCallback) { } @Override - public void close(MethodChannel.Result result) { + public void close(@Nullable OperationFinishedCallback callback) { } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 25215c8943b7..88936cb4cb77 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -4,17 +4,18 @@ import android.app.Application; import android.graphics.Bitmap; import android.graphics.Matrix; -import android.hardware.camera2.CameraAccessException; import android.net.Uri; import android.os.Bundle; import android.provider.MediaStore; import android.support.annotation.Nullable; import android.support.media.ExifInterface; +import android.util.Log; import com.google.firebase.ml.vision.common.FirebaseVisionImage; import java.io.File; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,7 +28,7 @@ import io.flutter.plugins.firebasemlvision.live.Camera; import io.flutter.plugins.firebasemlvision.live.CameraInfo; import io.flutter.plugins.firebasemlvision.live.CameraInfoException; -import io.flutter.view.FlutterView; +import io.flutter.plugins.firebasemlvision.live.LegacyCamera; /** * FirebaseMlVisionPlugin @@ -38,7 +39,7 @@ public class FirebaseMlVisionPlugin implements MethodCallHandler { private Activity activity; @Nullable - private Camera camera; + private LegacyCamera camera; private FirebaseMlVisionPlugin(Registrar registrar) { this.registrar = registrar; @@ -60,13 +61,16 @@ public void onActivityStarted(Activity activity) { @Override public void onActivityResumed(Activity activity) { - if (camera != null && camera.getRequestingPermission()) { - camera.setRequestingPermission(false); - return; - } +// if (camera != null && camera.getRequestingPermission()) { +// camera.setRequestingPermission(false); +// return; +// } if (activity == FirebaseMlVisionPlugin.this.activity) { if (camera != null) { - camera.open(null); + try { + camera.start(null); + } catch (IOException ignored) { + } } } } @@ -75,7 +79,7 @@ public void onActivityResumed(Activity activity) { public void onActivityPaused(Activity activity) { if (activity == FirebaseMlVisionPlugin.this.activity) { if (camera != null) { - camera.close(); + camera.stop(); } } } @@ -84,7 +88,7 @@ public void onActivityPaused(Activity activity) { public void onActivityStopped(Activity activity) { if (activity == FirebaseMlVisionPlugin.this.activity) { if (camera != null) { - camera.close(); + camera.stop(); } } } @@ -113,29 +117,52 @@ public void onMethodCall(MethodCall call, final Result result) { switch (call.method) { case "init": if (camera != null) { - camera.close(); + camera.stop(); } result.success(null); break; case "availableCameras": - try { - List> cameras = CameraInfo.getAvailableCameras(registrar.activeContext()); - result.success(cameras); - } catch (CameraInfoException e) { - result.error("cameraAccess", e.getMessage(), null); - } +// try { +// List> cameras = CameraInfo.getAvailableCameras(registrar.activeContext()); +// result.success(cameras); +// } catch (CameraInfoException e) { +// result.error("cameraAccess", e.getMessage(), null); +// } + List> cameras = LegacyCamera.listAvailableCameraDetails(); + result.success(cameras); break; case "initialize": - String cameraName = call.argument("cameraName"); + Log.d("ML", "initialize"); + int cameraName = call.argument("cameraName"); //TODO: set camera facing String resolutionPreset = call.argument("resolutionPreset"); if (camera != null) { - camera.close(); + camera.stop(); + } + camera = new LegacyCamera(registrar); //new Camera(registrar, cameraName, resolutionPreset, result); + camera.setMachineLearningFrameProcessor(TextDetector.instance); + try { + camera.start(new LegacyCamera.OnCameraOpenedCallback() { + @Override + public void onOpened(long textureId, int width, int height) { + Map reply = new HashMap<>(); + reply.put("textureId", textureId); + reply.put("previewWidth", width); + reply.put("previewHeight", height); + result.success(reply); + } + + @Override + public void onFailed(Exception e) { + result.error("CameraInitializationError", e.getLocalizedMessage(), null); + } + }); + } catch (IOException e) { + result.error("CameraInitializationError", e.getLocalizedMessage(), null); } - camera = new Camera(registrar, cameraName, resolutionPreset, result); break; case "dispose": { if (camera != null) { - camera.dispose(); + camera.release(); camera = null; } result.success(null); @@ -145,20 +172,30 @@ public void onMethodCall(MethodCall call, final Result result) { break; case "BarcodeDetector#detectInImage": FirebaseVisionImage image = filePathToVisionImage((String) call.arguments, result); - if (image != null) BarcodeDetector.instance.handleDetection(image, new Detector.OnDetectionFinishedCallback() { + if (image != null) BarcodeDetector.instance.handleDetection(image, new Detector.OperationFinishedCallback() { @Override - public void dataReady(Detector detector, Object data) { + public void success(Detector detector, Object data) { result.success(data); } @Override - public void detectionError(DetectorException e) { + public void error(DetectorException e) { e.sendError(result); } }); break; case "BarcodeDetector#close": - BarcodeDetector.instance.close(result); + BarcodeDetector.instance.close(new Detector.OperationFinishedCallback() { + @Override + public void success(Detector detector, Object data) { + result.success(null); + } + + @Override + public void error(DetectorException e) { + e.sendError(result); + } + }); break; case "FaceDetector#detectInImage": break; @@ -170,20 +207,30 @@ public void detectionError(DetectorException e) { break; case "TextDetector#detectInImage": image = filePathToVisionImage((String) call.arguments, result); - if (image != null) TextDetector.instance.handleDetection(image, new Detector.OnDetectionFinishedCallback() { + if (image != null) TextDetector.instance.handleDetection(image, new Detector.OperationFinishedCallback() { @Override - public void dataReady(Detector detector, Object data) { + public void success(Detector detector, Object data) { result.success(data); } @Override - public void detectionError(DetectorException e) { + public void error(DetectorException e) { e.sendError(result); } }); break; case "TextDetector#close": - TextDetector.instance.close(result); + TextDetector.instance.close(new Detector.OperationFinishedCallback() { + @Override + public void success(Detector detector, Object data) { + result.success(null); + } + + @Override + public void error(DetectorException e) { + e.sendError(result); + } + }); break; default: result.notImplemented(); @@ -226,7 +273,7 @@ private class CameraRequestPermissionsListener public boolean onRequestPermissionsResult(int id, String[] permissions, int[] grantResults) { if (id == CAMERA_REQUEST_ID) { if (camera != null) { - camera.continueRequestingPermissions(); +// camera.continueRequestingPermissions(); } return true; } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java index c4c04fce7281..31dd5ed6b725 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java @@ -1,17 +1,18 @@ package io.flutter.plugins.firebasemlvision; +import android.support.annotation.Nullable; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; public class LabelDetector extends Detector { @Override - public void handleDetection(FirebaseVisionImage image, OnDetectionFinishedCallback finishedCallback) { + void processImage(FirebaseVisionImage image, OperationFinishedCallback finishedCallback) { } @Override - public void close(MethodChannel.Result result) {} + public void close(@Nullable OperationFinishedCallback callback) {} } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java index dfe26756ebf3..49f19ce9fe10 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java @@ -3,6 +3,7 @@ import android.graphics.Point; import android.graphics.Rect; import android.support.annotation.NonNull; +import android.support.annotation.Nullable; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; @@ -11,10 +12,6 @@ import com.google.firebase.ml.vision.text.FirebaseVisionText; import com.google.firebase.ml.vision.text.FirebaseVisionTextDetector; -import org.w3c.dom.Text; - -import io.flutter.plugin.common.EventChannel; -import io.flutter.plugin.common.MethodChannel; import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; import java.io.IOException; @@ -22,16 +19,19 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; public class TextDetector extends Detector { public static final TextDetector instance = new TextDetector(); private static FirebaseVisionTextDetector textDetector; + private final AtomicBoolean shouldThrottle = new AtomicBoolean(false); + private TextDetector() { } @Override - public void handleDetection(FirebaseVisionImage image, final OnDetectionFinishedCallback finishedCallback) { + void processImage(FirebaseVisionImage image, final OperationFinishedCallback finishedCallback) { if (textDetector == null) textDetector = FirebaseVision.getInstance().getVisionTextDetector(); textDetector .detectInImage(image) @@ -67,26 +67,30 @@ public void onSuccess(FirebaseVisionText firebaseVisionText) { blockData.put("lines", lines); blocks.add(blockData); } - finishedCallback.dataReady(TextDetector.this, blocks); + finishedCallback.success(TextDetector.this, blocks); } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception exception) { - finishedCallback.detectionError(new DetectorException("textDetectorError", exception.getLocalizedMessage(), null)); + finishedCallback.error(new DetectorException("textDetectorError", exception.getLocalizedMessage(), null)); } }); - } - public void close(MethodChannel.Result result) { + @Override + public void close(@Nullable OperationFinishedCallback callback) { if (textDetector != null) { try { textDetector.close(); - result.success(null); + if (callback != null) { + callback.success(TextDetector.this, null); + } } catch (IOException exception) { - result.error("textDetectorError", exception.getLocalizedMessage(), null); + if (callback != null) { + callback.error(new DetectorException("textDetectorError", exception.getLocalizedMessage(), null)); + } } textDetector = null; diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index f3f73c81ef0d..3b5009958228 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -6,7 +6,6 @@ import android.content.Context; import android.content.pm.PackageManager; import android.graphics.ImageFormat; -import android.graphics.Rect; import android.graphics.SurfaceTexture; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; @@ -31,11 +30,6 @@ import android.view.Surface; import android.view.WindowManager; -import com.google.android.gms.tasks.OnFailureListener; -import com.google.android.gms.tasks.OnSuccessListener; -import com.google.firebase.ml.vision.FirebaseVision; -import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode; -import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector; import com.google.firebase.ml.vision.common.FirebaseVisionImage; import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; @@ -54,17 +48,12 @@ import io.flutter.plugin.common.PluginRegistry; import io.flutter.plugins.firebasemlvision.BarcodeDetector; import io.flutter.plugins.firebasemlvision.DetectorException; -import io.flutter.plugins.firebasemlvision.FlutterResultWrapper; import io.flutter.plugins.firebasemlvision.TextDetector; -import io.flutter.plugins.firebasemlvision.util.DetectedItemUtils; import io.flutter.view.FlutterView; import io.flutter.plugins.firebasemlvision.Detector; import static io.flutter.plugins.firebasemlvision.FirebaseMlVisionPlugin.CAMERA_REQUEST_ID; -import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.BARCODE_DISPLAY_VALUE; -import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.BARCODE_RAW_VALUE; -import static io.flutter.plugins.firebasemlvision.constants.VisionBarcodeConstants.BARCODE_VALUE_TYPE; @TargetApi(Build.VERSION_CODES.LOLLIPOP) public class Camera { @@ -98,33 +87,9 @@ public class Camera { private WindowManager windowManager; private Detector currentDetector = TextDetector.instance; - private FlutterResultWrapper liveDetectorResultWrapper = new FlutterResultWrapper() { - @SuppressWarnings("unchecked") + private Detector.OperationFinishedCallback liveDetectorFinishedCallback = new Detector.OperationFinishedCallback() { @Override - public Object wrapFlutterResultData(Detector detector, Object data) { - Map event = new HashMap<>(); - event.put("eventType", "recognized"); - String dataType; - String dataLabel; - if (detector instanceof BarcodeDetector) { - dataType = "barcode"; - dataLabel = "barcodeData"; - } else if (detector instanceof TextDetector) { - dataType = "text"; - dataLabel = "textData"; - } else { - // unsupported live detector - return data; - } - event.put("recognitionType", dataType); - event.put(dataLabel, data); - return event; - } - }; - - private Detector.OnDetectionFinishedCallback liveDetectorFinishedCallback = new Detector.OnDetectionFinishedCallback() { - @Override - public void dataReady(Detector detector, Object data) { + public void success(Detector detector, Object data) { shouldThrottle.set(false); Map event = new HashMap<>(); event.put("eventType", "recognized"); @@ -146,7 +111,7 @@ public void dataReady(Detector detector, Object data) { } @Override - public void detectionError(DetectorException e) { + public void error(DetectorException e) { shouldThrottle.set(false); e.sendError(eventSink); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java new file mode 100644 index 000000000000..bb6c76cbbe59 --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java @@ -0,0 +1,829 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package io.flutter.plugins.firebasemlvision.live; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.Activity; +import android.content.Context; +import android.graphics.ImageFormat; +import android.graphics.SurfaceTexture; +import android.hardware.Camera; +import android.hardware.Camera.CameraInfo; +import android.support.annotation.Nullable; +import android.support.annotation.RequiresPermission; +import android.util.Log; +import android.view.Surface; +import android.view.SurfaceHolder; +import android.view.WindowManager; + +import com.google.android.gms.common.images.Size; +import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; + +import java.io.IOException; +import java.lang.Thread.State; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugin.common.PluginRegistry; +import io.flutter.plugins.firebasemlvision.BarcodeDetector; +import io.flutter.plugins.firebasemlvision.Detector; +import io.flutter.plugins.firebasemlvision.DetectorException; +import io.flutter.plugins.firebasemlvision.TextDetector; +import io.flutter.view.FlutterView; + +/** + * Manages the camera and allows UI updates on top of it (e.g. overlaying extra Graphics or + * displaying extra information). This receives preview frames from the camera at a specified rate, + * sending those frames to child classes' detectors / classifiers as fast as it is able to process. + */ +@SuppressLint("MissingPermission") +public class LegacyCamera { + @SuppressLint("InlinedApi") + public static final int CAMERA_FACING_BACK = CameraInfo.CAMERA_FACING_BACK; + + @SuppressLint("InlinedApi") + public static final int CAMERA_FACING_FRONT = CameraInfo.CAMERA_FACING_FRONT; + + private static final String TAG = "MIDemoApp:CameraSource"; + + public interface OnCameraOpenedCallback { + void onOpened(long textureId, int width, int height); + void onFailed(Exception e); + } + + /** + * The dummy surface texture must be assigned a chosen name. Since we never use an OpenGL context, + * we can choose any ID we want here. The dummy surface texture is not a crazy hack - it is + * actually how the camera team recommends using the camera without a preview. + */ + private static final int DUMMY_TEXTURE_NAME = 100; + + /** + * If the absolute difference between a preview size aspect ratio and a picture size aspect ratio + * is less than this tolerance, they are considered to be the same aspect ratio. + */ + private static final float ASPECT_RATIO_TOLERANCE = 0.01f; + + protected Activity activity; + + private PluginRegistry.Registrar registrar; + + private final FlutterView.SurfaceTextureEntry textureEntry; + + private EventChannel.EventSink eventSink; + + private Camera camera; + + protected int facing = CAMERA_FACING_BACK; + + /** + * Rotation of the device, and thus the associated preview images captured from the device. See + * Frame.Metadata#getRotation(). + */ + private int rotation; + + private Size previewSize; + + // These values may be requested by the caller. Due to hardware limitations, we may need to + // select close, but not exactly the same values for these. + private final float requestedFps = 20.0f; + private final int requestedPreviewWidth = 1280; + private final int requestedPreviewHeight = 960; + private final boolean requestedAutoFocus = true; + + // These instances need to be held onto to avoid GC of their underlying resources. Even though + // these aren't used outside of the method that creates them, they still must have hard + // references maintained to them. + private SurfaceTexture dummySurfaceTexture; + + // True if a SurfaceTexture is being used for the preview, false if a SurfaceHolder is being + // used for the preview. We want to be compatible back to Gingerbread, but SurfaceTexture + // wasn't introduced until Honeycomb. Since the interface cannot use a SurfaceTexture, if the + // developer wants to display a preview we must use a SurfaceHolder. If the developer doesn't + // want to display a preview we use a SurfaceTexture if we are running at least Honeycomb. + private boolean usingSurfaceTexture; + + /** + * Dedicated thread and associated runnable for calling into the detector with frames, as the + * frames become available from the camera. + */ + private Thread processingThread; + + private final FrameProcessingRunnable processingRunnable; + + private final Object processorLock = new Object(); + private Detector detector; + + /** + * Map to convert between a byte array, received from the camera, and its associated byte buffer. + * We use byte buffers internally because this is a more efficient way to call into native code + * later (avoids a potential copy). + *

+ *

Note: uses IdentityHashMap here instead of HashMap because the behavior of an array's + * equals, hashCode and toString methods is both useless and unexpected. IdentityHashMap enforces + * identity ('==') check on the keys. + */ + private final Map bytesToByteBuffer = new IdentityHashMap<>(); + + private Detector.OperationFinishedCallback liveDetectorFinishedCallback = new Detector.OperationFinishedCallback() { + @Override + public void success(Detector detector, Object data) { + Map event = new HashMap<>(); + event.put("eventType", "recognized"); + String dataType; + String dataLabel; + if (detector instanceof BarcodeDetector) { + dataType = "barcode"; + dataLabel = "barcodeData"; + } else if (detector instanceof TextDetector) { + dataType = "text"; + dataLabel = "textData"; + } else { + // unsupported live detector + return; + } + event.put("recognitionType", dataType); + event.put(dataLabel, data); + eventSink.success(event); + } + + @Override + public void error(DetectorException e) { + e.sendError(eventSink); + } + }; + + + public LegacyCamera(PluginRegistry.Registrar registrar) { + this.registrar = registrar; + this.activity = registrar.activity(); + this.textureEntry = registrar.view().createSurfaceTexture(); + processingRunnable = new FrameProcessingRunnable(); + + registerEventChannel(); + } + + private void registerEventChannel() { + new EventChannel( + registrar.messenger(), "plugins.flutter.io/firebase_ml_vision/liveViewEvents" + textureEntry.id()) + .setStreamHandler( + new EventChannel.StreamHandler() { + @Override + public void onListen(Object arguments, EventChannel.EventSink eventSink) { + LegacyCamera.this.eventSink = eventSink; + } + + @Override + public void onCancel(Object arguments) { + LegacyCamera.this.eventSink = null; + } + }); + } + + // ============================================================================================== + // Public + // ============================================================================================== + + /** + * Stops the camera and releases the resources of the camera and underlying detector. + */ + public void release() { + synchronized (processorLock) { + stop(); + processingRunnable.release(); + + if (detector != null) { + detector.close(null); + } + } + } + +// /** +// * Opens the camera and starts sending preview frames to the underlying detector. The preview +// * frames are not displayed. +// * +// * @throws IOException if the camera's preview texture or display could not be initialized +// */ +// @SuppressLint("MissingPermission") +// @RequiresPermission(Manifest.permission.CAMERA) +// public synchronized LegacyCamera start(OnCameraOpenedCallback callback) throws IOException { +// if (camera != null) { +// return this; +// } +// +// camera = createCamera(callback); +// dummySurfaceTexture = new SurfaceTexture(DUMMY_TEXTURE_NAME); +// camera.setPreviewTexture(dummySurfaceTexture); +// usingSurfaceTexture = true; +// camera.startPreview(); +// +// processingThread = new Thread(processingRunnable); +// processingRunnable.setActive(true); +// processingThread.start(); +// return this; +// } + + /** + * Opens the camera and starts sending preview frames to the underlying detector. The supplied + * surface holder is used for the preview so frames can be displayed to the user. + * +// * @param surfaceHolder the surface holder to use for the preview frames + * @throws IOException if the supplied surface holder could not be used as the preview display + */ + @RequiresPermission(Manifest.permission.CAMERA) + public synchronized LegacyCamera start(OnCameraOpenedCallback callback) throws IOException { + if (camera != null) { + return this; + } + + camera = createCamera(callback); + + SurfaceTexture surfaceTexture = textureEntry.surfaceTexture(); + surfaceTexture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); + + camera.setPreviewTexture(surfaceTexture); + usingSurfaceTexture = true; + camera.startPreview(); + + processingThread = new Thread(processingRunnable); + processingRunnable.setActive(true); + processingThread.start(); + + return this; + } + + /** + * Closes the camera and stops sending frames to the underlying frame detector. + *

+ *

This camera source may be restarted again by calling {@link + * #start(OnCameraOpenedCallback)}. + *

+ *

Call {@link #release()} instead to completely shut down this camera source and release the + * resources of the underlying detector. + */ + public synchronized void stop() { + processingRunnable.setActive(false); + if (processingThread != null) { + try { + // Wait for the thread to complete to ensure that we can't have multiple threads + // executing at the same time (i.e., which would happen if we called start too + // quickly after stop). + processingThread.join(); + } catch (InterruptedException e) { + Log.d(TAG, "Frame processing thread interrupted on release."); + } + processingThread = null; + } + + if (camera != null) { + camera.stopPreview(); + camera.setPreviewCallbackWithBuffer(null); + try { + if (usingSurfaceTexture) { + camera.setPreviewTexture(null); + } else { + camera.setPreviewDisplay(null); + } + } catch (Exception e) { + Log.e(TAG, "Failed to clear camera preview: " + e); + } + camera.release(); + camera = null; + } + + // Release the reference to any image buffers, since these will no longer be in use. + bytesToByteBuffer.clear(); + } + + /** + * Changes the facing of the camera. + */ + public synchronized void setFacing(int facing) { + if ((facing != CAMERA_FACING_BACK) && (facing != CAMERA_FACING_FRONT)) { + throw new IllegalArgumentException("Invalid camera: " + facing); + } + this.facing = facing; + } + + /** + * Returns the preview size that is currently in use by the underlying camera. + */ + public Size getPreviewSize() { + return previewSize; + } + + /** + * Returns the selected camera; one of {@link #CAMERA_FACING_BACK} or {@link + * #CAMERA_FACING_FRONT}. + */ + public int getCameraFacing() { + return facing; + } + + /** + * Opens the camera and applies the user settings. + * + * @throws IOException if camera cannot be found or preview cannot be processed + */ + @SuppressLint("InlinedApi") + private Camera createCamera(@Nullable OnCameraOpenedCallback callback) throws IOException { + int requestedCameraId = getIdForRequestedCamera(facing); + if (requestedCameraId == -1) { + throw new IOException("Could not find requested camera."); + } + Camera camera = Camera.open(requestedCameraId); + + SizePair sizePair = selectSizePair(camera, requestedPreviewWidth, requestedPreviewHeight); + if (sizePair == null) { + throw new IOException("Could not find suitable preview size."); + } + Size pictureSize = sizePair.pictureSize(); + previewSize = sizePair.previewSize(); + + int[] previewFpsRange = selectPreviewFpsRange(camera, requestedFps); + if (previewFpsRange == null) { + throw new IOException("Could not find suitable preview frames per second range."); + } + + Camera.Parameters parameters = camera.getParameters(); + + if (pictureSize != null) { + parameters.setPictureSize(pictureSize.getWidth(), pictureSize.getHeight()); + } + parameters.setPreviewSize(previewSize.getWidth(), previewSize.getHeight()); + parameters.setPreviewFpsRange( + previewFpsRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], + previewFpsRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); + parameters.setPreviewFormat(ImageFormat.NV21); + + setRotation(camera, parameters, requestedCameraId); + + if (requestedAutoFocus) { + if (parameters + .getSupportedFocusModes() + .contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { + parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); + } else { + Log.i(TAG, "Camera auto focus is not supported on this device."); + } + } + + camera.setParameters(parameters); + + if (callback != null) { + callback.onOpened(textureEntry.id(), previewSize.getWidth(), previewSize.getHeight()); + } + + // Four frame buffers are needed for working with the camera: + // + // one for the frame that is currently being executed upon in doing detection + // one for the next pending frame to process immediately upon completing detection + // two for the frames that the camera uses to populate future preview images + // + // Through trial and error it appears that two free buffers, in addition to the two buffers + // used in this code, are needed for the camera to work properly. Perhaps the camera has + // one thread for acquiring images, and another thread for calling into user code. If only + // three buffers are used, then the camera will spew thousands of warning messages when + // detection takes a non-trivial amount of time. + camera.setPreviewCallbackWithBuffer(new CameraPreviewCallback()); + camera.addCallbackBuffer(createPreviewBuffer(previewSize)); + camera.addCallbackBuffer(createPreviewBuffer(previewSize)); + camera.addCallbackBuffer(createPreviewBuffer(previewSize)); + camera.addCallbackBuffer(createPreviewBuffer(previewSize)); + + return camera; + } + + /** + * Gets the id for the camera specified by the direction it is facing. Returns -1 if no such + * camera was found. + * + * @param facing the desired camera (front-facing or rear-facing) + */ + private static int getIdForRequestedCamera(int facing) { + CameraInfo cameraInfo = new CameraInfo(); + for (int i = 0; i < Camera.getNumberOfCameras(); ++i) { + Camera.getCameraInfo(i, cameraInfo); + if (cameraInfo.facing == facing) { + return i; + } + } + return -1; + } + + public static List> listAvailableCameraDetails() { + List> availableCameraDetails = new ArrayList<>(); + int cameraCount = Camera.getNumberOfCameras(); + for (int i = 0; i < cameraCount; ++i) { + Map detailsMap = new HashMap<>(); + CameraInfo info = new CameraInfo(); + Camera.getCameraInfo(i, info); + detailsMap.put("name", i); + Log.d("ML", "camera Name: " + i); + if (info.facing == CAMERA_FACING_BACK) { + detailsMap.put("lensFacing", "back"); + } else { + detailsMap.put("lensFacing", "front"); + } + availableCameraDetails.add(detailsMap); + } + return availableCameraDetails; + } + + /** + * Selects the most suitable preview and picture size, given the desired width and height. + *

+ *

Even though we only need to find the preview size, it's necessary to find both the preview + * size and the picture size of the camera together, because these need to have the same aspect + * ratio. On some hardware, if you would only set the preview size, you will get a distorted + * image. + * + * @param camera the camera to select a preview size from + * @param desiredWidth the desired width of the camera preview frames + * @param desiredHeight the desired height of the camera preview frames + * @return the selected preview and picture size pair + */ + private static SizePair selectSizePair(Camera camera, int desiredWidth, int desiredHeight) { + List validPreviewSizes = generateValidPreviewSizeList(camera); + + // The method for selecting the best size is to minimize the sum of the differences between + // the desired values and the actual values for width and height. This is certainly not the + // only way to select the best size, but it provides a decent tradeoff between using the + // closest aspect ratio vs. using the closest pixel area. + SizePair selectedPair = null; + int minDiff = Integer.MAX_VALUE; + for (SizePair sizePair : validPreviewSizes) { + Size size = sizePair.previewSize(); + int diff = + Math.abs(size.getWidth() - desiredWidth) + Math.abs(size.getHeight() - desiredHeight); + if (diff < minDiff) { + selectedPair = sizePair; + minDiff = diff; + } + } + + return selectedPair; + } + + /** + * Stores a preview size and a corresponding same-aspect-ratio picture size. To avoid distorted + * preview images on some devices, the picture size must be set to a size that is the same aspect + * ratio as the preview size or the preview may end up being distorted. If the picture size is + * null, then there is no picture size with the same aspect ratio as the preview size. + */ + private static class SizePair { + private final Size preview; + private Size picture; + + SizePair( + android.hardware.Camera.Size previewSize, + @Nullable android.hardware.Camera.Size pictureSize) { + preview = new Size(previewSize.width, previewSize.height); + if (pictureSize != null) { + picture = new Size(pictureSize.width, pictureSize.height); + } + } + + Size previewSize() { + return preview; + } + + @Nullable + Size pictureSize() { + return picture; + } + } + + /** + * Generates a list of acceptable preview sizes. Preview sizes are not acceptable if there is not + * a corresponding picture size of the same aspect ratio. If there is a corresponding picture size + * of the same aspect ratio, the picture size is paired up with the preview size. + *

+ *

This is necessary because even if we don't use still pictures, the still picture size must + * be set to a size that is the same aspect ratio as the preview size we choose. Otherwise, the + * preview images may be distorted on some devices. + */ + private static List generateValidPreviewSizeList(Camera camera) { + Camera.Parameters parameters = camera.getParameters(); + List supportedPreviewSizes = + parameters.getSupportedPreviewSizes(); + List supportedPictureSizes = + parameters.getSupportedPictureSizes(); + List validPreviewSizes = new ArrayList<>(); + for (android.hardware.Camera.Size previewSize : supportedPreviewSizes) { + float previewAspectRatio = (float) previewSize.width / (float) previewSize.height; + + // By looping through the picture sizes in order, we favor the higher resolutions. + // We choose the highest resolution in order to support taking the full resolution + // picture later. + for (android.hardware.Camera.Size pictureSize : supportedPictureSizes) { + float pictureAspectRatio = (float) pictureSize.width / (float) pictureSize.height; + if (Math.abs(previewAspectRatio - pictureAspectRatio) < ASPECT_RATIO_TOLERANCE) { + validPreviewSizes.add(new SizePair(previewSize, pictureSize)); + break; + } + } + } + + // If there are no picture sizes with the same aspect ratio as any preview sizes, allow all + // of the preview sizes and hope that the camera can handle it. Probably unlikely, but we + // still account for it. + if (validPreviewSizes.size() == 0) { + Log.w(TAG, "No preview sizes have a corresponding same-aspect-ratio picture size"); + for (android.hardware.Camera.Size previewSize : supportedPreviewSizes) { + // The null picture size will let us know that we shouldn't set a picture size. + validPreviewSizes.add(new SizePair(previewSize, null)); + } + } + + return validPreviewSizes; + } + + /** + * Selects the most suitable preview frames per second range, given the desired frames per second. + * + * @param camera the camera to select a frames per second range from + * @param desiredPreviewFps the desired frames per second for the camera preview frames + * @return the selected preview frames per second range + */ + @SuppressLint("InlinedApi") + private static int[] selectPreviewFpsRange(Camera camera, float desiredPreviewFps) { + // The camera API uses integers scaled by a factor of 1000 instead of floating-point frame + // rates. + int desiredPreviewFpsScaled = (int) (desiredPreviewFps * 1000.0f); + + // The method for selecting the best range is to minimize the sum of the differences between + // the desired value and the upper and lower bounds of the range. This may select a range + // that the desired value is outside of, but this is often preferred. For example, if the + // desired frame rate is 29.97, the range (30, 30) is probably more desirable than the + // range (15, 30). + int[] selectedFpsRange = null; + int minDiff = Integer.MAX_VALUE; + List previewFpsRangeList = camera.getParameters().getSupportedPreviewFpsRange(); + for (int[] range : previewFpsRangeList) { + int deltaMin = desiredPreviewFpsScaled - range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; + int deltaMax = desiredPreviewFpsScaled - range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; + int diff = Math.abs(deltaMin) + Math.abs(deltaMax); + if (diff < minDiff) { + selectedFpsRange = range; + minDiff = diff; + } + } + return selectedFpsRange; + } + + /** + * Calculates the correct rotation for the given camera id and sets the rotation in the + * parameters. It also sets the camera's display orientation and rotation. + * + * @param parameters the camera parameters for which to set the rotation + * @param cameraId the camera id to set rotation based on + */ + private void setRotation(Camera camera, Camera.Parameters parameters, int cameraId) { + WindowManager windowManager = (WindowManager) activity.getSystemService(Context.WINDOW_SERVICE); + int degrees = 0; + int rotation = windowManager.getDefaultDisplay().getRotation(); + switch (rotation) { + case Surface.ROTATION_0: + degrees = 0; + break; + case Surface.ROTATION_90: + degrees = 90; + break; + case Surface.ROTATION_180: + degrees = 180; + break; + case Surface.ROTATION_270: + degrees = 270; + break; + default: + Log.e(TAG, "Bad rotation value: " + rotation); + } + + CameraInfo cameraInfo = new CameraInfo(); + Camera.getCameraInfo(cameraId, cameraInfo); + + int angle; + int displayAngle; + if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) { + angle = (cameraInfo.orientation + degrees) % 360; + displayAngle = (360 - angle) % 360; // compensate for it being mirrored + } else { // back-facing + angle = (cameraInfo.orientation - degrees + 360) % 360; + displayAngle = angle; + } + + // This corresponds to the rotation constants. + this.rotation = angle / 90; + + camera.setDisplayOrientation(displayAngle); + parameters.setRotation(angle); + } + + /** + * Creates one buffer for the camera preview callback. The size of the buffer is based off of the + * camera preview size and the format of the camera image. + * + * @return a new preview buffer of the appropriate size for the current camera settings + */ + @SuppressLint("InlinedApi") + private byte[] createPreviewBuffer(Size previewSize) { + int bitsPerPixel = ImageFormat.getBitsPerPixel(ImageFormat.NV21); + long sizeInBits = (long) previewSize.getHeight() * previewSize.getWidth() * bitsPerPixel; + int bufferSize = (int) Math.ceil(sizeInBits / 8.0d) + 1; + + // Creating the byte array this way and wrapping it, as opposed to using .allocate(), + // should guarantee that there will be an array to work with. + byte[] byteArray = new byte[bufferSize]; + ByteBuffer buffer = ByteBuffer.wrap(byteArray); + if (!buffer.hasArray() || (buffer.array() != byteArray)) { + // I don't think that this will ever happen. But if it does, then we wouldn't be + // passing the preview content to the underlying detector later. + throw new IllegalStateException("Failed to create valid buffer for camera source."); + } + + bytesToByteBuffer.put(byteArray, buffer); + return byteArray; + } + + // ============================================================================================== + // Frame processing + // ============================================================================================== + + /** + * Called when the camera has a new preview frame. + */ + private class CameraPreviewCallback implements Camera.PreviewCallback { + @Override + public void onPreviewFrame(byte[] data, Camera camera) { + processingRunnable.setNextFrame(data, camera); + } + } + + public void setMachineLearningFrameProcessor(Detector processor) { + synchronized (processorLock) { + if (detector != null) { + detector.close(null); + } + detector = processor; + } + } + + /** + * This runnable controls access to the underlying receiver, calling it to process frames when + * available from the camera. This is designed to run detection on frames as fast as possible + * (i.e., without unnecessary context switching or waiting on the next frame). + *

+ *

While detection is running on a frame, new frames may be received from the camera. As these + * frames come in, the most recent frame is held onto as pending. As soon as detection and its + * associated processing is done for the previous frame, detection on the mostly recently received + * frame will immediately start on the same thread. + */ + private class FrameProcessingRunnable implements Runnable { + + // This lock guards all of the member variables below. + private final Object lock = new Object(); + private boolean active = true; + + // These pending variables hold the state associated with the new frame awaiting processing. + private ByteBuffer pendingFrameData; + + FrameProcessingRunnable() { + } + + /** + * Releases the underlying receiver. This is only safe to do after the associated thread has + * completed, which is managed in camera source's release method above. + */ + @SuppressLint("Assert") + void release() { + assert (processingThread.getState() == State.TERMINATED); + } + + /** + * Marks the runnable as active/not active. Signals any blocked threads to continue. + */ + void setActive(boolean active) { + synchronized (lock) { + this.active = active; + lock.notifyAll(); + } + } + + /** + * Sets the frame data received from the camera. This adds the previous unused frame buffer (if + * present) back to the camera, and keeps a pending reference to the frame data for future use. + */ + void setNextFrame(byte[] data, Camera camera) { + synchronized (lock) { + if (pendingFrameData != null) { + camera.addCallbackBuffer(pendingFrameData.array()); + pendingFrameData = null; + } + + if (!bytesToByteBuffer.containsKey(data)) { + Log.d( + TAG, + "Skipping frame. Could not find ByteBuffer associated with the image " + + "data from the camera."); + return; + } + + pendingFrameData = bytesToByteBuffer.get(data); + + // Notify the processor thread if it is waiting on the next frame (see below). + lock.notifyAll(); + } + } + + /** + * As long as the processing thread is active, this executes detection on frames continuously. + * The next pending frame is either immediately available or hasn't been received yet. Once it + * is available, we transfer the frame info to local variables and run detection on that frame. + * It immediately loops back for the next frame without pausing. + *

+ *

If detection takes longer than the time in between new frames from the camera, this will + * mean that this loop will run without ever waiting on a frame, avoiding any context switching + * or frame acquisition time latency. + *

+ *

If you find that this is using more CPU than you'd like, you should probably decrease the + * FPS setting above to allow for some idle time in between frames. + */ + @SuppressLint("InlinedApi") + @SuppressWarnings("GuardedBy") + @Override + public void run() { + ByteBuffer data; + + while (true) { + synchronized (lock) { + while (active && (pendingFrameData == null)) { + try { + // Wait for the next frame to be received from the camera, since we + // don't have it yet. + lock.wait(); + } catch (InterruptedException e) { + Log.d(TAG, "Frame processing loop terminated.", e); + return; + } + } + + if (!active) { + // Exit the loop once this camera source is stopped or released. We check + // this here, immediately after the wait() above, to handle the case where + // setActive(false) had been called, triggering the termination of this + // loop. + return; + } + + // Hold onto the frame data locally, so that we can use this for detection + // below. We need to clear pendingFrameData to ensure that this buffer isn't + // recycled back to the camera before we are done using that data. + data = pendingFrameData; + pendingFrameData = null; + } + + // The code below needs to run outside of synchronization, because this will allow + // the camera to add pending frame(s) while we are running detection on the current + // frame. + + try { + synchronized (processorLock) { + FirebaseVisionImageMetadata metadata = + new FirebaseVisionImageMetadata.Builder() + .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) + .setWidth(previewSize.getWidth()) + .setHeight(previewSize.getHeight()) + .setRotation(rotation) + .build(); + FirebaseVisionImage image = FirebaseVisionImage.fromByteBuffer(data, metadata); + detector.handleDetection(image, liveDetectorFinishedCallback); + } + } catch (Throwable t) { + Log.e(TAG, "Exception thrown from receiver.", t); + } finally { + camera.addCallbackBuffer(data.array()); + } + } + } + } +} diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index c3b0c64d80ab..266e1b85bf55 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -64,7 +64,7 @@ class LivePreviewState extends State { @override void dispose() { super.dispose(); - _readyLoadState?.controller.dispose(); + _readyLoadState?.controller?.dispose(); } @override diff --git a/packages/firebase_ml_vision/lib/src/live_view.dart b/packages/firebase_ml_vision/lib/src/live_view.dart index 9d95e3a8ddb4..4b78825443dc 100644 --- a/packages/firebase_ml_vision/lib/src/live_view.dart +++ b/packages/firebase_ml_vision/lib/src/live_view.dart @@ -53,7 +53,7 @@ Future> availableCameras() async { } class LiveViewCameraDescription { - final String name; + final int name; final LiveViewCameraLensDirection lensDirection; LiveViewCameraDescription({this.name, this.lensDirection}); From 6054f37a10045ed5fc57957ffbee975349ac9224 Mon Sep 17 00:00:00 2001 From: dustin Date: Thu, 19 Jul 2018 10:50:44 -0700 Subject: [PATCH 17/34] Android: allow detector and resolution to be set --- .../FirebaseMlVisionPlugin.java | 30 +++++--- .../firebasemlvision/live/LegacyCamera.java | 70 +++++++------------ .../example/lib/live_preview.dart | 16 +++-- .../ios/Classes/FirebaseMlVisionPlugin.m | 6 +- .../firebase_ml_vision/ios/Classes/LiveView.m | 8 +++ .../lib/firebase_ml_vision.dart | 8 +-- .../lib/{src => }/live_view.dart | 10 ++- .../lib/src/barcode_detector.dart | 4 ++ .../lib/src/firebase_vision.dart | 8 +-- 9 files changed, 79 insertions(+), 81 deletions(-) rename packages/firebase_ml_vision/lib/{src => }/live_view.dart (97%) diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 88936cb4cb77..c2fca06ca476 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -61,6 +61,7 @@ public void onActivityStarted(Activity activity) { @Override public void onActivityResumed(Activity activity) { + //TODO: handle camera permission requesting // if (camera != null && camera.getRequestingPermission()) { // camera.setRequestingPermission(false); // return; @@ -122,23 +123,16 @@ public void onMethodCall(MethodCall call, final Result result) { result.success(null); break; case "availableCameras": -// try { -// List> cameras = CameraInfo.getAvailableCameras(registrar.activeContext()); -// result.success(cameras); -// } catch (CameraInfoException e) { -// result.error("cameraAccess", e.getMessage(), null); -// } List> cameras = LegacyCamera.listAvailableCameraDetails(); result.success(cameras); break; case "initialize": - Log.d("ML", "initialize"); - int cameraName = call.argument("cameraName"); //TODO: set camera facing + int cameraFacing = call.argument("cameraName"); String resolutionPreset = call.argument("resolutionPreset"); if (camera != null) { camera.stop(); } - camera = new LegacyCamera(registrar); //new Camera(registrar, cameraName, resolutionPreset, result); + camera = new LegacyCamera(registrar, resolutionPreset, cameraFacing); //new Camera(registrar, cameraName, resolutionPreset, result); camera.setMachineLearningFrameProcessor(TextDetector.instance); try { camera.start(new LegacyCamera.OnCameraOpenedCallback() { @@ -168,7 +162,23 @@ public void onFailed(Exception e) { result.success(null); break; } - case "LiveView#setRecognizer": + case "LiveView#setDetector": + if (camera != null) { + String detectorType = call.argument("detectorType"); + Detector detector; + switch (detectorType) { + case "text": + detector = TextDetector.instance; + break; + case "barcode": + detector = BarcodeDetector.instance; + break; + default: + detector = TextDetector.instance; + } + camera.setMachineLearningFrameProcessor(detector); + } + result.success(null); break; case "BarcodeDetector#detectInImage": FirebaseVisionImage image = filePathToVisionImage((String) call.arguments, result); diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java index bb6c76cbbe59..a7f3d1111aba 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java @@ -50,6 +50,9 @@ import io.flutter.plugins.firebasemlvision.TextDetector; import io.flutter.view.FlutterView; +import static android.hardware.Camera.CameraInfo.CAMERA_FACING_BACK; +import static android.hardware.Camera.CameraInfo.CAMERA_FACING_FRONT; + /** * Manages the camera and allows UI updates on top of it (e.g. overlaying extra Graphics or * displaying extra information). This receives preview frames from the camera at a specified rate, @@ -57,11 +60,6 @@ */ @SuppressLint("MissingPermission") public class LegacyCamera { - @SuppressLint("InlinedApi") - public static final int CAMERA_FACING_BACK = CameraInfo.CAMERA_FACING_BACK; - - @SuppressLint("InlinedApi") - public static final int CAMERA_FACING_FRONT = CameraInfo.CAMERA_FACING_FRONT; private static final String TAG = "MIDemoApp:CameraSource"; @@ -70,13 +68,6 @@ public interface OnCameraOpenedCallback { void onFailed(Exception e); } - /** - * The dummy surface texture must be assigned a chosen name. Since we never use an OpenGL context, - * we can choose any ID we want here. The dummy surface texture is not a crazy hack - it is - * actually how the camera team recommends using the camera without a preview. - */ - private static final int DUMMY_TEXTURE_NAME = 100; - /** * If the absolute difference between a preview size aspect ratio and a picture size aspect ratio * is less than this tolerance, they are considered to be the same aspect ratio. @@ -106,8 +97,8 @@ public interface OnCameraOpenedCallback { // These values may be requested by the caller. Due to hardware limitations, we may need to // select close, but not exactly the same values for these. private final float requestedFps = 20.0f; - private final int requestedPreviewWidth = 1280; - private final int requestedPreviewHeight = 960; + private int requestedPreviewWidth = 1280; + private int requestedPreviewHeight = 960; private final boolean requestedAutoFocus = true; // These instances need to be held onto to avoid GC of their underlying resources. Even though @@ -173,13 +164,30 @@ public void error(DetectorException e) { }; - public LegacyCamera(PluginRegistry.Registrar registrar) { + public LegacyCamera(PluginRegistry.Registrar registrar, String resolutionPreset, int cameraFacing) { this.registrar = registrar; this.activity = registrar.activity(); this.textureEntry = registrar.view().createSurfaceTexture(); processingRunnable = new FrameProcessingRunnable(); registerEventChannel(); + + switch (resolutionPreset) { + case "high": + requestedPreviewWidth = 1024; + requestedPreviewHeight = 768; + break; + case "medium": + requestedPreviewWidth = 640; + requestedPreviewHeight = 480; + break; + case "low": + requestedPreviewWidth = 320; + requestedPreviewHeight = 240; + break; + } + + setFacing(cameraFacing); } private void registerEventChannel() { @@ -217,36 +225,10 @@ public void release() { } } -// /** -// * Opens the camera and starts sending preview frames to the underlying detector. The preview -// * frames are not displayed. -// * -// * @throws IOException if the camera's preview texture or display could not be initialized -// */ -// @SuppressLint("MissingPermission") -// @RequiresPermission(Manifest.permission.CAMERA) -// public synchronized LegacyCamera start(OnCameraOpenedCallback callback) throws IOException { -// if (camera != null) { -// return this; -// } -// -// camera = createCamera(callback); -// dummySurfaceTexture = new SurfaceTexture(DUMMY_TEXTURE_NAME); -// camera.setPreviewTexture(dummySurfaceTexture); -// usingSurfaceTexture = true; -// camera.startPreview(); -// -// processingThread = new Thread(processingRunnable); -// processingRunnable.setActive(true); -// processingThread.start(); -// return this; -// } - /** * Opens the camera and starts sending preview frames to the underlying detector. The supplied * surface holder is used for the preview so frames can be displayed to the user. * -// * @param surfaceHolder the surface holder to use for the preview frames * @throws IOException if the supplied surface holder could not be used as the preview display */ @RequiresPermission(Manifest.permission.CAMERA) @@ -332,8 +314,8 @@ public Size getPreviewSize() { } /** - * Returns the selected camera; one of {@link #CAMERA_FACING_BACK} or {@link - * #CAMERA_FACING_FRONT}. + * Returns the selected camera; one of {@link CameraInfo#CAMERA_FACING_BACK} or {@link + * CameraInfo#CAMERA_FACING_FRONT}. */ public int getCameraFacing() { return facing; @@ -624,7 +606,7 @@ private void setRotation(Camera camera, Camera.Parameters parameters, int camera int angle; int displayAngle; - if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) { + if (cameraInfo.facing == CAMERA_FACING_FRONT) { angle = (cameraInfo.orientation + degrees) % 360; displayAngle = (360 - angle) % 360; // compensate for it being mirrored } else { // back-facing diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 266e1b85bf55..2f02792fcab2 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -1,7 +1,7 @@ import 'dart:async'; import 'package:firebase_ml_vision/firebase_ml_vision.dart'; -import 'package:firebase_ml_vision/src/live_view.dart'; +import 'package:firebase_ml_vision/live_view.dart'; import 'package:firebase_ml_vision_example/detector_painters.dart'; import 'package:flutter/material.dart'; @@ -25,12 +25,15 @@ class LivePreviewState extends State { Stream _prepareCameraPreview() async* { if (_readyLoadState != null) { + await setLiveViewDetector(); yield _readyLoadState; } else { yield new LiveViewCameraLoadStateLoading(); final List cameras = await availableCameras(); - final backCamera = cameras.firstWhere((cameraDescription) => - cameraDescription.lensDirection == LiveViewCameraLensDirection.back); + final LiveViewCameraDescription backCamera = cameras.firstWhere( + (LiveViewCameraDescription cameraDescription) => + cameraDescription.lensDirection == + LiveViewCameraLensDirection.back); if (backCamera != null) { yield new LiveViewCameraLoadStateLoaded(backCamera); try { @@ -38,7 +41,7 @@ class LivePreviewState extends State { new LiveViewCameraController( backCamera, LiveViewResolutionPreset.high); await controller.initialize(); - setLiveViewDetector(); + await setLiveViewDetector(); yield new LiveViewCameraLoadStateReady(controller); } on LiveViewCameraException catch (e) { yield new LiveViewCameraLoadStateFailed( @@ -56,9 +59,8 @@ class LivePreviewState extends State { setLiveViewDetector(); } - void setLiveViewDetector() async { - // set the initial recognizer - await FirebaseVision.instance.setLiveViewRecognizer(widget.detector); + Future setLiveViewDetector() async { + return FirebaseVision.instance.setLiveViewRecognizer(widget.detector); } @override diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index be1c277a4404..43fc31518a7c 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -120,11 +120,9 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result [_registry unregisterTexture:textureId]; [_camera close]; result(nil); - } else if ([@"LiveView#setRecognizer" isEqualToString:call.method]) { - NSLog(@"setRecognizer called"); + } else if ([@"LiveView#setDetector" isEqualToString:call.method]) { NSDictionary *argsMap = call.arguments; - NSString *recognizerType = ((NSString *)argsMap[@"recognizerType"]); - NSLog(recognizerType); + NSString *detectorType = ((NSString *)argsMap[@"detectorType"]); if (_camera) { NSLog(@"got a camera, setting the recognizer"); // [_camera setRecognizerType:recognizerType]; diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.m b/packages/firebase_ml_vision/ios/Classes/LiveView.m index c15920551c09..98ab74d0f442 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.m +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.m @@ -154,6 +154,14 @@ - (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleB visionImage.metadata = metadata; CGFloat imageWidth = CVPixelBufferGetWidth(newBuffer); CGFloat imageHeight = CVPixelBufferGetHeight(newBuffer); + switch (_currentDetector) { + case <#constant#>: + <#statements#> + break; + + default: + break; + } [TextDetector handleDetection:visionImage result:_eventSink resultWrapper:^id(id _Nullable result) { _isRecognizing = NO; return @{@"eventType": @"recognized", @"recognitionType": @"text", @"textData": result}; diff --git a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart index 52db1e55ffca..1bc5a0eace88 100644 --- a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart +++ b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart @@ -9,16 +9,12 @@ import 'dart:io'; import 'dart:math'; import 'package:firebase_ml_vision/src/vision_model_utils.dart'; -import 'package:firebase_ml_vision/src/live_view.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; +import 'package:firebase_ml_vision/live_view.dart'; part 'src/barcode_detector.dart'; part 'src/face_detector.dart'; part 'src/firebase_vision.dart'; part 'src/label_detector.dart'; -part 'src/text_detector.dart'; - -const String barcodeValueType = "barcode_value_type"; -const String barcodeDisplayValue = "barcode_display_value"; -const String barcodeRawValue = "barcode_raw_value"; \ No newline at end of file +part 'src/text_detector.dart'; \ No newline at end of file diff --git a/packages/firebase_ml_vision/lib/src/live_view.dart b/packages/firebase_ml_vision/lib/live_view.dart similarity index 97% rename from packages/firebase_ml_vision/lib/src/live_view.dart rename to packages/firebase_ml_vision/lib/live_view.dart index 4b78825443dc..2f880cd4375b 100644 --- a/packages/firebase_ml_vision/lib/src/live_view.dart +++ b/packages/firebase_ml_vision/lib/live_view.dart @@ -5,7 +5,7 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; -enum LiveViewCameraLensDirection { front, back, external } +enum LiveViewCameraLensDirection { front, back } enum LiveViewResolutionPreset { low, medium, high } @@ -28,8 +28,6 @@ LiveViewCameraLensDirection _parseCameraLensDirection(String string) { return LiveViewCameraLensDirection.front; case 'back': return LiveViewCameraLensDirection.back; - case 'external': - return LiveViewCameraLensDirection.external; } throw new ArgumentError('Unknown CameraLensDirection value'); } @@ -268,15 +266,15 @@ class LiveViewCameraController extends ValueNotifier { } switch (map['eventType']) { case 'recognized': - String recognitionType = event['recognitionType']; - if (recognitionType == "barcode") { + String detectionType = event['recognitionType']; + if (detectionType == "barcode") { final List reply = event['barcodeData']; final List barcodes = []; reply.forEach((dynamic barcodeMap) { barcodes.add(new BarcodeContainer(barcodeMap)); }); value = value.copyWith(detectedData: barcodes); - } else if (recognitionType == "text") { + } else if (detectionType == "text") { final List reply = event['textData']; final detectedData = reply.map((dynamic block) { return TextBlock.fromBlockData(block); diff --git a/packages/firebase_ml_vision/lib/src/barcode_detector.dart b/packages/firebase_ml_vision/lib/src/barcode_detector.dart index 30c5d276cc74..8c36f0be3cb9 100644 --- a/packages/firebase_ml_vision/lib/src/barcode_detector.dart +++ b/packages/firebase_ml_vision/lib/src/barcode_detector.dart @@ -4,6 +4,10 @@ part of firebase_ml_vision; +const String barcodeValueType = "barcode_value_type"; +const String barcodeDisplayValue = "barcode_display_value"; +const String barcodeRawValue = "barcode_raw_value"; + class BarcodeDetector extends FirebaseVisionDetector { BarcodeDetector._(BarcodeDetectorOptions options); diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index ff48ec0a04d5..07ab591766b5 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -46,13 +46,13 @@ class FirebaseVision { Future setLiveViewRecognizer(FirebaseVisionDetectorType type) async { print("sending setLiveVieRecognizer to device"); - final String typeMessage = recognizerMessageType(type); + final String typeMessage = detectorMessageType(type); if (typeMessage == null) return; - await FirebaseVision.channel.invokeMethod("LiveView#setRecognizer", - {"recognizerType": typeMessage}); + await FirebaseVision.channel.invokeMethod("LiveView#setDetector", + {"detectorType": typeMessage}); } - String recognizerMessageType(FirebaseVisionDetectorType type) { + String detectorMessageType(FirebaseVisionDetectorType type) { switch (type) { case FirebaseVisionDetectorType.barcode: return "barcode"; From de93bc7f4bb08005435b602e3fb571975a77f359 Mon Sep 17 00:00:00 2001 From: dustin Date: Thu, 19 Jul 2018 19:42:28 -0700 Subject: [PATCH 18/34] Working live detection implementation for Android and iOS there are some defects on both platforms that need to be resolved. Android & iOS: barcode detector boundaries are wrong iOS: MLKit detector doesn't work when preview image is not landscape Android: Live text detection block boundaries are off. iOS: utilizing some iOS 10 APIs, need to find iOS 8 work-arounds --- .../FirebaseMlVisionPlugin.java | 4 +- .../firebasemlvision/live/LegacyCamera.java | 11 ++-- .../ios/Runner.xcodeproj/project.pbxproj | 2 +- .../ios/Classes/BarcodeDetector.m | 35 +++++++------ .../ios/Classes/FaceDetector.m | 19 +++++++ .../ios/Classes/FirebaseMlVisionPlugin.h | 11 ++-- .../ios/Classes/FirebaseMlVisionPlugin.m | 52 ++++++++++++------- .../ios/Classes/LabelDetector.m | 20 +++++++ .../firebase_ml_vision/ios/Classes/LiveView.h | 1 + .../firebase_ml_vision/ios/Classes/LiveView.m | 22 +++----- .../ios/Classes/NSError+FlutterError.h | 12 +++++ .../ios/Classes/NSError+FlutterError.m | 16 ++++++ .../ios/Classes/TextDetector.m | 36 +++++++------ .../firebase_ml_vision/lib/live_view.dart | 12 ++--- 14 files changed, 167 insertions(+), 86 deletions(-) create mode 100644 packages/firebase_ml_vision/ios/Classes/NSError+FlutterError.h create mode 100644 packages/firebase_ml_vision/ios/Classes/NSError+FlutterError.m diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index c2fca06ca476..c39b4c6f4bef 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -127,12 +127,12 @@ public void onMethodCall(MethodCall call, final Result result) { result.success(cameras); break; case "initialize": - int cameraFacing = call.argument("cameraName"); + String cameraName = call.argument("cameraName"); String resolutionPreset = call.argument("resolutionPreset"); if (camera != null) { camera.stop(); } - camera = new LegacyCamera(registrar, resolutionPreset, cameraFacing); //new Camera(registrar, cameraName, resolutionPreset, result); + camera = new LegacyCamera(registrar, resolutionPreset, Integer.parseInt(cameraName)); //new Camera(registrar, cameraName, resolutionPreset, result); camera.setMachineLearningFrameProcessor(TextDetector.instance); try { camera.start(new LegacyCamera.OnCameraOpenedCallback() { diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java index a7f3d1111aba..74b04d8b399b 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java @@ -139,21 +139,18 @@ public interface OnCameraOpenedCallback { @Override public void success(Detector detector, Object data) { Map event = new HashMap<>(); - event.put("eventType", "recognized"); + event.put("eventType", "detection"); String dataType; - String dataLabel; if (detector instanceof BarcodeDetector) { dataType = "barcode"; - dataLabel = "barcodeData"; } else if (detector instanceof TextDetector) { dataType = "text"; - dataLabel = "textData"; } else { // unsupported live detector return; } - event.put("recognitionType", dataType); - event.put(dataLabel, data); + event.put("detectionType", dataType); + event.put("data", data); eventSink.success(event); } @@ -419,7 +416,7 @@ public static List> listAvailableCameraDetails() { Map detailsMap = new HashMap<>(); CameraInfo info = new CameraInfo(); Camera.getCameraInfo(i, info); - detailsMap.put("name", i); + detailsMap.put("name", String.valueOf(i)); Log.d("ML", "camera Name: " + i); if (info.facing == CAMERA_FACING_BACK) { detailsMap.put("lensFacing", "back"); diff --git a/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj b/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj index 0ae5c14c9cd2..11f2fd5defde 100644 --- a/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj +++ b/packages/firebase_ml_vision/example/ios/Runner.xcodeproj/project.pbxproj @@ -248,7 +248,7 @@ ); inputPaths = ( "${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh", - "${PODS_ROOT}/../.symlinks/flutter/ios/Flutter.framework", + "${PODS_ROOT}/../.symlinks/flutter/ios-release/Flutter.framework", ); name = "[CP] Embed Pods Frameworks"; outputPaths = ( diff --git a/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m b/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m index 241f0f4173b5..999462b374c8 100644 --- a/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m @@ -3,7 +3,20 @@ @implementation BarcodeDetector static FIRVisionBarcodeDetector *barcodeDetector; -+ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result resultWrapper:(FlutterResultWrapper)wrapper { ++ (id)sharedInstance { + static BarcodeDetector *sharedInstance = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + sharedInstance = [[self alloc] init]; + }); + return sharedInstance; +} + +- (void)respondToCallback:(OperationFinishedCallback)callback withData:(id _Nullable) data { + callback(data, @"barcode"); +} + +- (void)handleDetection:(FIRVisionImage *)image finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)errorCallback { if (barcodeDetector == nil) { FIRVision *vision = [FIRVision vision]; barcodeDetector = [vision barcodeDetector]; @@ -13,35 +26,27 @@ + (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result res detectInImage:image completion:^(NSArray * _Nullable barcodes, NSError * _Nullable error) { if (error) { - [FLTFirebaseMlVisionPlugin handleError:error result:result]; + [FLTFirebaseMlVisionPlugin handleError:error finishedCallback:errorCallback]; return; } else if (!barcodes) { - result(@[]); + [self respondToCallback:callback withData:@[]]; return; } NSMutableArray *blocks = [NSMutableArray array]; for (FIRVisionBarcode *barcode in barcodes) { - NSDictionary *barcodeData = [BarcodeDetector getBarcodeData:barcode]; + NSDictionary *barcodeData = [self getBarcodeData:barcode]; [blocks addObject:barcodeData]; } - - result(wrapper(blocks)); + [self respondToCallback:callback withData:blocks]; }]; } -+ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { - [BarcodeDetector handleDetection:image result:result resultWrapper:^id(id _Nullable result) { - return result; - }]; -} - -+ (void)close { +- (void)close { barcodeDetector = nil; } - -+ (NSDictionary *)getBarcodeData:(FIRVisionBarcode *)barcode { +- (NSDictionary *)getBarcodeData:(FIRVisionBarcode *)barcode { CGRect frame = barcode.frame; NSString *displayValue = barcode.displayValue == nil ? @"" : barcode.displayValue; NSString *rawValue = barcode.rawValue == nil ? @"" : barcode.rawValue; diff --git a/packages/firebase_ml_vision/ios/Classes/FaceDetector.m b/packages/firebase_ml_vision/ios/Classes/FaceDetector.m index 218b8f1a4f94..731678afc70b 100644 --- a/packages/firebase_ml_vision/ios/Classes/FaceDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/FaceDetector.m @@ -1,4 +1,23 @@ #import "FirebaseMlVisionPlugin.h" @implementation FaceDetector +static FIRVisionFaceDetector *faceDetector; + +- (void)close { + faceDetector = nil; +} + +- (void)handleDetection:(FIRVisionImage *)image finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)error { + +} + ++ (id)sharedInstance { + static FaceDetector *sharedInstance = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + sharedInstance = [[self alloc] init]; + }); + return sharedInstance; +} + @end diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h index 95002c1e411a..e7c1a8917fdf 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h @@ -6,17 +6,18 @@ A callback type to allow the caller to format the detected response before it is sent back to Flutter */ -typedef id (^FlutterResultWrapper)(id _Nullable result); +typedef void (^OperationFinishedCallback)(id _Nullable result, NSString *detectorType); +typedef void (^OperationErrorCallback)(FlutterError *error); @interface FLTFirebaseMlVisionPlugin : NSObject -+ (void)handleError:(NSError *)error result:(FlutterResult)result; ++ (void)handleError:(NSError *)error finishedCallback:(OperationErrorCallback)callback; @end @protocol Detector @required -+ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result resultWrapper:(FlutterResultWrapper)wrapper; -+ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result; -+ (void)close; ++ (id)sharedInstance; +- (void)handleDetection:(FIRVisionImage *)image finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)error; +- (void)close; @optional @end diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index 43fc31518a7c..55fd9672d3cd 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -1,17 +1,6 @@ #import "FirebaseMlVisionPlugin.h" #import "LiveView.h" - -@interface NSError (FlutterError) -@property(readonly, nonatomic) FlutterError *flutterError; -@end - -@implementation NSError (FlutterError) -- (FlutterError *)flutterError { - return [FlutterError errorWithCode:[NSString stringWithFormat:@"Error %d", (int)self.code] - message:self.domain - details:self.localizedDescription]; -} -@end +#import "NSError+FlutterError.h" @interface FLTFirebaseMlVisionPlugin() @property(readonly, nonatomic) NSObject *registry; @@ -20,8 +9,8 @@ @interface FLTFirebaseMlVisionPlugin() @end @implementation FLTFirebaseMlVisionPlugin -+ (void)handleError:(NSError *)error result:(FlutterResult)result { - result([error flutterError]); ++ (void)handleError:(NSError *)error finishedCallback:(OperationErrorCallback)callback { + callback([error flutterError]); } + (void)registerWithRegistrar:(NSObject *)registrar { @@ -87,7 +76,6 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result if (error) { result([error flutterError]); } else { - NSLog(@"initialize called"); if (_camera) { [_camera close]; } @@ -123,30 +111,54 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result } else if ([@"LiveView#setDetector" isEqualToString:call.method]) { NSDictionary *argsMap = call.arguments; NSString *detectorType = ((NSString *)argsMap[@"detectorType"]); + id detector = [FLTFirebaseMlVisionPlugin detectorForDetectorTypeString:detectorType]; if (_camera) { - NSLog(@"got a camera, setting the recognizer"); + NSLog(@"got a camera, setting the detector"); + _camera.currentDetector = detector; // [_camera setRecognizerType:recognizerType]; } result(nil); } else if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { FIRVisionImage *image = [self filePathToVisionImage:call.arguments]; - [BarcodeDetector handleDetection:image result:result]; + [[BarcodeDetector sharedInstance] handleDetection:image finishedCallback:^(id _Nullable r, NSString *detectorType) { + result(r); + } errorCallback:^(FlutterError *e) { + result(e); + }]; } else if ([@"BarcodeDetector#close" isEqualToString:call.method]) { - [BarcodeDetector close]; + [[BarcodeDetector sharedInstance] close]; } else if ([@"FaceDetector#detectInImage" isEqualToString:call.method]) { } else if ([@"FaceDetector#close" isEqualToString:call.method]) { } else if ([@"LabelDetector#detectInImage" isEqualToString:call.method]) { } else if ([@"LabelDetector#close" isEqualToString:call.method]) { } else if ([@"TextDetector#detectInImage" isEqualToString:call.method]) { FIRVisionImage *image = [self filePathToVisionImage:call.arguments]; - [TextDetector handleDetection:image result:result]; + [[TextDetector sharedInstance] handleDetection:image finishedCallback:^(id _Nullable r, NSString *detectorType) { + result(r); + } errorCallback:^(FlutterError *error) { + result(error); + }]; } else if ([@"TextDetector#close" isEqualToString:call.method]) { - [TextDetector close]; + [[TextDetector sharedInstance] close]; } else { result(FlutterMethodNotImplemented); } } ++ (NSObject*)detectorForDetectorTypeString:(NSString *)detectorType { + if ([detectorType isEqualToString:@"text"]) { + return [TextDetector sharedInstance]; + } else if ([detectorType isEqualToString:@"barcode"]) { + return [BarcodeDetector sharedInstance]; + } else if ([detectorType isEqualToString:@"label"]) { + return [LabelDetector sharedInstance]; + } else if ([detectorType isEqualToString:@"face"]) { + return [FaceDetector sharedInstance]; + } else { + return [TextDetector sharedInstance]; + } +} + - (FIRVisionImage *)filePathToVisionImage:(NSString *)path { UIImage *image = [UIImage imageWithContentsOfFile:path]; return [[FIRVisionImage alloc] initWithImage:image]; diff --git a/packages/firebase_ml_vision/ios/Classes/LabelDetector.m b/packages/firebase_ml_vision/ios/Classes/LabelDetector.m index 20b149d6a2b8..6a623eed4df1 100644 --- a/packages/firebase_ml_vision/ios/Classes/LabelDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/LabelDetector.m @@ -1,4 +1,24 @@ #import "FirebaseMlVisionPlugin.h" @implementation LabelDetector +static FIRVisionLabelDetector *labelDetector; + +- (void)close { + labelDetector = nil; +} + +- (void)handleDetection:(FIRVisionImage *)image finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)error { + +} + + ++ (id)sharedInstance { + static LabelDetector *sharedInstance = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + sharedInstance = [[self alloc] init]; + }); + return sharedInstance; +} + @end diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.h b/packages/firebase_ml_vision/ios/Classes/LiveView.h index e9e9de5f4931..11e1e7882fbc 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.h +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.h @@ -26,6 +26,7 @@ AVCaptureAudioDataOutputSampleBufferDelegate, FlutterStreamHandler> @property(strong, nonatomic) AVCaptureAudioDataOutput *audioOutput; @property(assign, nonatomic) BOOL isRecording; @property(assign, nonatomic) BOOL isAudioSetup; +@property (strong, nonatomic) NSObject *currentDetector; - (instancetype)initWithCameraName:(NSString *)cameraName resolutionPreset:(NSString *)resolutionPreset error:(NSError **)error; diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.m b/packages/firebase_ml_vision/ios/Classes/LiveView.m index 98ab74d0f442..3e1ac95ce1d1 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.m +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.m @@ -3,6 +3,7 @@ #import "UIUtilities.h" #import #import +#import "NSError+FlutterError.h" static NSString *const sessionQueueLabel = @"io.flutter.plugins.firebaseml.visiondetector.SessionQueue"; static NSString *const videoDataOutputQueueLabel = @"io.flutter.plugins.firebaseml.visiondetector.VideoDataOutputQueue"; @@ -154,22 +155,13 @@ - (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleB visionImage.metadata = metadata; CGFloat imageWidth = CVPixelBufferGetWidth(newBuffer); CGFloat imageHeight = CVPixelBufferGetHeight(newBuffer); - switch (_currentDetector) { - case <#constant#>: - <#statements#> - break; - - default: - break; - } - [TextDetector handleDetection:visionImage result:_eventSink resultWrapper:^id(id _Nullable result) { - _isRecognizing = NO; - return @{@"eventType": @"recognized", @"recognitionType": @"text", @"textData": result}; + [_currentDetector handleDetection:visionImage finishedCallback:^(id _Nullable result, NSString *detectorType) { + self->_isRecognizing = NO; + self->_eventSink(@{@"eventType": @"detection", @"detectionType": detectorType, @"data": result}); + } errorCallback:^(FlutterError *error) { + self->_isRecognizing = NO; + self->_eventSink(error); }]; - // [BarcodeDetector handleDetection:visionImage result:_eventSink resultWrapper:^id(id _Nullable result) { - // _isRecognizing = NO; - // return @{@"eventType": @"recognized", @"recognitionType": @"barcode", @"barcodeData": result}; - // }]; } CFRetain(newBuffer); CVPixelBufferRef old = _latestPixelBuffer; diff --git a/packages/firebase_ml_vision/ios/Classes/NSError+FlutterError.h b/packages/firebase_ml_vision/ios/Classes/NSError+FlutterError.h new file mode 100644 index 000000000000..1db09ddc1223 --- /dev/null +++ b/packages/firebase_ml_vision/ios/Classes/NSError+FlutterError.h @@ -0,0 +1,12 @@ +// +// NSError+FlutterError.h +// firebase_ml_vision +// +// Created by Dustin Graham on 7/19/18. +// +#import +#import + +@interface NSError (FlutterError) +@property(readonly, nonatomic) FlutterError *flutterError; +@end diff --git a/packages/firebase_ml_vision/ios/Classes/NSError+FlutterError.m b/packages/firebase_ml_vision/ios/Classes/NSError+FlutterError.m new file mode 100644 index 000000000000..92d6a2bb57aa --- /dev/null +++ b/packages/firebase_ml_vision/ios/Classes/NSError+FlutterError.m @@ -0,0 +1,16 @@ +// +// NSError+FlutterError.m +// firebase_ml_vision +// +// Created by Dustin Graham on 7/19/18. +// + +#import "NSError+FlutterError.h" + +@implementation NSError (FlutterError) +- (FlutterError *)flutterError { + return [FlutterError errorWithCode:[NSString stringWithFormat:@"Error %d", (int)self.code] + message:self.domain + details:self.localizedDescription]; +} +@end diff --git a/packages/firebase_ml_vision/ios/Classes/TextDetector.m b/packages/firebase_ml_vision/ios/Classes/TextDetector.m index 526e347f3716..1e9a8bc4a7ec 100644 --- a/packages/firebase_ml_vision/ios/Classes/TextDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/TextDetector.m @@ -3,7 +3,20 @@ @implementation TextDetector static FIRVisionTextDetector *textDetector; -+ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result resultWrapper:(FlutterResultWrapper)wrapper { ++ (id)sharedInstance { + static TextDetector *sharedInstance = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + sharedInstance = [[self alloc] init]; + }); + return sharedInstance; +} + +- (void)respondToCallback:(OperationFinishedCallback)callback withData:(id _Nullable) data { + callback(data, @"text"); +} + +- (void)handleDetection:(FIRVisionImage *)image finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)errorCallback { if (textDetector == nil) { FIRVision *vision = [FIRVision vision]; textDetector = [vision textDetector]; @@ -13,10 +26,10 @@ + (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result res detectInImage:image completion:^(NSArray> *_Nullable features, NSError *_Nullable error) { if (error) { - [FLTFirebaseMlVisionPlugin handleError:error result:result]; + [FLTFirebaseMlVisionPlugin handleError:error finishedCallback:errorCallback]; return; } else if (!features) { - result(@[]); + [self respondToCallback:callback withData:@[]]; return; } @@ -61,22 +74,15 @@ + (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result res [blocks addObject:blockData]; } - - result(wrapper(blocks)); + [self respondToCallback:callback withData:blocks]; }]; } -+ (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { - [TextDetector handleDetection:image result:result resultWrapper:^id(id _Nullable result) { - return result; - }]; -} - -+ (void)close { +- (void)close { textDetector = nil; } -+ (NSDictionary *)getTextData:(CGRect)frame +- (NSDictionary *)getTextData:(CGRect)frame cornerPoints:(NSArray *)cornerPoints text:(NSString *)text { __block NSMutableArray *points = [NSMutableArray array]; @@ -95,7 +101,7 @@ + (NSDictionary *)getTextData:(CGRect)frame }; } -+ (NSMutableArray *)getLineData:(NSArray *)lines { +- (NSMutableArray *)getLineData:(NSArray *)lines { NSMutableArray *lineDataArray = [NSMutableArray array]; for (FIRVisionTextLine *line in lines) { @@ -110,7 +116,7 @@ + (NSMutableArray *)getLineData:(NSArray *)lines { return lineDataArray; } -+ (NSMutableArray *)getElementData:(NSArray *)elements { +- (NSMutableArray *)getElementData:(NSArray *)elements { NSMutableArray *elementDataArray = [NSMutableArray array]; for (FIRVisionTextElement *element in elements) { diff --git a/packages/firebase_ml_vision/lib/live_view.dart b/packages/firebase_ml_vision/lib/live_view.dart index 2f880cd4375b..fc9e67a7dfc6 100644 --- a/packages/firebase_ml_vision/lib/live_view.dart +++ b/packages/firebase_ml_vision/lib/live_view.dart @@ -51,7 +51,7 @@ Future> availableCameras() async { } class LiveViewCameraDescription { - final int name; + final String name; final LiveViewCameraLensDirection lensDirection; LiveViewCameraDescription({this.name, this.lensDirection}); @@ -265,18 +265,18 @@ class LiveViewCameraController extends ValueNotifier { return; } switch (map['eventType']) { - case 'recognized': - String detectionType = event['recognitionType']; + case 'detection': + final String detectionType = event['detectionType']; if (detectionType == "barcode") { - final List reply = event['barcodeData']; + final List reply = event['data']; final List barcodes = []; reply.forEach((dynamic barcodeMap) { barcodes.add(new BarcodeContainer(barcodeMap)); }); value = value.copyWith(detectedData: barcodes); } else if (detectionType == "text") { - final List reply = event['textData']; - final detectedData = reply.map((dynamic block) { + final List reply = event['data']; + final List detectedData = reply.map((dynamic block) { return TextBlock.fromBlockData(block); }).toList(); value = value.copyWith(detectedData: detectedData); From 4c5bf215a7c35e286d27a9c1fda2cf6483c0a1d5 Mon Sep 17 00:00:00 2001 From: dustin Date: Fri, 20 Jul 2018 14:32:49 -0700 Subject: [PATCH 19/34] update Android with latest from upstream. --- .../firebasemlvision/BarcodeDetector.java | 16 +++- .../plugins/firebasemlvision/Detector.java | 5 +- .../firebasemlvision/FaceDetector.java | 15 ++- .../FirebaseMlVisionPlugin.java | 27 +++++- .../firebasemlvision/LabelDetector.java | 17 +++- .../firebasemlvision/TextDetector.java | 16 +++- .../plugins/firebasemlvision/live/Camera.java | 2 +- .../firebasemlvision/live/LegacyCamera.java | 12 ++- .../example/lib/detector_painters.dart | 21 ++++- .../example/lib/live_preview.dart | 12 +-- .../firebase_ml_vision/example/lib/main.dart | 27 +++++- .../firebase_ml_vision/lib/live_view.dart | 91 +++++++++++++------ .../lib/src/barcode_detector.dart | 4 +- .../lib/src/face_detector.dart | 22 +++-- .../lib/src/firebase_vision.dart | 8 +- .../lib/src/label_detector.dart | 4 +- 16 files changed, 219 insertions(+), 80 deletions(-) diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java index c72c785de8e8..9b08ba5f0f39 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java @@ -3,6 +3,8 @@ import android.graphics.Point; import android.graphics.Rect; import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; @@ -15,15 +17,19 @@ import java.util.List; import java.util.Map; -class BarcodeDetector implements Detector { +public class BarcodeDetector extends Detector { public static final BarcodeDetector instance = new BarcodeDetector(); private static FirebaseVisionBarcodeDetector barcodeDetector; private BarcodeDetector() {} @Override - public void handleDetection( - FirebaseVisionImage image, Map options, final MethodChannel.Result result) { + public void close(@Nullable OperationFinishedCallback callback) { + + } + + @Override + void processImage(FirebaseVisionImage image, Map options, final OperationFinishedCallback finishedCallback) { if (barcodeDetector == null) barcodeDetector = FirebaseVision.getInstance().getVisionBarcodeDetector(); @@ -178,14 +184,14 @@ public void onSuccess(List firebaseVisionBarcodes) { barcodes.add(barcodeMap); } - result.success(barcodes); + finishedCallback.success(BarcodeDetector.this, barcodes); } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception exception) { - result.error("barcodeDetectorError", exception.getLocalizedMessage(), null); + finishedCallback.error(new DetectorException("barcodeDetectorError", exception.getLocalizedMessage(), null)); } }); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java index 86687dd7f1b6..6a72847ed887 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java @@ -4,6 +4,7 @@ import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; public abstract class Detector { @@ -25,7 +26,7 @@ public void handleDetection( if (shouldThrottle.get()) { return; } - processImage(image, new OperationFinishedCallback() { + processImage(image, options, new OperationFinishedCallback() { @Override public void success(Detector detector, Object data) { shouldThrottle.set(false); @@ -45,5 +46,5 @@ public void error(DetectorException e) { } abstract void processImage( - FirebaseVisionImage image, OperationFinishedCallback finishedCallback); + FirebaseVisionImage image, Map options, OperationFinishedCallback finishedCallback); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java index 4f416f2f0272..f05035e9e644 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java @@ -1,6 +1,8 @@ package io.flutter.plugins.firebasemlvision; import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; @@ -15,15 +17,18 @@ import java.util.List; import java.util.Map; -class FaceDetector implements Detector { +public class FaceDetector extends Detector { public static final FaceDetector instance = new FaceDetector(); private FaceDetector() {} @Override - public void handleDetection( - FirebaseVisionImage image, Map options, final MethodChannel.Result result) { + public void close(@Nullable OperationFinishedCallback callback) { + // TODO: figure out if we still need to do this + } + @Override + void processImage(FirebaseVisionImage image, Map options, final OperationFinishedCallback finishedCallback) { FirebaseVisionFaceDetector detector; if (options == null) { detector = FirebaseVision.getInstance().getVisionFaceDetector(); @@ -72,14 +77,14 @@ public void onSuccess(List firebaseVisionFaces) { faces.add(faceData); } - result.success(faces); + finishedCallback.success(FaceDetector.this, faces); } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception exception) { - result.error("faceDetectorError", exception.getLocalizedMessage(), null); + finishedCallback.error(new DetectorException("faceDetectorError", exception.getLocalizedMessage(), null)); } }); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 8fae3b3e4ddc..6c6b014ecd6f 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -134,7 +134,7 @@ public void onMethodCall(MethodCall call, final Result result) { camera.stop(); } camera = new LegacyCamera(registrar, resolutionPreset, Integer.parseInt(cameraName)); //new Camera(registrar, cameraName, resolutionPreset, result); - camera.setMachineLearningFrameProcessor(TextDetector.instance); + camera.setMachineLearningFrameProcessor(TextDetector.instance, options); try { camera.start(new LegacyCamera.OnCameraOpenedCallback() { @Override @@ -174,17 +174,20 @@ public void onFailed(Exception e) { case "barcode": detector = BarcodeDetector.instance; break; + case "face": + detector = FaceDetector.instance; + break; default: detector = TextDetector.instance; } - camera.setMachineLearningFrameProcessor(detector); + camera.setMachineLearningFrameProcessor(detector, options); } result.success(null); break; case "BarcodeDetector#detectInImage": try { image = filePathToVisionImage((String) call.argument("path")); - BarcodeDetector.instance.handleDetection(image, options, result); + BarcodeDetector.instance.handleDetection(image, options, handleDetection(result)); } catch (IOException e) { result.error("barcodeDetectorIOError", e.getLocalizedMessage(), null); } catch (Exception e) { @@ -194,7 +197,7 @@ public void onFailed(Exception e) { case "FaceDetector#detectInImage": try { image = filePathToVisionImage((String) call.argument("path")); - FaceDetector.instance.handleDetection(image, options, result); + FaceDetector.instance.handleDetection(image, options, handleDetection(result)); } catch (IOException e) { result.error("faceDetectorIOError", e.getLocalizedMessage(), null); } catch (Exception e) { @@ -206,7 +209,7 @@ public void onFailed(Exception e) { case "TextDetector#detectInImage": try { image = filePathToVisionImage((String) call.argument("path")); - TextDetector.instance.handleDetection(image, options, result); + TextDetector.instance.handleDetection(image, options, handleDetection(result)); } catch (IOException e) { result.error("textDetectorIOError", e.getLocalizedMessage(), null); } catch (Exception e) { @@ -218,6 +221,20 @@ public void onFailed(Exception e) { } } + private Detector.OperationFinishedCallback handleDetection(final Result result) { + return new Detector.OperationFinishedCallback() { + @Override + public void success(Detector detector, Object data) { + result.success(data); + } + + @Override + public void error(DetectorException e) { + e.sendError(result); + } + }; + } + private FirebaseVisionImage filePathToVisionImage(String path) throws IOException { File file = new File(path); return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file)); diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java index 3de27514aea9..502b35b4c37e 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java @@ -1,11 +1,20 @@ package io.flutter.plugins.firebasemlvision; +import android.support.annotation.Nullable; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; import io.flutter.plugin.common.MethodChannel; import java.util.Map; -class LabelDetector implements Detector { +public class LabelDetector extends Detector { + + @Override + public void close(@Nullable OperationFinishedCallback callback) { + + } + @Override - public void handleDetection( - FirebaseVisionImage image, Map options, final MethodChannel.Result result) {} -} + void processImage(FirebaseVisionImage image, Map options, OperationFinishedCallback finishedCallback) { + + } +} \ No newline at end of file diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java index 86ca386bea2d..37a7c02aa939 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java @@ -3,6 +3,8 @@ import android.graphics.Point; import android.graphics.Rect; import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; @@ -15,15 +17,19 @@ import java.util.List; import java.util.Map; -public class TextDetector implements Detector { +public class TextDetector extends Detector { public static final TextDetector instance = new TextDetector(); private static FirebaseVisionTextDetector textDetector; private TextDetector() {} @Override - public void handleDetection( - FirebaseVisionImage image, Map options, final MethodChannel.Result result) { + public void close(@Nullable OperationFinishedCallback callback) { + + } + + @Override + void processImage(FirebaseVisionImage image, Map options, final OperationFinishedCallback finishedCallback) { if (textDetector == null) textDetector = FirebaseVision.getInstance().getVisionTextDetector(); textDetector .detectInImage(image) @@ -59,14 +65,14 @@ public void onSuccess(FirebaseVisionText firebaseVisionText) { blockData.put("lines", lines); blocks.add(blockData); } - result.success(blocks); + finishedCallback.success(TextDetector.this, blocks); } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception exception) { - result.error("textDetectorError", exception.getLocalizedMessage(), null); + finishedCallback.error(new DetectorException("textDetectorError", exception.getLocalizedMessage(), null)); } }); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index 3b5009958228..e4c867fcfadd 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -359,7 +359,7 @@ private void processImage(Image image) { .build(); FirebaseVisionImage firebaseVisionImage = FirebaseVisionImage.fromByteBuffer(imageBuffer, metadata); - currentDetector.handleDetection(firebaseVisionImage, liveDetectorFinishedCallback); + currentDetector.handleDetection(firebaseVisionImage, new HashMap(), liveDetectorFinishedCallback); // FirebaseVisionBarcodeDetector visionBarcodeDetector = FirebaseVision.getInstance().getVisionBarcodeDetector(); // visionBarcodeDetector.detectInImage(firebaseVisionImage).addOnSuccessListener(new OnSuccessListener>() { diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java index 74b04d8b399b..e6ef380d4f26 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java @@ -47,6 +47,8 @@ import io.flutter.plugins.firebasemlvision.BarcodeDetector; import io.flutter.plugins.firebasemlvision.Detector; import io.flutter.plugins.firebasemlvision.DetectorException; +import io.flutter.plugins.firebasemlvision.FaceDetector; +import io.flutter.plugins.firebasemlvision.LabelDetector; import io.flutter.plugins.firebasemlvision.TextDetector; import io.flutter.view.FlutterView; @@ -123,6 +125,7 @@ public interface OnCameraOpenedCallback { private final Object processorLock = new Object(); private Detector detector; + private Map detectorOptions; /** * Map to convert between a byte array, received from the camera, and its associated byte buffer. @@ -145,6 +148,10 @@ public void success(Detector detector, Object data) { dataType = "barcode"; } else if (detector instanceof TextDetector) { dataType = "text"; + } else if (detector instanceof LabelDetector) { + dataType = "label"; + } else if (detector instanceof FaceDetector) { + dataType = "face"; } else { // unsupported live detector return; @@ -658,12 +665,13 @@ public void onPreviewFrame(byte[] data, Camera camera) { } } - public void setMachineLearningFrameProcessor(Detector processor) { + public void setMachineLearningFrameProcessor(Detector processor, @Nullable Map options) { synchronized (processorLock) { if (detector != null) { detector.close(null); } detector = processor; + detectorOptions = options; } } @@ -795,7 +803,7 @@ public void run() { .setRotation(rotation) .build(); FirebaseVisionImage image = FirebaseVisionImage.fromByteBuffer(data, metadata); - detector.handleDetection(image, liveDetectorFinishedCallback); + detector.handleDetection(image, detectorOptions, liveDetectorFinishedCallback); } } catch (Throwable t) { Log.e(TAG, "Exception thrown from receiver.", t); diff --git a/packages/firebase_ml_vision/example/lib/detector_painters.dart b/packages/firebase_ml_vision/example/lib/detector_painters.dart index 327247a16670..5e6268fc2246 100644 --- a/packages/firebase_ml_vision/example/lib/detector_painters.dart +++ b/packages/firebase_ml_vision/example/lib/detector_painters.dart @@ -5,9 +5,28 @@ import 'dart:ui'; import 'package:firebase_ml_vision/firebase_ml_vision.dart'; +import 'package:firebase_ml_vision/live_view.dart'; import 'package:flutter/material.dart'; -enum Detector { barcode, face, label, text } +CustomPaint customPaintForResults( + Size imageSize, LiveViewDetectionList results) { + CustomPainter painter; + if (results is LiveViewBarcodeDetectionList) { + painter = new BarcodeDetectorPainter(imageSize, results.data); + } else if (results is LiveViewTextDetectionList) { + painter = new TextDetectorPainter(imageSize, results.data); + } else if (results is LiveViewFaceDetectionList) { + painter = new FaceDetectorPainter(imageSize, results.data); + } else if (results is LiveViewLabelDetectionList) { + painter = new LabelDetectorPainter(imageSize, results.data); + } else { + painter = null; + } + + return new CustomPaint( + painter: painter, + ); +} class BarcodeDetectorPainter extends CustomPainter { BarcodeDetectorPainter(this.absoluteImageSize, this.barcodeLocations); diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 2f02792fcab2..d5b2ae25d18a 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -22,6 +22,7 @@ class LivePreview extends StatefulWidget { class LivePreviewState extends State { bool _isShowingPreview = false; LiveViewCameraLoadStateReady _readyLoadState; + GlobalKey _liveViewKey = new GlobalKey(); Stream _prepareCameraPreview() async* { if (_readyLoadState != null) { @@ -60,7 +61,7 @@ class LivePreviewState extends State { } Future setLiveViewDetector() async { - return FirebaseVision.instance.setLiveViewRecognizer(widget.detector); + return _readyLoadState?.controller?.setDetector(widget.detector); } @override @@ -76,7 +77,7 @@ class LivePreviewState extends State { initialData: new LiveViewCameraLoadStateLoading(), builder: (BuildContext context, AsyncSnapshot snapshot) { - final loadState = snapshot.data; + final LiveViewCameraLoadState loadState = snapshot.data; if (loadState != null) { if (loadState is LiveViewCameraLoadStateLoading || loadState is LiveViewCameraLoadStateLoaded) { @@ -91,12 +92,11 @@ class LivePreviewState extends State { aspectRatio: _readyLoadState.controller.value.aspectRatio, child: new LiveView( controller: _readyLoadState.controller, - overlayBuilder: - (BuildContext context, Size previewSize, dynamic data) { + overlayBuilder: (BuildContext context, Size previewSize, + LiveViewDetectionList data) { return data == null ? new Container() - : customPaintForResults( - widget.detector, previewSize, data); + : customPaintForResults(previewSize, data); }, ), ); diff --git a/packages/firebase_ml_vision/example/lib/main.dart b/packages/firebase_ml_vision/example/lib/main.dart index ee0d4230ae23..80f7f91e13dd 100644 --- a/packages/firebase_ml_vision/example/lib/main.dart +++ b/packages/firebase_ml_vision/example/lib/main.dart @@ -113,6 +113,31 @@ class _MyHomePageState extends State<_MyHomePage> }); } + CustomPaint _buildResults(Size imageSize, List results) { + CustomPainter painter; + + switch (_currentDetector) { + case FirebaseVisionDetectorType.barcode: + painter = new BarcodeDetectorPainter(_imageSize, results); + break; + case FirebaseVisionDetectorType.face: + painter = new FaceDetectorPainter(_imageSize, results); + break; + case FirebaseVisionDetectorType.label: + painter = new LabelDetectorPainter(_imageSize, results); + break; + case FirebaseVisionDetectorType.text: + painter = new TextDetectorPainter(_imageSize, results); + break; + default: + break; + } + + return new CustomPaint( + painter: painter, + ); + } + Widget _buildImage() { return new Container( constraints: const BoxConstraints.expand(), @@ -132,7 +157,7 @@ class _MyHomePageState extends State<_MyHomePage> ), ), ) - : customPaintForResults(_currentDetector, _imageSize, _scanResults), + : _buildResults(_imageSize, _scanResults), ); } diff --git a/packages/firebase_ml_vision/lib/live_view.dart b/packages/firebase_ml_vision/lib/live_view.dart index fc9e67a7dfc6..f990cdc4dabc 100644 --- a/packages/firebase_ml_vision/lib/live_view.dart +++ b/packages/firebase_ml_vision/lib/live_view.dart @@ -86,7 +86,7 @@ class LiveViewCameraException implements Exception { } typedef Widget OverlayBuilder( - BuildContext context, Size previewImageSize, dynamic data); + BuildContext context, Size previewImageSize, LiveViewDetectionList data); // Build the UI texture view of the video data with textureId. class LiveView extends StatefulWidget { @@ -102,15 +102,11 @@ class LiveView extends StatefulWidget { } class LiveViewState extends State { - List scannedCodes = []; - @override void initState() { super.initState(); widget.controller.addListener(() { - setState(() { - scannedCodes = widget.controller.value.detectedData; - }); + setState(() {}); }); } @@ -123,7 +119,10 @@ class LiveViewState extends State { new Container( constraints: const BoxConstraints.expand(), child: widget.overlayBuilder( - context, widget.controller.value.previewSize, scannedCodes), + context, + widget.controller.value.previewSize, + widget.controller.value.detectedData, + ), ) ], ) @@ -143,16 +142,13 @@ class LiveViewCameraValue { /// Is `null` until [isInitialized] is `true`. final Size previewSize; - final List detectedData; - - final FirebaseVisionDetectorType recognizerType; + final LiveViewDetectionList detectedData; const LiveViewCameraValue({ this.isInitialized, this.errorDescription, this.previewSize, this.detectedData, - this.recognizerType, }); const LiveViewCameraValue.uninitialized() @@ -173,15 +169,13 @@ class LiveViewCameraValue { bool isTakingPicture, String errorDescription, Size previewSize, - List detectedData, - FirebaseVisionDetectorType recognizerType, + LiveViewDetectionList detectedData, }) { return new LiveViewCameraValue( isInitialized: isInitialized ?? this.isInitialized, errorDescription: errorDescription, previewSize: previewSize ?? this.previewSize, detectedData: detectedData ?? this.detectedData, - recognizerType: recognizerType ?? this.recognizerType, ); } @@ -250,10 +244,12 @@ class LiveViewCameraController extends ValueNotifier { return _creatingCompleter.future; } - Future setRecognizer( - FirebaseVisionDetectorType recognizerType) async { - await FirebaseVision.instance.setLiveViewRecognizer(recognizerType); - value = value.copyWith(recognizerType: recognizerType); + Future setDetector(FirebaseVisionDetectorType detectorType, + [Map options]) async { + if (detectorType == FirebaseVisionDetectorType.face && options == null) { + options = new FaceDetectorOptions().optionsMap; + } + await FirebaseVision.instance.setLiveViewDetector(detectorType, options); } /// Listen to events from the native plugins. @@ -267,20 +263,37 @@ class LiveViewCameraController extends ValueNotifier { switch (map['eventType']) { case 'detection': final String detectionType = event['detectionType']; + final List reply = event['data']; + LiveViewDetectionList dataList; if (detectionType == "barcode") { - final List reply = event['data']; - final List barcodes = []; + final List barcodes = []; reply.forEach((dynamic barcodeMap) { - barcodes.add(new BarcodeContainer(barcodeMap)); + barcodes.add(new Barcode(barcodeMap)); }); - value = value.copyWith(detectedData: barcodes); + dataList = new LiveViewBarcodeDetectionList(barcodes); } else if (detectionType == "text") { - final List reply = event['data']; - final List detectedData = reply.map((dynamic block) { - return TextBlock.fromBlockData(block); + final List texts = []; + reply.map((dynamic block) { + texts.add(TextBlock.fromBlockData(block)); }).toList(); - value = value.copyWith(detectedData: detectedData); + dataList = new LiveViewTextDetectionList(texts); + } else if (detectionType == "face") { + final List faces = []; + reply.map((dynamic f) { + faces.add(new Face(f)); + }); + dataList = new LiveViewFaceDetectionList(faces); + } else if (detectionType == "label") { + final List

+ * *

Note: uses IdentityHashMap here instead of HashMap because the behavior of an array's * equals, hashCode and toString methods is both useless and unexpected. IdentityHashMap enforces * identity ('==') check on the keys. */ private final Map bytesToByteBuffer = new IdentityHashMap<>(); - private Detector.OperationFinishedCallback liveDetectorFinishedCallback = new Detector.OperationFinishedCallback() { - @Override - public void success(Detector detector, Object data) { - Map event = new HashMap<>(); - event.put("eventType", "detection"); - String dataType; - if (detector instanceof BarcodeDetector) { - dataType = "barcode"; - } else if (detector instanceof TextDetector) { - dataType = "text"; - } else if (detector instanceof LabelDetector) { - dataType = "label"; - } else if (detector instanceof FaceDetector) { - dataType = "face"; - } else { - // unsupported live detector - return; - } - event.put("detectionType", dataType); - event.put("data", data); - eventSink.success(event); - } - - @Override - public void error(DetectorException e) { - e.sendError(eventSink); - } - }; + private final Detector.OperationFinishedCallback liveDetectorFinishedCallback = + new Detector.OperationFinishedCallback() { + @Override + public void success(Detector detector, Object data) { + Map event = new HashMap<>(); + event.put("eventType", "detection"); + String dataType; + if (detector instanceof BarcodeDetector) { + dataType = "barcode"; + } else if (detector instanceof TextDetector) { + dataType = "text"; + } else if (detector instanceof LabelDetector) { + dataType = "label"; + } else if (detector instanceof FaceDetector) { + dataType = "face"; + } else { + // unsupported live detector + return; + } + event.put("detectionType", dataType); + event.put("data", data); + eventSink.success(event); + } + @Override + public void error(DetectorException e) { + e.sendError(eventSink); + } + }; - public LegacyCamera(PluginRegistry.Registrar registrar, String resolutionPreset, int cameraFacing) { + public LegacyCamera( + PluginRegistry.Registrar registrar, String resolutionPreset, int cameraFacing) { this.registrar = registrar; this.activity = registrar.activity(); this.textureEntry = registrar.view().createSurfaceTexture(); @@ -196,28 +190,27 @@ public LegacyCamera(PluginRegistry.Registrar registrar, String resolutionPreset, private void registerEventChannel() { new EventChannel( - registrar.messenger(), "plugins.flutter.io/firebase_ml_vision/liveViewEvents" + textureEntry.id()) - .setStreamHandler( - new EventChannel.StreamHandler() { - @Override - public void onListen(Object arguments, EventChannel.EventSink eventSink) { - LegacyCamera.this.eventSink = eventSink; - } - - @Override - public void onCancel(Object arguments) { - LegacyCamera.this.eventSink = null; - } - }); + registrar.messenger(), + "plugins.flutter.io/firebase_ml_vision/liveViewEvents" + textureEntry.id()) + .setStreamHandler( + new EventChannel.StreamHandler() { + @Override + public void onListen(Object arguments, EventChannel.EventSink eventSink) { + LegacyCamera.this.eventSink = eventSink; + } + + @Override + public void onCancel(Object arguments) { + LegacyCamera.this.eventSink = null; + } + }); } // ============================================================================================== // Public // ============================================================================================== - /** - * Stops the camera and releases the resources of the camera and underlying detector. - */ + /** Stops the camera and releases the resources of the camera and underlying detector. */ public void release() { synchronized (processorLock) { stop(); @@ -255,10 +248,13 @@ public synchronized LegacyCamera start(OnCameraOpenedCallback callback) throws I /** * Closes the camera and stops sending frames to the underlying frame detector. + * *

- *

This camera source may be restarted again by calling {@link - * #start(OnCameraOpenedCallback)}. + * + *

This camera source may be restarted again by calling {@link #start(OnCameraOpenedCallback)}. + * *

+ * *

Call {@link #release()} instead to completely shut down this camera source and release the * resources of the underlying detector. */ @@ -296,9 +292,7 @@ public synchronized void stop() { bytesToByteBuffer.clear(); } - /** - * Changes the facing of the camera. - */ + /** Changes the facing of the camera. */ public synchronized void setFacing(int facing) { if ((facing != CAMERA_FACING_BACK) && (facing != CAMERA_FACING_FRONT)) { throw new IllegalArgumentException("Invalid camera: " + facing); @@ -306,9 +300,7 @@ public synchronized void setFacing(int facing) { this.facing = facing; } - /** - * Returns the preview size that is currently in use by the underlying camera. - */ + /** Returns the preview size that is currently in use by the underlying camera. */ public Size getPreviewSize() { return previewSize; } @@ -353,20 +345,18 @@ private Camera createCamera(@Nullable OnCameraOpenedCallback callback) throws IO } parameters.setPreviewSize(previewSize.getWidth(), previewSize.getHeight()); parameters.setPreviewFpsRange( - previewFpsRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], - previewFpsRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); + previewFpsRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], + previewFpsRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); parameters.setPreviewFormat(ImageFormat.NV21); setRotation(camera, parameters, requestedCameraId); - if (requestedAutoFocus) { - if (parameters + if (parameters .getSupportedFocusModes() .contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { - parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); - } else { - Log.i(TAG, "Camera auto focus is not supported on this device."); - } + parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); + } else { + Log.i(TAG, "Camera auto focus is not supported on this device."); } camera.setParameters(parameters); @@ -433,14 +423,16 @@ public static List> listAvailableCameraDetails() { /** * Selects the most suitable preview and picture size, given the desired width and height. + * *

+ * *

Even though we only need to find the preview size, it's necessary to find both the preview * size and the picture size of the camera together, because these need to have the same aspect * ratio. On some hardware, if you would only set the preview size, you will get a distorted * image. * - * @param camera the camera to select a preview size from - * @param desiredWidth the desired width of the camera preview frames + * @param camera the camera to select a preview size from + * @param desiredWidth the desired width of the camera preview frames * @param desiredHeight the desired height of the camera preview frames * @return the selected preview and picture size pair */ @@ -456,7 +448,7 @@ private static SizePair selectSizePair(Camera camera, int desiredWidth, int desi for (SizePair sizePair : validPreviewSizes) { Size size = sizePair.previewSize(); int diff = - Math.abs(size.getWidth() - desiredWidth) + Math.abs(size.getHeight() - desiredHeight); + Math.abs(size.getWidth() - desiredWidth) + Math.abs(size.getHeight() - desiredHeight); if (diff < minDiff) { selectedPair = sizePair; minDiff = diff; @@ -477,8 +469,8 @@ private static class SizePair { private Size picture; SizePair( - android.hardware.Camera.Size previewSize, - @Nullable android.hardware.Camera.Size pictureSize) { + android.hardware.Camera.Size previewSize, + @Nullable android.hardware.Camera.Size pictureSize) { preview = new Size(previewSize.width, previewSize.height); if (pictureSize != null) { picture = new Size(pictureSize.width, pictureSize.height); @@ -499,17 +491,17 @@ Size pictureSize() { * Generates a list of acceptable preview sizes. Preview sizes are not acceptable if there is not * a corresponding picture size of the same aspect ratio. If there is a corresponding picture size * of the same aspect ratio, the picture size is paired up with the preview size. + * *

+ * *

This is necessary because even if we don't use still pictures, the still picture size must * be set to a size that is the same aspect ratio as the preview size we choose. Otherwise, the * preview images may be distorted on some devices. */ private static List generateValidPreviewSizeList(Camera camera) { Camera.Parameters parameters = camera.getParameters(); - List supportedPreviewSizes = - parameters.getSupportedPreviewSizes(); - List supportedPictureSizes = - parameters.getSupportedPictureSizes(); + List supportedPreviewSizes = parameters.getSupportedPreviewSizes(); + List supportedPictureSizes = parameters.getSupportedPictureSizes(); List validPreviewSizes = new ArrayList<>(); for (android.hardware.Camera.Size previewSize : supportedPreviewSizes) { float previewAspectRatio = (float) previewSize.width / (float) previewSize.height; @@ -543,7 +535,7 @@ private static List generateValidPreviewSizeList(Camera camera) { /** * Selects the most suitable preview frames per second range, given the desired frames per second. * - * @param camera the camera to select a frames per second range from + * @param camera the camera to select a frames per second range from * @param desiredPreviewFps the desired frames per second for the camera preview frames * @return the selected preview frames per second range */ @@ -578,11 +570,12 @@ private static int[] selectPreviewFpsRange(Camera camera, float desiredPreviewFp * parameters. It also sets the camera's display orientation and rotation. * * @param parameters the camera parameters for which to set the rotation - * @param cameraId the camera id to set rotation based on + * @param cameraId the camera id to set rotation based on */ private void setRotation(Camera camera, Camera.Parameters parameters, int cameraId) { WindowManager windowManager = (WindowManager) activity.getSystemService(Context.WINDOW_SERVICE); int degrees = 0; + assert windowManager != null; int rotation = windowManager.getDefaultDisplay().getRotation(); switch (rotation) { case Surface.ROTATION_0: @@ -651,9 +644,7 @@ private byte[] createPreviewBuffer(Size previewSize) { // Frame processing // ============================================================================================== - /** - * Called when the camera has a new preview frame. - */ + /** Called when the camera has a new preview frame. */ private class CameraPreviewCallback implements Camera.PreviewCallback { @Override public void onPreviewFrame(byte[] data, Camera camera) { @@ -661,7 +652,8 @@ public void onPreviewFrame(byte[] data, Camera camera) { } } - public void setMachineLearningFrameProcessor(Detector processor, @Nullable Map options) { + public void setMachineLearningFrameProcessor( + Detector processor, @Nullable Map options) { synchronized (processorLock) { detector = processor; detectorOptions = options; @@ -672,7 +664,9 @@ public void setMachineLearningFrameProcessor(Detector processor, @Nullable Map + * *

While detection is running on a frame, new frames may be received from the camera. As these * frames come in, the most recent frame is held onto as pending. As soon as detection and its * associated processing is done for the previous frame, detection on the mostly recently received @@ -687,8 +681,7 @@ private class FrameProcessingRunnable implements Runnable { // These pending variables hold the state associated with the new frame awaiting processing. private ByteBuffer pendingFrameData; - FrameProcessingRunnable() { - } + FrameProcessingRunnable() {} /** * Releases the underlying receiver. This is only safe to do after the associated thread has @@ -699,9 +692,7 @@ void release() { assert (processingThread.getState() == State.TERMINATED); } - /** - * Marks the runnable as active/not active. Signals any blocked threads to continue. - */ + /** Marks the runnable as active/not active. Signals any blocked threads to continue. */ void setActive(boolean active) { synchronized (lock) { this.active = active; @@ -722,9 +713,9 @@ void setNextFrame(byte[] data, Camera camera) { if (!bytesToByteBuffer.containsKey(data)) { Log.d( - TAG, - "Skipping frame. Could not find ByteBuffer associated with the image " - + "data from the camera."); + TAG, + "Skipping frame. Could not find ByteBuffer associated with the image " + + "data from the camera."); return; } @@ -740,11 +731,15 @@ void setNextFrame(byte[] data, Camera camera) { * The next pending frame is either immediately available or hasn't been received yet. Once it * is available, we transfer the frame info to local variables and run detection on that frame. * It immediately loops back for the next frame without pausing. + * *

+ * *

If detection takes longer than the time in between new frames from the camera, this will * mean that this loop will run without ever waiting on a frame, avoiding any context switching * or frame acquisition time latency. + * *

+ * *

If you find that this is using more CPU than you'd like, you should probably decrease the * FPS setting above to allow for some idle time in between frames. */ @@ -789,12 +784,12 @@ public void run() { try { synchronized (processorLock) { FirebaseVisionImageMetadata metadata = - new FirebaseVisionImageMetadata.Builder() - .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) - .setWidth(previewSize.getWidth()) - .setHeight(previewSize.getHeight()) - .setRotation(rotation) - .build(); + new FirebaseVisionImageMetadata.Builder() + .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) + .setWidth(previewSize.getWidth()) + .setHeight(previewSize.getHeight()) + .setRotation(rotation) + .build(); FirebaseVisionImage image = FirebaseVisionImage.fromByteBuffer(data, metadata); detector.handleDetection(image, detectorOptions, liveDetectorFinishedCallback); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/util/DetectedItemUtils.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/util/DetectedItemUtils.java deleted file mode 100644 index e0dec74695ca..000000000000 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/util/DetectedItemUtils.java +++ /dev/null @@ -1,20 +0,0 @@ -package io.flutter.plugins.firebasemlvision.util; - -import android.graphics.Rect; - -import java.util.HashMap; -import java.util.Map; -import static io.flutter.plugins.firebasemlvision.constants.VisionBaseConstants.*; - -public class DetectedItemUtils { - - public static Map rectToFlutterMap(Rect boundingBox) { - Map out = new HashMap<>(); - out.put(LEFT, boundingBox.left); - out.put(TOP, boundingBox.top); - out.put(WIDTH, boundingBox.width()); - out.put(HEIGHT, boundingBox.height()); - return out; - } - -} diff --git a/packages/firebase_ml_vision/example/lib/main.dart b/packages/firebase_ml_vision/example/lib/main.dart index 80f7f91e13dd..2d6d043f14f8 100644 --- a/packages/firebase_ml_vision/example/lib/main.dart +++ b/packages/firebase_ml_vision/example/lib/main.dart @@ -174,7 +174,8 @@ class _MyHomePageState extends State<_MyHomePage> if (_imageFile != null) _scanImage(_imageFile); }); }, - itemBuilder: (BuildContext context) => >[ + itemBuilder: (BuildContext context) => + >[ const PopupMenuItem( child: const Text('Detect Barcode'), value: FirebaseVisionDetectorType.barcode, diff --git a/packages/firebase_ml_vision/ios/Classes/FaceDetector.m b/packages/firebase_ml_vision/ios/Classes/FaceDetector.m index 2baa6f403889..cf59b2cdf193 100644 --- a/packages/firebase_ml_vision/ios/Classes/FaceDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/FaceDetector.m @@ -40,35 +40,48 @@ - (void)handleDetection:(FIRVisionImage *)image options:(NSDictionary *)options @"top" : @((int)face.frame.origin.y), @"width" : @((int)face.frame.size.width), @"height" : @((int)face.frame.size.height), - @"headEulerAngleY" : face.hasHeadEulerAngleY ? @(face.headEulerAngleY) - : [NSNull null], - @"headEulerAngleZ" : face.hasHeadEulerAngleZ ? @(face.headEulerAngleZ) - : [NSNull null], + @"headEulerAngleY" : + face.hasHeadEulerAngleY ? @(face.headEulerAngleY) + : [NSNull null], + @"headEulerAngleZ" : + face.hasHeadEulerAngleZ ? @(face.headEulerAngleZ) + : [NSNull null], @"smilingProbability" : smileProb, @"leftEyeOpenProbability" : leftProb, @"rightEyeOpenProbability" : rightProb, - @"trackingId" : face.hasTrackingID ? @(face.trackingID) : [NSNull null], + @"trackingId" : + face.hasTrackingID ? @(face.trackingID) : [NSNull null], @"landmarks" : @{ - @"bottomMouth" : [FaceDetector getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeMouthBottom], - @"leftCheek" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeLeftCheek], - @"leftEar" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeLeftEar], - @"leftEye" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeLeftEye], - @"leftMouth" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeMouthLeft], - @"noseBase" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeNoseBase], - @"rightCheek" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeRightCheek], - @"rightEar" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeRightEar], - @"rightEye" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeRightEye], - @"rightMouth" : - [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeMouthRight], + @"bottomMouth" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeMouthBottom], + @"leftCheek" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeLeftCheek], + @"leftEar" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeLeftEar], + @"leftEye" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeLeftEye], + @"leftMouth" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeMouthLeft], + @"noseBase" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeNoseBase], + @"rightCheek" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeRightCheek], + @"rightEar" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeRightEar], + @"rightEye" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeRightEye], + @"rightMouth" : [FaceDetector + getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeMouthRight], }, }; diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h index 8df60fe58094..7a71e69d260a 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h @@ -10,7 +10,8 @@ typedef void (^OperationFinishedCallback)(id _Nullable result, NSString *detecto typedef void (^OperationErrorCallback)(FlutterError *error); @interface FLTFirebaseMlVisionPlugin : NSObject -+ (void)handleError:(NSError *)error finishedCallback:(OperationErrorCallback)callback; ++ (void)handleError:(NSError *)error + finishedCallback:(OperationErrorCallback)callback; @end @protocol Detector diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.h b/packages/firebase_ml_vision/ios/Classes/LiveView.h index b08b222d0acb..6c375d6806d1 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.h +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.h @@ -35,5 +35,6 @@ AVCaptureAudioDataOutputSampleBufferDelegate, FlutterStreamHandler> - (void)startVideoRecordingAtPath:(NSString *)path result:(FlutterResult)result; - (void)stopVideoRecordingWithResult:(FlutterResult)result; - (void)captureToFile:(NSString *)filename result:(FlutterResult)result; -- (void)setDetector:(NSObject *)detector withOptions:(NSDictionary *)detectorOptions; +- (void)setDetector:(NSObject *)detector + withOptions:(NSDictionary *)detectorOptions; @end diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.m b/packages/firebase_ml_vision/ios/Classes/LiveView.m index c44f0c1f2e8e..17abfbb63a65 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.m +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.m @@ -154,13 +154,20 @@ - (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleB visionImage.metadata = metadata; CGFloat imageWidth = CVPixelBufferGetWidth(newBuffer); CGFloat imageHeight = CVPixelBufferGetHeight(newBuffer); - [_currentDetector handleDetection:visionImage options:_currentDetectorOptions finishedCallback:^(id _Nullable result, NSString *detectorType) { - self->_isRecognizing = NO; - self->_eventSink(@{@"eventType": @"detection", @"detectionType": detectorType, @"data": result}); - } errorCallback:^(FlutterError *error) { - self->_isRecognizing = NO; - self->_eventSink(error); - }]; + [_currentDetector handleDetection:visionImage + options:_currentDetectorOptions + finishedCallback:^(id _Nullable result, NSString *detectorType) { + self->_isRecognizing = NO; + self->_eventSink(@{ + @"eventType" : @"detection", + @"detectionType" : detectorType, + @"data" : result + }); + } + errorCallback:^(FlutterError *error) { + self->_isRecognizing = NO; + self->_eventSink(error); + }]; } CFRetain(newBuffer); CVPixelBufferRef old = _latestPixelBuffer; diff --git a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart index 08f56d393d20..6627bd24c2b7 100644 --- a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart +++ b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart @@ -17,4 +17,4 @@ part 'src/face_detector.dart'; part 'src/firebase_vision.dart'; part 'src/label_detector.dart'; part 'src/text_detector.dart'; -part 'src/live_view.dart'; \ No newline at end of file +part 'src/live_view.dart'; From e688aefb7f650d23a4c31982f889e6608c3c663a Mon Sep 17 00:00:00 2001 From: dustin Date: Fri, 20 Jul 2018 20:45:55 -0600 Subject: [PATCH 24/34] fix barcode test --- packages/firebase_ml_vision/lib/src/firebase_vision.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 61d28db74f96..0e4d9ee39d74 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -17,8 +17,7 @@ class FirebaseVision { @visibleForTesting static final MethodChannel channel = - const MethodChannel('plugins.flutter.io/firebase_ml_vision') - ..invokeMethod('init'); + const MethodChannel('plugins.flutter.io/firebase_ml_vision'); /// Singleton of [FirebaseVision]. /// From 762b04b7adc108051db898d1cae678affe5136d2 Mon Sep 17 00:00:00 2001 From: dustin Date: Fri, 20 Jul 2018 21:33:08 -0600 Subject: [PATCH 25/34] revert accidental camera plugin changes --- .../src/main/java/io/flutter/plugins/camera/CameraPlugin.java | 2 -- packages/camera/example/android/build.gradle | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 6cf38c5f6239..1d4bedefbbf4 100644 --- a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -131,7 +131,6 @@ public static void registerWith(Registrar registrar) { cameraManager = (CameraManager) registrar.activity().getSystemService(Context.CAMERA_SERVICE); - channel.setMethodCallHandler( new CameraPlugin(registrar, registrar.view(), registrar.activity())); } @@ -736,6 +735,5 @@ private void dispose() { close(); textureEntry.release(); } - } } diff --git a/packages/camera/example/android/build.gradle b/packages/camera/example/android/build.gradle index 23d41d039885..d4225c7905bc 100644 --- a/packages/camera/example/android/build.gradle +++ b/packages/camera/example/android/build.gradle @@ -5,7 +5,7 @@ buildscript { } dependencies { - classpath 'com.android.tools.build:gradle:3.1.3' + classpath 'com.android.tools.build:gradle:3.1.2' } } From 8190bffbef5614dad7fa087971eb6c0e7c71bd65 Mon Sep 17 00:00:00 2001 From: dustin Date: Fri, 20 Jul 2018 21:40:41 -0600 Subject: [PATCH 26/34] clang-format iOS files --- .../ios/Classes/BarcodeDetector.m | 38 ++-- .../ios/Classes/FaceDetector.m | 68 +++--- .../ios/Classes/FirebaseMlVisionPlugin.h | 19 +- .../ios/Classes/FirebaseMlVisionPlugin.m | 78 ++++--- .../ios/Classes/LabelDetector.m | 7 +- .../firebase_ml_vision/ios/Classes/LiveView.h | 13 +- .../firebase_ml_vision/ios/Classes/LiveView.m | 195 +++++++++--------- .../ios/Classes/TextDetector.m | 5 +- .../ios/Classes/UIUtilities.h | 3 +- .../ios/Classes/UIUtilities.m | 25 ++- 10 files changed, 238 insertions(+), 213 deletions(-) diff --git a/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m b/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m index 586bfbdd6148..750815dcb590 100644 --- a/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/BarcodeDetector.m @@ -12,28 +12,32 @@ + (id)sharedInstance { return sharedInstance; } -- (void)handleDetection:(FIRVisionImage *)image options:(NSDictionary *)options finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)errorCallback { +- (void)handleDetection:(FIRVisionImage *)image + options:(NSDictionary *)options + finishedCallback:(OperationFinishedCallback)callback + errorCallback:(OperationErrorCallback)errorCallback { if (barcodeDetector == nil) { FIRVision *vision = [FIRVision vision]; barcodeDetector = [vision barcodeDetector]; } NSMutableArray *ret = [NSMutableArray array]; - [barcodeDetector detectInImage:image - completion:^(NSArray *barcodes, NSError *error) { - if (error) { - [FLTFirebaseMlVisionPlugin handleError:error finishedCallback:errorCallback]; - return; - } else if (!barcodes) { - callback(@[], @"barcode"); - return; - } - - // Scanned barcode - for (FIRVisionBarcode *barcode in barcodes) { - [ret addObject:visionBarcodeToDictionary(barcode)]; - } - callback(ret, @"barcode"); - }]; + [barcodeDetector + detectInImage:image + completion:^(NSArray *barcodes, NSError *error) { + if (error) { + [FLTFirebaseMlVisionPlugin handleError:error finishedCallback:errorCallback]; + return; + } else if (!barcodes) { + callback(@[], @"barcode"); + return; + } + + // Scanned barcode + for (FIRVisionBarcode *barcode in barcodes) { + [ret addObject:visionBarcodeToDictionary(barcode)]; + } + callback(ret, @"barcode"); + }]; } NSDictionary *visionBarcodeToDictionary(FIRVisionBarcode *barcode) { diff --git a/packages/firebase_ml_vision/ios/Classes/FaceDetector.m b/packages/firebase_ml_vision/ios/Classes/FaceDetector.m index cf59b2cdf193..5f9c25e2e532 100644 --- a/packages/firebase_ml_vision/ios/Classes/FaceDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/FaceDetector.m @@ -12,7 +12,10 @@ + (id)sharedInstance { return sharedInstance; } -- (void)handleDetection:(FIRVisionImage *)image options:(NSDictionary *)options finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)errorCallback { +- (void)handleDetection:(FIRVisionImage *)image + options:(NSDictionary *)options + finishedCallback:(OperationFinishedCallback)callback + errorCallback:(OperationErrorCallback)errorCallback { FIRVision *vision = [FIRVision vision]; faceDetector = [vision faceDetectorWithOptions:[FaceDetector parseOptions:options]]; @@ -40,48 +43,35 @@ - (void)handleDetection:(FIRVisionImage *)image options:(NSDictionary *)options @"top" : @((int)face.frame.origin.y), @"width" : @((int)face.frame.size.width), @"height" : @((int)face.frame.size.height), - @"headEulerAngleY" : - face.hasHeadEulerAngleY ? @(face.headEulerAngleY) - : [NSNull null], - @"headEulerAngleZ" : - face.hasHeadEulerAngleZ ? @(face.headEulerAngleZ) - : [NSNull null], + @"headEulerAngleY" : face.hasHeadEulerAngleY ? @(face.headEulerAngleY) + : [NSNull null], + @"headEulerAngleZ" : face.hasHeadEulerAngleZ ? @(face.headEulerAngleZ) + : [NSNull null], @"smilingProbability" : smileProb, @"leftEyeOpenProbability" : leftProb, @"rightEyeOpenProbability" : rightProb, - @"trackingId" : - face.hasTrackingID ? @(face.trackingID) : [NSNull null], + @"trackingId" : face.hasTrackingID ? @(face.trackingID) : [NSNull null], @"landmarks" : @{ - @"bottomMouth" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeMouthBottom], - @"leftCheek" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeLeftCheek], - @"leftEar" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeLeftEar], - @"leftEye" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeLeftEye], - @"leftMouth" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeMouthLeft], - @"noseBase" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeNoseBase], - @"rightCheek" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeRightCheek], - @"rightEar" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeRightEar], - @"rightEye" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeRightEye], - @"rightMouth" : [FaceDetector - getLandmarkPosition:face - landmark:FIRFaceLandmarkTypeMouthRight], + @"bottomMouth" : [FaceDetector getLandmarkPosition:face + landmark:FIRFaceLandmarkTypeMouthBottom], + @"leftCheek" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeLeftCheek], + @"leftEar" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeLeftEar], + @"leftEye" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeLeftEye], + @"leftMouth" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeMouthLeft], + @"noseBase" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeNoseBase], + @"rightCheek" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeRightCheek], + @"rightEar" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeRightEar], + @"rightEye" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeRightEye], + @"rightMouth" : + [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeMouthRight], }, }; diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h index 7a71e69d260a..5106fdb16b30 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h @@ -9,29 +9,28 @@ typedef void (^OperationFinishedCallback)(id _Nullable result, NSString *detectorType); typedef void (^OperationErrorCallback)(FlutterError *error); -@interface FLTFirebaseMlVisionPlugin : NSObject -+ (void)handleError:(NSError *)error - finishedCallback:(OperationErrorCallback)callback; +@interface FLTFirebaseMlVisionPlugin : NSObject ++ (void)handleError:(NSError *)error finishedCallback:(OperationErrorCallback)callback; @end @protocol Detector @required + (id)sharedInstance; - (void)handleDetection:(FIRVisionImage *)image - options:(NSDictionary *)options - finishedCallback:(OperationFinishedCallback)callback - errorCallback:(OperationErrorCallback)error; + options:(NSDictionary *)options + finishedCallback:(OperationFinishedCallback)callback + errorCallback:(OperationErrorCallback)error; @optional @end -@interface BarcodeDetector : NSObject +@interface BarcodeDetector : NSObject @end -@interface FaceDetector : NSObject +@interface FaceDetector : NSObject @end -@interface LabelDetector : NSObject +@interface LabelDetector : NSObject @end -@interface TextDetector : NSObject +@interface TextDetector : NSObject @end diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index 52fd09d276fe..7a70a035aa72 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -2,7 +2,7 @@ #import "LiveView.h" #import "NSError+FlutterError.h" -@interface FLTFirebaseMlVisionPlugin() +@interface FLTFirebaseMlVisionPlugin () @property(readonly, nonatomic) NSObject *registry; @property(readonly, nonatomic) NSObject *messenger; @property(readonly, nonatomic) FLTCam *camera; @@ -17,7 +17,9 @@ + (void)registerWithRegistrar:(NSObject *)registrar { FlutterMethodChannel *channel = [FlutterMethodChannel methodChannelWithName:@"plugins.flutter.io/firebase_ml_vision" binaryMessenger:[registrar messenger]]; - FLTFirebaseMlVisionPlugin *instance = [[FLTFirebaseMlVisionPlugin alloc] initWithRegistry:[registrar textures] messenger:[registrar messenger]]; + FLTFirebaseMlVisionPlugin *instance = + [[FLTFirebaseMlVisionPlugin alloc] initWithRegistry:[registrar textures] + messenger:[registrar messenger]]; [registrar addMethodCallDelegate:instance channel:channel]; } @@ -42,12 +44,12 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result result(nil); } else if ([@"availableCameras" isEqualToString:call.method]) { AVCaptureDeviceDiscoverySession *discoverySession = [AVCaptureDeviceDiscoverySession - discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeBuiltInWideAngleCamera ] - mediaType:AVMediaTypeVideo - position:AVCaptureDevicePositionUnspecified]; + discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeBuiltInWideAngleCamera ] + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionUnspecified]; NSArray *devices = discoverySession.devices; NSMutableArray *> *reply = - [[NSMutableArray alloc] initWithCapacity:devices.count]; + [[NSMutableArray alloc] initWithCapacity:devices.count]; for (AVCaptureDevice *device in devices) { NSString *lensFacing; switch ([device position]) { @@ -62,9 +64,9 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result break; } [reply addObject:@{ - @"name" : [device uniqueID], - @"lensFacing" : lensFacing, - }]; + @"name" : [device uniqueID], + @"lensFacing" : lensFacing, + }]; } result(reply); } else if ([@"initialize" isEqualToString:call.method]) { @@ -86,20 +88,21 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result [_registry textureFrameAvailable:textureId]; }; FlutterEventChannel *eventChannel = [FlutterEventChannel - eventChannelWithName:[NSString - stringWithFormat:@"plugins.flutter.io/firebase_ml_vision/liveViewEvents%lld", - textureId] - binaryMessenger:_messenger]; + eventChannelWithName: + [NSString + stringWithFormat:@"plugins.flutter.io/firebase_ml_vision/liveViewEvents%lld", + textureId] + binaryMessenger:_messenger]; [eventChannel setStreamHandler:cam]; cam.eventChannel = eventChannel; cam.onSizeAvailable = ^{ result(@{ - @"textureId" : @(textureId), - @"previewWidth" : @(cam.previewSize.width), - @"previewHeight" : @(cam.previewSize.height), - @"captureWidth" : @(cam.captureSize.width), - @"captureHeight" : @(cam.captureSize.height), - }); + @"textureId" : @(textureId), + @"previewWidth" : @(cam.previewSize.width), + @"previewHeight" : @(cam.previewSize.height), + @"captureWidth" : @(cam.captureSize.width), + @"captureHeight" : @(cam.captureSize.height), + }); }; [cam start]; } @@ -122,32 +125,41 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result // image file detection FIRVisionImage *image = [self filePathToVisionImage:call.arguments[@"path"]]; NSDictionary *options = call.arguments[@"options"]; - if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { - [[BarcodeDetector sharedInstance] handleDetection:image options:options finishedCallback:[self handleSuccess:result] errorCallback:[self handleError:result]]; - } else if ([@"FaceDetector#detectInImage" isEqualToString:call.method]) { - [[FaceDetector sharedInstance] handleDetection:image options:options finishedCallback:[self handleSuccess:result] errorCallback:[self handleError:result]]; - } else if ([@"LabelDetector#detectInImage" isEqualToString:call.method]) { - } else if ([@"TextDetector#detectInImage" isEqualToString:call.method]) { - [[TextDetector sharedInstance] handleDetection:image options:options finishedCallback:[self handleSuccess:result] errorCallback:[self handleError:result]]; - } else { - result(FlutterMethodNotImplemented); - } + if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { + [[BarcodeDetector sharedInstance] handleDetection:image + options:options + finishedCallback:[self handleSuccess:result] + errorCallback:[self handleError:result]]; + } else if ([@"FaceDetector#detectInImage" isEqualToString:call.method]) { + [[FaceDetector sharedInstance] handleDetection:image + options:options + finishedCallback:[self handleSuccess:result] + errorCallback:[self handleError:result]]; + } else if ([@"LabelDetector#detectInImage" isEqualToString:call.method]) { + } else if ([@"TextDetector#detectInImage" isEqualToString:call.method]) { + [[TextDetector sharedInstance] handleDetection:image + options:options + finishedCallback:[self handleSuccess:result] + errorCallback:[self handleError:result]]; + } else { + result(FlutterMethodNotImplemented); + } } } -- (OperationFinishedCallback) handleSuccess:(FlutterResult) result { - return ^(id _Nullable r, NSString *detectorType) { +- (OperationFinishedCallback)handleSuccess:(FlutterResult)result { + return ^(id _Nullable r, NSString *detectorType) { result(r); }; } -- (OperationErrorCallback) handleError:(FlutterResult) result { +- (OperationErrorCallback)handleError:(FlutterResult)result { return ^(FlutterError *error) { result(error); }; } -+ (NSObject*)detectorForDetectorTypeString:(NSString *)detectorType { ++ (NSObject *)detectorForDetectorTypeString:(NSString *)detectorType { if ([detectorType isEqualToString:@"text"]) { return [TextDetector sharedInstance]; } else if ([detectorType isEqualToString:@"barcode"]) { diff --git a/packages/firebase_ml_vision/ios/Classes/LabelDetector.m b/packages/firebase_ml_vision/ios/Classes/LabelDetector.m index 8345722f4c20..c214ce47b15d 100644 --- a/packages/firebase_ml_vision/ios/Classes/LabelDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/LabelDetector.m @@ -12,9 +12,10 @@ + (id)sharedInstance { return sharedInstance; } -- (void)handleDetection:(FIRVisionImage *)image options:(NSDictionary *)options finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)errorCallback { - +- (void)handleDetection:(FIRVisionImage *)image + options:(NSDictionary *)options + finishedCallback:(OperationFinishedCallback)callback + errorCallback:(OperationErrorCallback)errorCallback { } - @end diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.h b/packages/firebase_ml_vision/ios/Classes/LiveView.h index 6c375d6806d1..73bcb0153f73 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.h +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.h @@ -1,11 +1,13 @@ -#import #import +#import #import -@interface FLTCam : NSObject +@interface FLTCam : NSObject @property(readonly, nonatomic) int64_t textureId; -@property (nonatomic) bool isUsingFrontCamera; +@property(nonatomic) bool isUsingFrontCamera; @property(nonatomic, copy) void (^onFrameAvailable)(); @property(nonatomic, copy) void (^onSizeAvailable)(); @property(nonatomic) FlutterEventChannel *eventChannel; @@ -35,6 +37,5 @@ AVCaptureAudioDataOutputSampleBufferDelegate, FlutterStreamHandler> - (void)startVideoRecordingAtPath:(NSString *)path result:(FlutterResult)result; - (void)stopVideoRecordingWithResult:(FlutterResult)result; - (void)captureToFile:(NSString *)filename result:(FlutterResult)result; -- (void)setDetector:(NSObject *)detector - withOptions:(NSDictionary *)detectorOptions; +- (void)setDetector:(NSObject *)detector withOptions:(NSDictionary *)detectorOptions; @end diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.m b/packages/firebase_ml_vision/ios/Classes/LiveView.m index 17abfbb63a65..7827557ddf9b 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.m +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.m @@ -1,18 +1,20 @@ -#import "FirebaseMlVisionPlugin.h" #import "LiveView.h" -#import "UIUtilities.h" #import #import +#import "FirebaseMlVisionPlugin.h" #import "NSError+FlutterError.h" +#import "UIUtilities.h" -static NSString *const sessionQueueLabel = @"io.flutter.plugins.firebaseml.visiondetector.SessionQueue"; -static NSString *const videoDataOutputQueueLabel = @"io.flutter.plugins.firebaseml.visiondetector.VideoDataOutputQueue"; +static NSString *const sessionQueueLabel = + @"io.flutter.plugins.firebaseml.visiondetector.SessionQueue"; +static NSString *const videoDataOutputQueueLabel = + @"io.flutter.plugins.firebaseml.visiondetector.VideoDataOutputQueue"; @interface FLTCam () -@property (assign, atomic) BOOL isRecognizing; -@property (nonatomic) dispatch_queue_t sessionQueue; -@property (strong, nonatomic) NSObject *currentDetector; -@property (strong, nonatomic) NSDictionary *currentDetectorOptions; +@property(assign, atomic) BOOL isRecognizing; +@property(nonatomic) dispatch_queue_t sessionQueue; +@property(strong, nonatomic) NSObject *currentDetector; +@property(strong, nonatomic) NSDictionary *currentDetectorOptions; @end @implementation FLTCam @@ -21,17 +23,18 @@ - (instancetype)initWithCameraName:(NSString *)cameraName error:(NSError **)error { self = [super init]; NSAssert(self, @"super init cannot be nil"); - + // Configure Captgure Session - + _isUsingFrontCamera = NO; _captureSession = [[AVCaptureSession alloc] init]; _sessionQueue = dispatch_queue_create(sessionQueueLabel.UTF8String, nil); - - // base example uses AVCaptureVideoPreviewLayer here and the layer is added to a view, Flutter Texture works differently here + + // base example uses AVCaptureVideoPreviewLayer here and the layer is added to a view, Flutter + // Texture works differently here [self setUpCaptureSessionOutputWithResolutionPreset:resolutionPreset]; [self setUpCaptureSessionInputWithCameraName:cameraName]; - + return self; } @@ -40,15 +43,14 @@ - (void)setDetector:(NSObject *)detector withOptions:(NSDictionary *)d _currentDetectorOptions = detectorOptions; } -- (AVCaptureSessionPreset) resolutionPresetForPreference:(NSString *)preference { +- (AVCaptureSessionPreset)resolutionPresetForPreference:(NSString *)preference { AVCaptureSessionPreset preset; if ([preference isEqualToString:@"high"]) { preset = AVCaptureSessionPresetHigh; } else if ([preference isEqualToString:@"medium"]) { preset = AVCaptureSessionPresetMedium; } else { - NSAssert([preference isEqualToString:@"low"], @"Unknown resolution preset %@", - preference); + NSAssert([preference isEqualToString:@"low"], @"Unknown resolution preset %@", preference); preset = AVCaptureSessionPresetLow; } return preset; @@ -58,9 +60,12 @@ - (void)setUpCaptureSessionOutputWithResolutionPreset:(NSString *)resolutionPres dispatch_async(_sessionQueue, ^{ [self->_captureSession beginConfiguration]; self->_captureSession.sessionPreset = [self resolutionPresetForPreference:resolutionPreset]; - + _captureVideoOutput = [[AVCaptureVideoDataOutput alloc] init]; - _captureVideoOutput.videoSettings = @{(id)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA]}; + _captureVideoOutput.videoSettings = @{ + (id) + kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA] + }; dispatch_queue_t outputQueue = dispatch_queue_create(videoDataOutputQueueLabel.UTF8String, nil); [_captureVideoOutput setSampleBufferDelegate:self queue:outputQueue]; if ([self.captureSession canAddOutput:_captureVideoOutput]) { @@ -76,7 +81,7 @@ - (void)setUpCaptureSessionInputWithCameraName:(NSString *)cameraName { dispatch_async(_sessionQueue, ^{ AVCaptureDevice *device = [AVCaptureDevice deviceWithUniqueID:cameraName]; CMVideoDimensions dimensions = - CMVideoFormatDescriptionGetDimensions([[device activeFormat] formatDescription]); + CMVideoFormatDescriptionGetDimensions([[device activeFormat] formatDescription]); _previewSize = CGSizeMake(dimensions.width, dimensions.height); if (_onSizeAvailable) { _onSizeAvailable(); @@ -95,19 +100,19 @@ - (void)setUpCaptureSessionInputWithCameraName:(NSString *)cameraName { } else { // TODO? ACaptureConnection? AVCaptureConnection *connection = - [AVCaptureConnection connectionWithInputPorts:_captureVideoInput.ports - output:_captureVideoOutput]; + [AVCaptureConnection connectionWithInputPorts:_captureVideoInput.ports + output:_captureVideoOutput]; if ([_captureDevice position] == AVCaptureDevicePositionFront) { connection.videoMirrored = YES; } -// connection.videoOrientation = AVCaptureVideoOrientationPortrait; + // connection.videoOrientation = AVCaptureVideoOrientationPortrait; [_captureSession addInputWithNoConnections:_captureVideoInput]; [_captureSession addConnection:connection]; -// if ([self.captureSession canAddInput:_captureVideoInput]) { -// [self.captureSession addInput:_captureVideoInput]; -// } else { -// NSLog(@"%@", @"Failed to add capture session input."); -// } + // if ([self.captureSession canAddInput:_captureVideoInput]) { + // [self.captureSession addInput:_captureVideoInput]; + // } else { + // NSLog(@"%@", @"Failed to add capture session input."); + // } } } else { NSLog(@"Failed to get capture device for camera position: %ld", cameraName); @@ -128,10 +133,10 @@ - (void)stop { } - (void)captureToFile:(NSString *)path result:(FlutterResult)result { -// AVCapturePhotoSettings *settings = [AVCapturePhotoSettings photoSettings]; -// [_capturePhotoOutput -// capturePhotoWithSettings:settings -// delegate:[[FLTSavePhotoDelegate alloc] initWithPath:path result:result]]; + // AVCapturePhotoSettings *settings = [AVCapturePhotoSettings photoSettings]; + // [_capturePhotoOutput + // capturePhotoWithSettings:settings + // delegate:[[FLTSavePhotoDelegate alloc] initWithPath:path result:result]]; } - (void)captureOutput:(AVCaptureOutput *)output @@ -140,16 +145,21 @@ - (void)captureOutput:(AVCaptureOutput *)output NSLog(@"Got Here!!!!"); } -- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { +- (void)captureOutput:(AVCaptureOutput *)output + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { CVImageBufferRef newBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); if (newBuffer) { if (!_isRecognizing) { _isRecognizing = YES; FIRVisionImage *visionImage = [[FIRVisionImage alloc] initWithBuffer:sampleBuffer]; FIRVisionImageMetadata *metadata = [[FIRVisionImageMetadata alloc] init]; - UIImageOrientation orientation = [UIUtilities imageOrientationFromDevicePosition:_isUsingFrontCamera ? AVCaptureDevicePositionFront : AVCaptureDevicePositionBack]; - FIRVisionDetectorImageOrientation visionOrientation = [UIUtilities visionImageOrientationFromImageOrientation:orientation]; - + UIImageOrientation orientation = [UIUtilities + imageOrientationFromDevicePosition:_isUsingFrontCamera ? AVCaptureDevicePositionFront + : AVCaptureDevicePositionBack]; + FIRVisionDetectorImageOrientation visionOrientation = + [UIUtilities visionImageOrientationFromImageOrientation:orientation]; + metadata.orientation = visionOrientation; visionImage.metadata = metadata; CGFloat imageWidth = CVPixelBufferGetWidth(newBuffer); @@ -158,11 +168,8 @@ - (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleB options:_currentDetectorOptions finishedCallback:^(id _Nullable result, NSString *detectorType) { self->_isRecognizing = NO; - self->_eventSink(@{ - @"eventType" : @"detection", - @"detectionType" : detectorType, - @"data" : result - }); + self->_eventSink( + @{@"eventType" : @"detection", @"detectionType" : detectorType, @"data" : result}); } errorCallback:^(FlutterError *error) { self->_isRecognizing = NO; @@ -181,27 +188,27 @@ - (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleB _onFrameAvailable(); } } -// switch (_currentDetector) { -// case DetectorOnDeviceFace: -// [self detectFacesOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; -// break; -// case DetectorOnDeviceText: -// [self detectTextOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; -// break; -// } + // switch (_currentDetector) { + // case DetectorOnDeviceFace: + // [self detectFacesOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; + // break; + // case DetectorOnDeviceText: + // [self detectTextOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; + // break; + // } if (!CMSampleBufferDataIsReady(sampleBuffer)) { _eventSink(@{ - @"event" : @"error", - @"errorDescription" : @"sample buffer is not ready. Skipping sample" - }); + @"event" : @"error", + @"errorDescription" : @"sample buffer is not ready. Skipping sample" + }); return; } if (_isRecording) { if (_videoWriter.status == AVAssetWriterStatusFailed) { _eventSink(@{ - @"event" : @"error", - @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] - }); + @"event" : @"error", + @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] + }); return; } CMTime lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); @@ -221,19 +228,19 @@ - (void)newVideoSample:(CMSampleBufferRef)sampleBuffer { if (_videoWriter.status != AVAssetWriterStatusWriting) { if (_videoWriter.status == AVAssetWriterStatusFailed) { _eventSink(@{ - @"event" : @"error", - @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] - }); + @"event" : @"error", + @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] + }); } return; } if (_videoWriterInput.readyForMoreMediaData) { if (![_videoWriterInput appendSampleBuffer:sampleBuffer]) { _eventSink(@{ - @"event" : @"error", - @"errorDescription" : - [NSString stringWithFormat:@"%@", @"Unable to write to video input"] - }); + @"event" : @"error", + @"errorDescription" : + [NSString stringWithFormat:@"%@", @"Unable to write to video input"] + }); } } } @@ -242,19 +249,19 @@ - (void)newAudioSample:(CMSampleBufferRef)sampleBuffer { if (_videoWriter.status != AVAssetWriterStatusWriting) { if (_videoWriter.status == AVAssetWriterStatusFailed) { _eventSink(@{ - @"event" : @"error", - @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] - }); + @"event" : @"error", + @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] + }); } return; } if (_audioWriterInput.readyForMoreMediaData) { if (![_audioWriterInput appendSampleBuffer:sampleBuffer]) { _eventSink(@{ - @"event" : @"error", - @"errorDescription" : - [NSString stringWithFormat:@"%@", @"Unable to write to audio input"] - }); + @"event" : @"error", + @"errorDescription" : + [NSString stringWithFormat:@"%@", @"Unable to write to audio input"] + }); } } } @@ -317,18 +324,18 @@ - (void)stopVideoRecordingWithResult:(FlutterResult)result { result(nil); } else { self->_eventSink(@{ - @"event" : @"error", - @"errorDescription" : @"AVAssetWriter could not finish writing!" - }); + @"event" : @"error", + @"errorDescription" : @"AVAssetWriter could not finish writing!" + }); } }]; } } else { -// NSError *error = -// [NSError errorWithDomain:NSCocoaErrorDomain -// code:NSURLErrorResourceUnavailable -// userInfo:@{NSLocalizedDescriptionKey : @"Video is not recording!"}]; -// result([error flutterError]); + // NSError *error = + // [NSError errorWithDomain:NSCocoaErrorDomain + // code:NSURLErrorResourceUnavailable + // userInfo:@{NSLocalizedDescriptionKey : @"Video is not recording!"}]; + // result([error flutterError]); } } @@ -344,22 +351,22 @@ - (BOOL)setupWriterForPath:(NSString *)path { [self setUpCaptureSessionForAudio]; } _videoWriter = - [[AVAssetWriter alloc] initWithURL:outputURL fileType:AVFileTypeQuickTimeMovie error:&error]; + [[AVAssetWriter alloc] initWithURL:outputURL fileType:AVFileTypeQuickTimeMovie error:&error]; NSParameterAssert(_videoWriter); if (error) { _eventSink(@{@"event" : @"error", @"errorDescription" : error.description}); return NO; } NSDictionary *videoSettings = [NSDictionary - dictionaryWithObjectsAndKeys:AVVideoCodecH264, AVVideoCodecKey, - [NSNumber numberWithInt:_previewSize.height], AVVideoWidthKey, - [NSNumber numberWithInt:_previewSize.width], AVVideoHeightKey, - nil]; + dictionaryWithObjectsAndKeys:AVVideoCodecH264, AVVideoCodecKey, + [NSNumber numberWithInt:_previewSize.height], AVVideoWidthKey, + [NSNumber numberWithInt:_previewSize.width], AVVideoHeightKey, + nil]; _videoWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:videoSettings]; NSParameterAssert(_videoWriterInput); _videoWriterInput.expectsMediaDataInRealTime = YES; - + // Add the audio input AudioChannelLayout acl; bzero(&acl, sizeof(acl)); @@ -367,11 +374,11 @@ - (BOOL)setupWriterForPath:(NSString *)path { NSDictionary *audioOutputSettings = nil; // Both type of audio inputs causes output video file to be corrupted. audioOutputSettings = [NSDictionary - dictionaryWithObjectsAndKeys:[NSNumber numberWithInt:kAudioFormatMPEG4AAC], AVFormatIDKey, - [NSNumber numberWithFloat:44100.0], AVSampleRateKey, - [NSNumber numberWithInt:1], AVNumberOfChannelsKey, - [NSData dataWithBytes:&acl length:sizeof(acl)], - AVChannelLayoutKey, nil]; + dictionaryWithObjectsAndKeys:[NSNumber numberWithInt:kAudioFormatMPEG4AAC], AVFormatIDKey, + [NSNumber numberWithFloat:44100.0], AVSampleRateKey, + [NSNumber numberWithInt:1], AVNumberOfChannelsKey, + [NSData dataWithBytes:&acl length:sizeof(acl)], + AVChannelLayoutKey, nil]; _audioWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings]; _audioWriterInput.expectsMediaDataInRealTime = YES; @@ -380,7 +387,7 @@ - (BOOL)setupWriterForPath:(NSString *)path { dispatch_queue_t queue = dispatch_queue_create("MyQueue", NULL); [_captureVideoOutput setSampleBufferDelegate:self queue:queue]; [_audioOutput setSampleBufferDelegate:self queue:queue]; - + return YES; } - (void)setUpCaptureSessionForAudio { @@ -389,24 +396,24 @@ - (void)setUpCaptureSessionForAudio { // Setup the audio input. AVCaptureDevice *audioDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; AVCaptureDeviceInput *audioInput = - [AVCaptureDeviceInput deviceInputWithDevice:audioDevice error:&error]; + [AVCaptureDeviceInput deviceInputWithDevice:audioDevice error:&error]; if (error) { _eventSink(@{@"event" : @"error", @"errorDescription" : error.description}); } // Setup the audio output. _audioOutput = [[AVCaptureAudioDataOutput alloc] init]; - + if ([_captureSession canAddInput:audioInput]) { [_captureSession addInput:audioInput]; - + if ([_captureSession canAddOutput:_audioOutput]) { [_captureSession addOutput:_audioOutput]; _isAudioSetup = YES; } else { _eventSink(@{ - @"event" : @"error", - @"errorDescription" : @"Unable to add Audio input/output to session capture" - }); + @"event" : @"error", + @"errorDescription" : @"Unable to add Audio input/output to session capture" + }); _isAudioSetup = NO; } } diff --git a/packages/firebase_ml_vision/ios/Classes/TextDetector.m b/packages/firebase_ml_vision/ios/Classes/TextDetector.m index 59ef4f8d2eca..99a7e9003a23 100644 --- a/packages/firebase_ml_vision/ios/Classes/TextDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/TextDetector.m @@ -12,7 +12,10 @@ + (id)sharedInstance { return sharedInstance; } -- (void)handleDetection:(FIRVisionImage *)image options:(NSDictionary *)options finishedCallback:(OperationFinishedCallback)callback errorCallback:(OperationErrorCallback)errorCallback { +- (void)handleDetection:(FIRVisionImage *)image + options:(NSDictionary *)options + finishedCallback:(OperationFinishedCallback)callback + errorCallback:(OperationErrorCallback)errorCallback { if (textDetector == nil) { FIRVision *vision = [FIRVision vision]; textDetector = [vision textDetector]; diff --git a/packages/firebase_ml_vision/ios/Classes/UIUtilities.h b/packages/firebase_ml_vision/ios/Classes/UIUtilities.h index e8e296fe0dcb..94e32d2ca4ff 100644 --- a/packages/firebase_ml_vision/ios/Classes/UIUtilities.h +++ b/packages/firebase_ml_vision/ios/Classes/UIUtilities.h @@ -22,7 +22,8 @@ @interface UIUtilities : NSObject + (void)imageOrientation; + (UIImageOrientation)imageOrientationFromDevicePosition:(AVCaptureDevicePosition)devicePosition; -+ (FIRVisionDetectorImageOrientation)visionImageOrientationFromImageOrientation:(UIImageOrientation)imageOrientation; ++ (FIRVisionDetectorImageOrientation)visionImageOrientationFromImageOrientation: + (UIImageOrientation)imageOrientation; + (UIDeviceOrientation)currentUIOrientation; @end diff --git a/packages/firebase_ml_vision/ios/Classes/UIUtilities.m b/packages/firebase_ml_vision/ios/Classes/UIUtilities.m index f5f49c8825c8..4bca947f5a95 100644 --- a/packages/firebase_ml_vision/ios/Classes/UIUtilities.m +++ b/packages/firebase_ml_vision/ios/Classes/UIUtilities.m @@ -27,20 +27,26 @@ + (void)imageOrientation { [self imageOrientationFromDevicePosition:AVCaptureDevicePositionBack]; } -+ (UIImageOrientation) imageOrientationFromDevicePosition:(AVCaptureDevicePosition)devicePosition { ++ (UIImageOrientation)imageOrientationFromDevicePosition:(AVCaptureDevicePosition)devicePosition { UIDeviceOrientation deviceOrientation = UIDevice.currentDevice.orientation; - if (deviceOrientation == UIDeviceOrientationFaceDown || deviceOrientation == UIDeviceOrientationFaceUp || deviceOrientation == UIDeviceOrientationUnknown) { + if (deviceOrientation == UIDeviceOrientationFaceDown || + deviceOrientation == UIDeviceOrientationFaceUp || + deviceOrientation == UIDeviceOrientationUnknown) { deviceOrientation = [self currentUIOrientation]; } switch (deviceOrientation) { case UIDeviceOrientationPortrait: - return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationLeftMirrored : UIImageOrientationRight; + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationLeftMirrored + : UIImageOrientationRight; case UIDeviceOrientationLandscapeLeft: - return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationDownMirrored : UIImageOrientationUp; + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationDownMirrored + : UIImageOrientationUp; case UIDeviceOrientationPortraitUpsideDown: - return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationRightMirrored : UIImageOrientationLeft; + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationRightMirrored + : UIImageOrientationLeft; case UIDeviceOrientationLandscapeRight: - return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationUpMirrored : UIImageOrientationDown; + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationUpMirrored + : UIImageOrientationDown; case UIDeviceOrientationFaceDown: case UIDeviceOrientationFaceUp: case UIDeviceOrientationUnknown: @@ -48,7 +54,8 @@ + (UIImageOrientation) imageOrientationFromDevicePosition:(AVCaptureDevicePositi } } -+ (FIRVisionDetectorImageOrientation) visionImageOrientationFromImageOrientation:(UIImageOrientation)imageOrientation { ++ (FIRVisionDetectorImageOrientation)visionImageOrientationFromImageOrientation: + (UIImageOrientation)imageOrientation { switch (imageOrientation) { case UIImageOrientationUp: return FIRVisionDetectorImageOrientationTopLeft; @@ -69,7 +76,7 @@ + (FIRVisionDetectorImageOrientation) visionImageOrientationFromImageOrientation } } -+ (UIDeviceOrientation) currentUIOrientation { ++ (UIDeviceOrientation)currentUIOrientation { UIDeviceOrientation (^deviceOrientation)(void) = ^UIDeviceOrientation(void) { switch (UIApplication.sharedApplication.statusBarOrientation) { case UIInterfaceOrientationLandscapeLeft: @@ -83,7 +90,7 @@ + (UIDeviceOrientation) currentUIOrientation { return UIDeviceOrientationPortrait; } }; - + if (NSThread.isMainThread) { return deviceOrientation(); } else { From f98ce94b15972a6ac3527a84bcfa7282692d8ac3 Mon Sep 17 00:00:00 2001 From: dustin Date: Thu, 26 Jul 2018 09:58:05 -0700 Subject: [PATCH 27/34] Clean up LiveView implementation and fix formatting issues. --- .../example/lib/live_preview.dart | 3 +- .../ios/Classes/FirebaseMlVisionPlugin.h | 10 +- .../ios/Classes/FirebaseMlVisionPlugin.m | 36 ++- .../firebase_ml_vision/ios/Classes/LiveView.h | 23 +- .../firebase_ml_vision/ios/Classes/LiveView.m | 256 ++---------------- .../ios/Classes/UIUtilities.m | 5 - 6 files changed, 60 insertions(+), 273 deletions(-) diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index ae08081ae4dc..5dbdb4673831 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -98,8 +98,7 @@ class LivePreviewState extends State { ), ); } else if (loadState is LiveViewCameraLoadStateFailed) { - return new Text("error loading camera ${loadState - .errorMessage}"); + return new Text("error loading camera ${loadState.errorMessage}"); } else { return const Text("Unknown Camera error"); } diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h index 5106fdb16b30..59d67339755f 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h @@ -9,7 +9,7 @@ typedef void (^OperationFinishedCallback)(id _Nullable result, NSString *detectorType); typedef void (^OperationErrorCallback)(FlutterError *error); -@interface FLTFirebaseMlVisionPlugin : NSObject +@interface FLTFirebaseMlVisionPlugin : NSObject + (void)handleError:(NSError *)error finishedCallback:(OperationErrorCallback)callback; @end @@ -23,14 +23,14 @@ typedef void (^OperationErrorCallback)(FlutterError *error); @optional @end -@interface BarcodeDetector : NSObject +@interface BarcodeDetector : NSObject @end -@interface FaceDetector : NSObject +@interface FaceDetector : NSObject @end -@interface LabelDetector : NSObject +@interface LabelDetector : NSObject @end -@interface TextDetector : NSObject +@interface TextDetector : NSObject @end diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index 7a70a035aa72..3192a9bb7b93 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -5,7 +5,7 @@ @interface FLTFirebaseMlVisionPlugin () @property(readonly, nonatomic) NSObject *registry; @property(readonly, nonatomic) NSObject *messenger; -@property(readonly, nonatomic) FLTCam *camera; +@property(readonly, nonatomic) LiveView *camera; @end @implementation FLTFirebaseMlVisionPlugin @@ -43,11 +43,17 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result } result(nil); } else if ([@"availableCameras" isEqualToString:call.method]) { - AVCaptureDeviceDiscoverySession *discoverySession = [AVCaptureDeviceDiscoverySession - discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeBuiltInWideAngleCamera ] - mediaType:AVMediaTypeVideo - position:AVCaptureDevicePositionUnspecified]; - NSArray *devices = discoverySession.devices; + NSArray *devices; + if (@available(iOS 10.0, *)) { + AVCaptureDeviceDiscoverySession *discoverySession = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeBuiltInWideAngleCamera ] + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionUnspecified]; + devices = discoverySession.devices; + } else { + // Fallback on earlier versions + devices = AVCaptureDevice.devices; + } NSMutableArray *> *reply = [[NSMutableArray alloc] initWithCapacity:devices.count]; for (AVCaptureDevice *device in devices) { @@ -73,9 +79,9 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result NSString *cameraName = call.arguments[@"cameraName"]; NSString *resolutionPreset = call.arguments[@"resolutionPreset"]; NSError *error; - FLTCam *cam = [[FLTCam alloc] initWithCameraName:cameraName - resolutionPreset:resolutionPreset - error:&error]; + LiveView *cam = [[LiveView alloc] initWithCameraName:cameraName + resolutionPreset:resolutionPreset + error:&error]; if (error) { result([error flutterError]); } else { @@ -85,7 +91,7 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result int64_t textureId = [_registry registerTexture:cam]; _camera = cam; cam.onFrameAvailable = ^{ - [_registry textureFrameAvailable:textureId]; + [self->_registry textureFrameAvailable:textureId]; }; FlutterEventChannel *eventChannel = [FlutterEventChannel eventChannelWithName: @@ -95,13 +101,13 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result binaryMessenger:_messenger]; [eventChannel setStreamHandler:cam]; cam.eventChannel = eventChannel; - cam.onSizeAvailable = ^{ + cam.onSizeAvailable = ^(CGSize previewSize, CGSize captureSize) { result(@{ @"textureId" : @(textureId), - @"previewWidth" : @(cam.previewSize.width), - @"previewHeight" : @(cam.previewSize.height), - @"captureWidth" : @(cam.captureSize.width), - @"captureHeight" : @(cam.captureSize.height), + @"previewWidth" : @(previewSize.width), + @"previewHeight" : @(previewSize.height), + @"captureWidth" : @(captureSize.width), + @"captureHeight" : @(captureSize.height), }); }; [cam start]; diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.h b/packages/firebase_ml_vision/ios/Classes/LiveView.h index 73bcb0153f73..be5791e1ee1d 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.h +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.h @@ -1,41 +1,30 @@ #import #import #import +#import "FirebaseMlVisionPlugin.h" +@import FirebaseMLVision; -@interface FLTCam : NSObject +@interface LiveView : NSObject @property(readonly, nonatomic) int64_t textureId; @property(nonatomic) bool isUsingFrontCamera; -@property(nonatomic, copy) void (^onFrameAvailable)(); -@property(nonatomic, copy) void (^onSizeAvailable)(); +@property(nonatomic, copy) void (^onFrameAvailable)(void); +@property(nonatomic, copy) void (^onSizeAvailable)(CGSize previewSize, CGSize captureSize); @property(nonatomic) FlutterEventChannel *eventChannel; @property(nonatomic) FlutterEventSink eventSink; @property(readonly, nonatomic) AVCaptureSession *captureSession; @property(readonly, nonatomic) AVCaptureDevice *captureDevice; -@property(readonly, nonatomic) AVCapturePhotoOutput *capturePhotoOutput; @property(readonly, nonatomic) AVCaptureVideoDataOutput *captureVideoOutput; @property(readonly, nonatomic) AVCaptureInput *captureVideoInput; @property(readonly) CVPixelBufferRef volatile latestPixelBuffer; @property(readonly, nonatomic) CGSize previewSize; @property(readonly, nonatomic) CGSize captureSize; -@property(strong, nonatomic) AVAssetWriter *videoWriter; -@property(strong, nonatomic) AVAssetWriterInput *videoWriterInput; -@property(strong, nonatomic) AVAssetWriterInput *audioWriterInput; -@property(strong, nonatomic) AVAssetWriterInputPixelBufferAdaptor *assetWriterPixelBufferAdaptor; @property(strong, nonatomic) AVCaptureVideoDataOutput *videoOutput; -@property(strong, nonatomic) AVCaptureAudioDataOutput *audioOutput; -@property(assign, nonatomic) BOOL isRecording; -@property(assign, nonatomic) BOOL isAudioSetup; - (instancetype)initWithCameraName:(NSString *)cameraName resolutionPreset:(NSString *)resolutionPreset error:(NSError **)error; - (void)start; - (void)stop; - (void)close; -- (void)startVideoRecordingAtPath:(NSString *)path result:(FlutterResult)result; -- (void)stopVideoRecordingWithResult:(FlutterResult)result; -- (void)captureToFile:(NSString *)filename result:(FlutterResult)result; - (void)setDetector:(NSObject *)detector withOptions:(NSDictionary *)detectorOptions; @end diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.m b/packages/firebase_ml_vision/ios/Classes/LiveView.m index 7827557ddf9b..cf54a776d12e 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.m +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.m @@ -10,14 +10,14 @@ static NSString *const videoDataOutputQueueLabel = @"io.flutter.plugins.firebaseml.visiondetector.VideoDataOutputQueue"; -@interface FLTCam () +@interface LiveView () @property(assign, atomic) BOOL isRecognizing; @property(nonatomic) dispatch_queue_t sessionQueue; @property(strong, nonatomic) NSObject *currentDetector; @property(strong, nonatomic) NSDictionary *currentDetectorOptions; @end -@implementation FLTCam +@implementation LiveView - (instancetype)initWithCameraName:(NSString *)cameraName resolutionPreset:(NSString *)resolutionPreset error:(NSError **)error { @@ -61,15 +61,15 @@ - (void)setUpCaptureSessionOutputWithResolutionPreset:(NSString *)resolutionPres [self->_captureSession beginConfiguration]; self->_captureSession.sessionPreset = [self resolutionPresetForPreference:resolutionPreset]; - _captureVideoOutput = [[AVCaptureVideoDataOutput alloc] init]; - _captureVideoOutput.videoSettings = @{ + self->_captureVideoOutput = [[AVCaptureVideoDataOutput alloc] init]; + self->_captureVideoOutput.videoSettings = @{ (id) kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA] }; dispatch_queue_t outputQueue = dispatch_queue_create(videoDataOutputQueueLabel.UTF8String, nil); - [_captureVideoOutput setSampleBufferDelegate:self queue:outputQueue]; - if ([self.captureSession canAddOutput:_captureVideoOutput]) { - [self.captureSession addOutputWithNoConnections:_captureVideoOutput]; + [self->_captureVideoOutput setSampleBufferDelegate:self queue:outputQueue]; + if ([self.captureSession canAddOutput:self->_captureVideoOutput]) { + [self.captureSession addOutputWithNoConnections:self->_captureVideoOutput]; [self.captureSession commitConfiguration]; } else { NSLog(@"%@", @"Failed to add capture session output."); @@ -82,9 +82,9 @@ - (void)setUpCaptureSessionInputWithCameraName:(NSString *)cameraName { AVCaptureDevice *device = [AVCaptureDevice deviceWithUniqueID:cameraName]; CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions([[device activeFormat] formatDescription]); - _previewSize = CGSizeMake(dimensions.width, dimensions.height); - if (_onSizeAvailable) { - _onSizeAvailable(); + self->_previewSize = CGSizeMake(dimensions.width, dimensions.height); + if (self->_onSizeAvailable) { + self->_onSizeAvailable(self->_previewSize, self->_captureSize); } if (device) { NSArray *currentInputs = self.captureSession.inputs; @@ -92,30 +92,22 @@ - (void)setUpCaptureSessionInputWithCameraName:(NSString *)cameraName { [self.captureSession removeInput:input]; } NSError *error; - _captureVideoInput = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error]; + self->_captureVideoInput = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error]; if (error) { NSLog(@"Failed to create capture device input: %@", error.localizedDescription); return; } else { - // TODO? ACaptureConnection? AVCaptureConnection *connection = - [AVCaptureConnection connectionWithInputPorts:_captureVideoInput.ports - output:_captureVideoOutput]; - if ([_captureDevice position] == AVCaptureDevicePositionFront) { + [AVCaptureConnection connectionWithInputPorts:self->_captureVideoInput.ports + output:self->_captureVideoOutput]; + connection.videoOrientation = AVCaptureVideoOrientationPortraitUpsideDown; + if ([self->_captureDevice position] == AVCaptureDevicePositionFront) { connection.videoMirrored = YES; } - // connection.videoOrientation = AVCaptureVideoOrientationPortrait; - [_captureSession addInputWithNoConnections:_captureVideoInput]; - [_captureSession addConnection:connection]; - // if ([self.captureSession canAddInput:_captureVideoInput]) { - // [self.captureSession addInput:_captureVideoInput]; - // } else { - // NSLog(@"%@", @"Failed to add capture session input."); - // } + [self->_captureSession addInputWithNoConnections:self->_captureVideoInput]; + [self->_captureSession addConnection:connection]; } - } else { - NSLog(@"Failed to get capture device for camera position: %ld", cameraName); } }); } @@ -132,19 +124,6 @@ - (void)stop { }); } -- (void)captureToFile:(NSString *)path result:(FlutterResult)result { - // AVCapturePhotoSettings *settings = [AVCapturePhotoSettings photoSettings]; - // [_capturePhotoOutput - // capturePhotoWithSettings:settings - // delegate:[[FLTSavePhotoDelegate alloc] initWithPath:path result:result]]; -} - -- (void)captureOutput:(AVCaptureOutput *)output - didOutput:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection { - NSLog(@"Got Here!!!!"); -} - - (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { @@ -162,18 +141,23 @@ - (void)captureOutput:(AVCaptureOutput *)output metadata.orientation = visionOrientation; visionImage.metadata = metadata; - CGFloat imageWidth = CVPixelBufferGetWidth(newBuffer); - CGFloat imageHeight = CVPixelBufferGetHeight(newBuffer); [_currentDetector handleDetection:visionImage options:_currentDetectorOptions finishedCallback:^(id _Nullable result, NSString *detectorType) { self->_isRecognizing = NO; - self->_eventSink( - @{@"eventType" : @"detection", @"detectionType" : detectorType, @"data" : result}); + if (self->_eventSink != nil) { + self->_eventSink(@{ + @"eventType" : @"detection", + @"detectionType" : detectorType, + @"data" : result + }); + } } errorCallback:^(FlutterError *error) { self->_isRecognizing = NO; - self->_eventSink(error); + if (self->_eventSink != nil) { + self->_eventSink(error); + } }]; } CFRetain(newBuffer); @@ -188,14 +172,6 @@ - (void)captureOutput:(AVCaptureOutput *)output _onFrameAvailable(); } } - // switch (_currentDetector) { - // case DetectorOnDeviceFace: - // [self detectFacesOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; - // break; - // case DetectorOnDeviceText: - // [self detectTextOnDeviceInImage:visionImage width:imageWidth height:imageHeight]; - // break; - // } if (!CMSampleBufferDataIsReady(sampleBuffer)) { _eventSink(@{ @"event" : @"error", @@ -203,67 +179,6 @@ - (void)captureOutput:(AVCaptureOutput *)output }); return; } - if (_isRecording) { - if (_videoWriter.status == AVAssetWriterStatusFailed) { - _eventSink(@{ - @"event" : @"error", - @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] - }); - return; - } - CMTime lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); - if (_videoWriter.status != AVAssetWriterStatusWriting) { - [_videoWriter startWriting]; - [_videoWriter startSessionAtSourceTime:lastSampleTime]; - } - if (output == _captureVideoOutput) { - [self newVideoSample:sampleBuffer]; - } else { - [self newAudioSample:sampleBuffer]; - } - } -} - -- (void)newVideoSample:(CMSampleBufferRef)sampleBuffer { - if (_videoWriter.status != AVAssetWriterStatusWriting) { - if (_videoWriter.status == AVAssetWriterStatusFailed) { - _eventSink(@{ - @"event" : @"error", - @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] - }); - } - return; - } - if (_videoWriterInput.readyForMoreMediaData) { - if (![_videoWriterInput appendSampleBuffer:sampleBuffer]) { - _eventSink(@{ - @"event" : @"error", - @"errorDescription" : - [NSString stringWithFormat:@"%@", @"Unable to write to video input"] - }); - } - } -} - -- (void)newAudioSample:(CMSampleBufferRef)sampleBuffer { - if (_videoWriter.status != AVAssetWriterStatusWriting) { - if (_videoWriter.status == AVAssetWriterStatusFailed) { - _eventSink(@{ - @"event" : @"error", - @"errorDescription" : [NSString stringWithFormat:@"%@", _videoWriter.error] - }); - } - return; - } - if (_audioWriterInput.readyForMoreMediaData) { - if (![_audioWriterInput appendSampleBuffer:sampleBuffer]) { - _eventSink(@{ - @"event" : @"error", - @"errorDescription" : - [NSString stringWithFormat:@"%@", @"Unable to write to audio input"] - }); - } - } } - (void)close { @@ -300,122 +215,5 @@ - (FlutterError *_Nullable)onListenWithArguments:(id _Nullable)arguments _eventSink = events; return nil; } -- (void)startVideoRecordingAtPath:(NSString *)path result:(FlutterResult)result { - if (!_isRecording) { - if (![self setupWriterForPath:path]) { - _eventSink(@{@"event" : @"error", @"errorDescription" : @"Setup Writer Failed"}); - return; - } - [_captureSession stopRunning]; - _isRecording = YES; - [_captureSession startRunning]; - result(nil); - } else { - _eventSink(@{@"event" : @"error", @"errorDescription" : @"Video is already recording!"}); - } -} - -- (void)stopVideoRecordingWithResult:(FlutterResult)result { - if (_isRecording) { - _isRecording = NO; - if (_videoWriter.status != AVAssetWriterStatusUnknown) { - [_videoWriter finishWritingWithCompletionHandler:^{ - if (self->_videoWriter.status == AVAssetWriterStatusCompleted) { - result(nil); - } else { - self->_eventSink(@{ - @"event" : @"error", - @"errorDescription" : @"AVAssetWriter could not finish writing!" - }); - } - }]; - } - } else { - // NSError *error = - // [NSError errorWithDomain:NSCocoaErrorDomain - // code:NSURLErrorResourceUnavailable - // userInfo:@{NSLocalizedDescriptionKey : @"Video is not recording!"}]; - // result([error flutterError]); - } -} - -- (BOOL)setupWriterForPath:(NSString *)path { - NSError *error = nil; - NSURL *outputURL; - if (path != nil) { - outputURL = [NSURL fileURLWithPath:path]; - } else { - return NO; - } - if (!_isAudioSetup) { - [self setUpCaptureSessionForAudio]; - } - _videoWriter = - [[AVAssetWriter alloc] initWithURL:outputURL fileType:AVFileTypeQuickTimeMovie error:&error]; - NSParameterAssert(_videoWriter); - if (error) { - _eventSink(@{@"event" : @"error", @"errorDescription" : error.description}); - return NO; - } - NSDictionary *videoSettings = [NSDictionary - dictionaryWithObjectsAndKeys:AVVideoCodecH264, AVVideoCodecKey, - [NSNumber numberWithInt:_previewSize.height], AVVideoWidthKey, - [NSNumber numberWithInt:_previewSize.width], AVVideoHeightKey, - nil]; - _videoWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo - outputSettings:videoSettings]; - NSParameterAssert(_videoWriterInput); - _videoWriterInput.expectsMediaDataInRealTime = YES; - - // Add the audio input - AudioChannelLayout acl; - bzero(&acl, sizeof(acl)); - acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono; - NSDictionary *audioOutputSettings = nil; - // Both type of audio inputs causes output video file to be corrupted. - audioOutputSettings = [NSDictionary - dictionaryWithObjectsAndKeys:[NSNumber numberWithInt:kAudioFormatMPEG4AAC], AVFormatIDKey, - [NSNumber numberWithFloat:44100.0], AVSampleRateKey, - [NSNumber numberWithInt:1], AVNumberOfChannelsKey, - [NSData dataWithBytes:&acl length:sizeof(acl)], - AVChannelLayoutKey, nil]; - _audioWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio - outputSettings:audioOutputSettings]; - _audioWriterInput.expectsMediaDataInRealTime = YES; - [_videoWriter addInput:_videoWriterInput]; - [_videoWriter addInput:_audioWriterInput]; - dispatch_queue_t queue = dispatch_queue_create("MyQueue", NULL); - [_captureVideoOutput setSampleBufferDelegate:self queue:queue]; - [_audioOutput setSampleBufferDelegate:self queue:queue]; - - return YES; -} -- (void)setUpCaptureSessionForAudio { - NSError *error = nil; - // Create a device input with the device and add it to the session. - // Setup the audio input. - AVCaptureDevice *audioDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; - AVCaptureDeviceInput *audioInput = - [AVCaptureDeviceInput deviceInputWithDevice:audioDevice error:&error]; - if (error) { - _eventSink(@{@"event" : @"error", @"errorDescription" : error.description}); - } - // Setup the audio output. - _audioOutput = [[AVCaptureAudioDataOutput alloc] init]; - - if ([_captureSession canAddInput:audioInput]) { - [_captureSession addInput:audioInput]; - if ([_captureSession canAddOutput:_audioOutput]) { - [_captureSession addOutput:_audioOutput]; - _isAudioSetup = YES; - } else { - _eventSink(@{ - @"event" : @"error", - @"errorDescription" : @"Unable to add Audio input/output to session capture" - }); - _isAudioSetup = NO; - } - } -} @end diff --git a/packages/firebase_ml_vision/ios/Classes/UIUtilities.m b/packages/firebase_ml_vision/ios/Classes/UIUtilities.m index 4bca947f5a95..191da95542e7 100644 --- a/packages/firebase_ml_vision/ios/Classes/UIUtilities.m +++ b/packages/firebase_ml_vision/ios/Classes/UIUtilities.m @@ -16,11 +16,6 @@ #import "UIUtilities.h" -static CGFloat const circleViewAlpha = 0.7; -static CGFloat const rectangleViewAlpha = 0.3; -static CGFloat const shapeViewAlpha = 0.3; -static CGFloat const rectangleViewCornerRadius = 10.0; - @implementation UIUtilities + (void)imageOrientation { From d7e6b3260925df5f45d166d0ec7af7f8b07b162a Mon Sep 17 00:00:00 2001 From: dustin Date: Thu, 26 Jul 2018 13:40:24 -0700 Subject: [PATCH 28/34] update to support new LabelDetector. --- packages/device_info/lib/device_info.dart | 4 +-- .../lib/src/firebase_database.dart | 5 ++- .../FirebaseMlVisionPlugin.java | 2 +- .../firebasemlvision/LabelDetector.java | 24 +++++++------ .../ios/Classes/FirebaseMlVisionPlugin.m | 34 +++++++++++++------ .../ios/Classes/LabelDetector.m | 22 ++++++++---- .../firebase_ml_vision/ios/Classes/LiveView.m | 2 +- .../firebase_ml_vision/lib/src/live_view.dart | 2 +- 8 files changed, 59 insertions(+), 36 deletions(-) diff --git a/packages/device_info/lib/device_info.dart b/packages/device_info/lib/device_info.dart index 19ae42cc5cca..bf727c265c18 100644 --- a/packages/device_info/lib/device_info.dart +++ b/packages/device_info/lib/device_info.dart @@ -21,8 +21,8 @@ class DeviceInfoPlugin { /// /// See: https://developer.android.com/reference/android/os/Build.html Future get androidInfo async => - _cachedAndroidDeviceInfo ??= AndroidDeviceInfo._fromMap( - await channel.invokeMethod('getAndroidDeviceInfo')); + _cachedAndroidDeviceInfo ??= AndroidDeviceInfo + ._fromMap(await channel.invokeMethod('getAndroidDeviceInfo')); /// This information does not change from call to call. Cache it. IosDeviceInfo _cachedIosDeviceInfo; diff --git a/packages/firebase_database/lib/src/firebase_database.dart b/packages/firebase_database/lib/src/firebase_database.dart index 29db8db76963..f397d965525e 100644 --- a/packages/firebase_database/lib/src/firebase_database.dart +++ b/packages/firebase_database/lib/src/firebase_database.dart @@ -39,9 +39,8 @@ class FirebaseDatabase { case 'DoTransaction': final MutableData mutableData = new MutableData.private(call.arguments['snapshot']); - final MutableData updated = - await _transactions[call.arguments['transactionKey']]( - mutableData); + final MutableData updated = await _transactions[ + call.arguments['transactionKey']](mutableData); return {'value': updated.value}; default: throw new MissingPluginException( diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index f46b00235b85..550e153463bc 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -190,7 +190,7 @@ public void onOpened(long textureId, int width, int height) { case "LabelDetector#detectInImage": try { image = filePathToVisionImage((String) call.argument("path")); - LabelDetector.instance.handleDetection(image, options, result); + LabelDetector.instance.handleDetection(image, options, handleDetection(result)); } catch (IOException e) { result.error("labelDetectorIOError", e.getLocalizedMessage(), null); } catch (Exception e) { diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java index 99a6860377f9..632ed9dc90b4 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java @@ -8,20 +8,26 @@ import com.google.firebase.ml.vision.label.FirebaseVisionLabel; import com.google.firebase.ml.vision.label.FirebaseVisionLabelDetector; import com.google.firebase.ml.vision.label.FirebaseVisionLabelDetectorOptions; -import io.flutter.plugin.common.MethodChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -class LabelDetector implements Detector { +public class LabelDetector extends Detector { public static final LabelDetector instance = new LabelDetector(); private LabelDetector() {} + private FirebaseVisionLabelDetectorOptions parseOptions(Map optionsData) { + float conf = (float) (double) optionsData.get("confidenceThreshold"); + return new FirebaseVisionLabelDetectorOptions.Builder().setConfidenceThreshold(conf).build(); + } + @Override - public void handleDetection( - FirebaseVisionImage image, Map options, final MethodChannel.Result result) { + void processImage( + FirebaseVisionImage image, + Map options, + final OperationFinishedCallback finishedCallback) { FirebaseVisionLabelDetector detector = FirebaseVision.getInstance().getVisionLabelDetector(parseOptions(options)); detector @@ -40,20 +46,16 @@ public void onSuccess(List firebaseVisionLabels) { labels.add(labelData); } - result.success(labels); + finishedCallback.success(LabelDetector.this, labels); } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { - result.error("labelDetectorError", e.getLocalizedMessage(), null); + finishedCallback.error( + new DetectorException("labelDetectorError", e.getLocalizedMessage(), null)); } }); } - - private FirebaseVisionLabelDetectorOptions parseOptions(Map optionsData) { - float conf = (float) (double) optionsData.get("confidenceThreshold"); - return new FirebaseVisionLabelDetectorOptions.Builder().setConfidenceThreshold(conf).build(); - } } diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index 4a8b2bd12f78..548434c6bba0 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -128,20 +128,32 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result } result(nil); } else { - // image file detectionFIRVisionImage *image = [self filePathToVisionImage:call.arguments[@"path"]]; - NSDictionary *options = call.arguments[@"options"]; - if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { - [[BarcodeDetector sharedInstance]handleDetection:image options:options finishedCallback:[self handleSuccess:result] + // image file detection + FIRVisionImage *image = [self filePathToVisionImage:call.arguments[@"path"]]; + NSDictionary *options = call.arguments[@"options"]; + if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { + [[BarcodeDetector sharedInstance] handleDetection:image + options:options + finishedCallback:[self handleSuccess:result] errorCallback:[self handleError:result]]; - } else if ([@"FaceDetector#detectInImage" isEqualToString:call.method]) { - [[FaceDetector sharedInstance]handleDetection:image options:options finishedCallback:[self handleSuccess:result] + } else if ([@"FaceDetector#detectInImage" isEqualToString:call.method]) { + [[FaceDetector sharedInstance] handleDetection:image + options:options + finishedCallback:[self handleSuccess:result] errorCallback:[self handleError:result]]; - } else if ([@"LabelDetector#detectInImage" isEqualToString:call.method]) { - [LabelDetector handleDetection:image options:options result:result];} else if ([@"TextDetector#detectInImage" isEqualToString:call.method]) { - [[TextDetector sharedInstance]handleDetection:image options:options finishedCallback:[self handleSuccess:result] + } else if ([@"LabelDetector#detectInImage" isEqualToString:call.method]) { + [[LabelDetector sharedInstance] handleDetection:image + options:options + finishedCallback:[self handleSuccess:result] + errorCallback:[self handleError:result]]; + } else if ([@"TextDetector#detectInImage" isEqualToString:call.method]) { + [[TextDetector sharedInstance] handleDetection:image + options:options + finishedCallback:[self handleSuccess:result] errorCallback:[self handleError:result]]; - } else { - result(FlutterMethodNotImplemented);} + } else { + result(FlutterMethodNotImplemented); + } } } diff --git a/packages/firebase_ml_vision/ios/Classes/LabelDetector.m b/packages/firebase_ml_vision/ios/Classes/LabelDetector.m index ca8e359f46c6..c89416aedfe2 100644 --- a/packages/firebase_ml_vision/ios/Classes/LabelDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/LabelDetector.m @@ -3,19 +3,29 @@ @implementation LabelDetector static FIRVisionLabelDetector *detector; -+ (void)handleDetection:(FIRVisionImage *)image ++ (id)sharedInstance { + static LabelDetector *sharedInstance = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + sharedInstance = [[self alloc] init]; + }); + return sharedInstance; +} + +- (void)handleDetection:(FIRVisionImage *)image options:(NSDictionary *)options - result:(FlutterResult)result { + finishedCallback:(OperationFinishedCallback)callback + errorCallback:(OperationErrorCallback)errorCallback { FIRVision *vision = [FIRVision vision]; detector = [vision labelDetectorWithOptions:[LabelDetector parseOptions:options]]; [detector detectInImage:image completion:^(NSArray *_Nullable labels, NSError *_Nullable error) { if (error) { - [FLTFirebaseMlVisionPlugin handleError:error result:result]; + [FLTFirebaseMlVisionPlugin handleError:error finishedCallback:errorCallback]; return; } else if (!labels) { - result(@[]); + callback(@[], @"label"); } NSMutableArray *labelData = [NSMutableArray array]; @@ -28,7 +38,7 @@ + (void)handleDetection:(FIRVisionImage *)image [labelData addObject:data]; } - result(labelData); + callback(labelData, @"label"); }]; } @@ -36,4 +46,4 @@ + (FIRVisionLabelDetectorOptions *)parseOptions:(NSDictionary *)optionsData { NSNumber *conf = optionsData[@"confidenceThreshold"]; return [[FIRVisionLabelDetectorOptions alloc] initWithConfidenceThreshold:[conf floatValue]]; } -@end \ No newline at end of file +@end diff --git a/packages/firebase_ml_vision/ios/Classes/LiveView.m b/packages/firebase_ml_vision/ios/Classes/LiveView.m index cf54a776d12e..9e99bd776937 100644 --- a/packages/firebase_ml_vision/ios/Classes/LiveView.m +++ b/packages/firebase_ml_vision/ios/Classes/LiveView.m @@ -101,7 +101,7 @@ - (void)setUpCaptureSessionInputWithCameraName:(NSString *)cameraName { AVCaptureConnection *connection = [AVCaptureConnection connectionWithInputPorts:self->_captureVideoInput.ports output:self->_captureVideoOutput]; - connection.videoOrientation = AVCaptureVideoOrientationPortraitUpsideDown; + // connection.videoOrientation = AVCaptureVideoOrientationPortrait; if ([self->_captureDevice position] == AVCaptureDevicePositionFront) { connection.videoMirrored = YES; } diff --git a/packages/firebase_ml_vision/lib/src/live_view.dart b/packages/firebase_ml_vision/lib/src/live_view.dart index e8dcaf3a3348..e49e52002d4d 100644 --- a/packages/firebase_ml_vision/lib/src/live_view.dart +++ b/packages/firebase_ml_vision/lib/src/live_view.dart @@ -242,7 +242,7 @@ class LiveViewCameraController extends ValueNotifier { Future setDetector(FirebaseVisionDetectorType detectorType, [Map options]) async { if (detectorType == FirebaseVisionDetectorType.face && options == null) { - options = new FaceDetectorOptions().optionsMap; + options = const FaceDetectorOptions().optionsMap; } await FirebaseVision.instance.setLiveViewDetector(detectorType, options); } From 776294df43f2d864f8ac608c9fe80ac46eb12dec Mon Sep 17 00:00:00 2001 From: dustin Date: Thu, 26 Jul 2018 14:02:02 -0700 Subject: [PATCH 29/34] undo inadvertent formatting changes outside firebase_ml_vision. --- packages/device_info/lib/device_info.dart | 4 ++-- packages/firebase_database/lib/src/firebase_database.dart | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/device_info/lib/device_info.dart b/packages/device_info/lib/device_info.dart index bf727c265c18..19ae42cc5cca 100644 --- a/packages/device_info/lib/device_info.dart +++ b/packages/device_info/lib/device_info.dart @@ -21,8 +21,8 @@ class DeviceInfoPlugin { /// /// See: https://developer.android.com/reference/android/os/Build.html Future get androidInfo async => - _cachedAndroidDeviceInfo ??= AndroidDeviceInfo - ._fromMap(await channel.invokeMethod('getAndroidDeviceInfo')); + _cachedAndroidDeviceInfo ??= AndroidDeviceInfo._fromMap( + await channel.invokeMethod('getAndroidDeviceInfo')); /// This information does not change from call to call. Cache it. IosDeviceInfo _cachedIosDeviceInfo; diff --git a/packages/firebase_database/lib/src/firebase_database.dart b/packages/firebase_database/lib/src/firebase_database.dart index f397d965525e..29db8db76963 100644 --- a/packages/firebase_database/lib/src/firebase_database.dart +++ b/packages/firebase_database/lib/src/firebase_database.dart @@ -39,8 +39,9 @@ class FirebaseDatabase { case 'DoTransaction': final MutableData mutableData = new MutableData.private(call.arguments['snapshot']); - final MutableData updated = await _transactions[ - call.arguments['transactionKey']](mutableData); + final MutableData updated = + await _transactions[call.arguments['transactionKey']]( + mutableData); return {'value': updated.value}; default: throw new MissingPluginException( From ef72e0a06ac95e0876f22974a2c3f5ded1753afb Mon Sep 17 00:00:00 2001 From: dustin Date: Thu, 26 Jul 2018 23:39:13 -0700 Subject: [PATCH 30/34] add camera plugin as a dependency to firebase_ml_vision This will help us add a live preview without re-implementing the camera --- .../flutter/plugins/camera/CameraPlugin.java | 112 ++++++++++++++---- .../plugins/camera/PreviewImageDelegate.java | 7 ++ .../xcshareddata/IDEWorkspaceChecks.plist | 8 ++ .../firebase_ml_vision/android/build.gradle | 3 +- .../FirebaseMlVisionPlugin.java | 17 ++- .../live/CameraPreviewImageProvider.java | 7 ++ .../example/android/app/build.gradle | 2 +- .../firebasemlvisionexample/MainActivity.java | 27 ++++- .../example/lib/live_preview.dart | 31 ++--- .../firebase_ml_vision/example/pubspec.yaml | 2 + packages/firebase_ml_vision/pubspec.yaml | 3 + 11 files changed, 172 insertions(+), 47 deletions(-) create mode 100644 packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java create mode 100644 packages/camera/example/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraPreviewImageProvider.java diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 1d4bedefbbf4..693f96d01a1a 100644 --- a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -21,19 +21,15 @@ import android.media.MediaRecorder; import android.os.Build; import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.util.Log; import android.util.Size; import android.util.SparseIntArray; import android.view.Surface; -import io.flutter.plugin.common.EventChannel; -import io.flutter.plugin.common.MethodCall; -import io.flutter.plugin.common.MethodChannel; -import io.flutter.plugin.common.MethodChannel.MethodCallHandler; -import io.flutter.plugin.common.MethodChannel.Result; -import io.flutter.plugin.common.PluginRegistry; -import io.flutter.plugin.common.PluginRegistry.Registrar; -import io.flutter.view.FlutterView; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -46,6 +42,15 @@ import java.util.List; import java.util.Map; +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugin.common.MethodCall; +import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugin.common.MethodChannel.MethodCallHandler; +import io.flutter.plugin.common.MethodChannel.Result; +import io.flutter.plugin.common.PluginRegistry; +import io.flutter.plugin.common.PluginRegistry.Registrar; +import io.flutter.view.FlutterView; + public class CameraPlugin implements MethodCallHandler { private static final int CAMERA_REQUEST_ID = 513469796; @@ -68,11 +73,17 @@ public class CameraPlugin implements MethodCallHandler { // The code to run after requesting camera permissions. private Runnable cameraPermissionContinuation; private boolean requestingPermission; + @Nullable private PreviewImageDelegate previewImageDelegate; private CameraPlugin(Registrar registrar, FlutterView view, Activity activity) { + Log.d("ML", "registering camera plugin"); this.registrar = registrar; this.view = view; this.activity = activity; + if (activity instanceof PreviewImageDelegate) { + Log.d("ML", "the activity is a PreviewImageDelegate, assigning the image delegate now"); + this.previewImageDelegate = (PreviewImageDelegate) activity; + } registrar.addRequestPermissionsResultListener(new CameraRequestPermissionsListener()); @@ -376,7 +387,8 @@ private void computeBestPreviewAndRecordingSize( } else { previewSize = goodEnough.get(0); - // Video capture size should not be greater than 1080 because MediaRecorder cannot handle higher resolutions. + // Video capture size should not be greater than 1080 because MediaRecorder cannot handle + // higher resolutions. videoSize = goodEnough.get(0); for (int i = goodEnough.size() - 1; i >= 0; i--) { if (goodEnough.get(i).getHeight() <= 1080) { @@ -419,14 +431,59 @@ private void prepareMediaRecorder(String outputFilePath) throws IOException { mediaRecorder.prepare(); } + private Handler mBackgroundHandler; + private HandlerThread mBackgroundThread; + private Surface imageReaderSurface; + private final ImageReader.OnImageAvailableListener imageAvailable = + new ImageReader.OnImageAvailableListener() { + @Override + public void onImageAvailable(ImageReader reader) { + Log.d("ML", "ImageReader image available..."); + Image image = reader.acquireLatestImage(); + if (image != null) { + // Log.d("ML", "image was not null"); + if (previewImageDelegate != null) { + previewImageDelegate.onImageAvailable(image); + } + image.close(); + } + } + }; + + /** Starts a background thread and its {@link Handler}. */ + private void startBackgroundThread() { + mBackgroundThread = new HandlerThread("CameraBackground"); + mBackgroundThread.start(); + mBackgroundHandler = new Handler(mBackgroundThread.getLooper()); + } + + /** Stops the background thread and its {@link Handler}. */ + private void stopBackgroundThread() { + if (mBackgroundThread != null) { + mBackgroundThread.quitSafely(); + try { + mBackgroundThread.join(); + mBackgroundThread = null; + mBackgroundHandler = null; + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + private void open(@Nullable final Result result) { if (!hasCameraPermission()) { if (result != null) result.error("cameraPermission", "Camera permission not granted", null); } else { try { + Log.d("ML", "opening camera"); + startBackgroundThread(); imageReader = ImageReader.newInstance( captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2); + imageReaderSurface = imageReader.getSurface(); + imageReader.setOnImageAvailableListener(imageAvailable, mBackgroundHandler); + Log.d("ML", "assigned image available listener, boo"); cameraManager.openCamera( cameraName, new CameraDevice.StateCallback() { @@ -519,20 +576,25 @@ private void takePicture(String filePath, @NonNull final Result result) { return; } - imageReader.setOnImageAvailableListener( - new ImageReader.OnImageAvailableListener() { - @Override - public void onImageAvailable(ImageReader reader) { - try (Image image = reader.acquireLatestImage()) { - ByteBuffer buffer = image.getPlanes()[0].getBuffer(); - writeToFile(buffer, file); - result.success(null); - } catch (IOException e) { - result.error("IOError", "Failed saving image", null); - } - } - }, - null); + Log.d("ML", "not setting the take picture listener"); +// imageReader.setOnImageAvailableListener( +// new ImageReader.OnImageAvailableListener() { +// @Override +// public void onImageAvailable(ImageReader reader) { +// try (Image image = reader.acquireLatestImage()) { +// if (previewImageDelegate != null) { +// Log.d("ML", "the preview image delegate is not null, sending the image"); +// previewImageDelegate.onImageAvailable(image); +// } +// ByteBuffer buffer = image.getPlanes()[0].getBuffer(); +// writeToFile(buffer, file); +// result.success(null); +// } catch (IOException e) { +// result.error("IOError", "Failed saving image", null); +// } +// } +// }, +// null); try { final CaptureRequest.Builder captureBuilder = @@ -667,7 +729,8 @@ private void startPreview() throws CameraAccessException { surfaces.add(previewSurface); captureRequestBuilder.addTarget(previewSurface); - surfaces.add(imageReader.getSurface()); + surfaces.add(imageReaderSurface); + captureRequestBuilder.addTarget(imageReaderSurface); cameraDevice.createCaptureSession( surfaces, @@ -729,6 +792,7 @@ private void close() { mediaRecorder.release(); mediaRecorder = null; } + stopBackgroundThread(); } private void dispose() { diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java new file mode 100644 index 000000000000..f0d3291eb499 --- /dev/null +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java @@ -0,0 +1,7 @@ +package io.flutter.plugins.camera; + +import android.media.Image; + +public interface PreviewImageDelegate { + void onImageAvailable(Image image); +} diff --git a/packages/camera/example/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/packages/camera/example/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 000000000000..18d981003d68 --- /dev/null +++ b/packages/camera/example/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/packages/firebase_ml_vision/android/build.gradle b/packages/firebase_ml_vision/android/build.gradle index e24abb469cec..e17215ec4ad3 100644 --- a/packages/firebase_ml_vision/android/build.gradle +++ b/packages/firebase_ml_vision/android/build.gradle @@ -25,7 +25,7 @@ android { compileSdkVersion 27 defaultConfig { - minSdkVersion 16 + minSdkVersion 21 testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" } lintOptions { @@ -34,5 +34,6 @@ android { dependencies { api 'com.google.firebase:firebase-ml-vision:16.0.0' api 'com.google.firebase:firebase-ml-vision-image-label-model:15.0.0' + implementation project(':camera') } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 550e153463bc..bf802445f3ae 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -2,9 +2,12 @@ import android.app.Activity; import android.app.Application; +import android.media.Image; import android.net.Uri; import android.os.Bundle; import android.support.annotation.Nullable; +import android.util.Log; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; @@ -12,7 +15,10 @@ import io.flutter.plugin.common.MethodChannel.Result; import io.flutter.plugin.common.PluginRegistry; import io.flutter.plugin.common.PluginRegistry.Registrar; +import io.flutter.plugins.camera.PreviewImageDelegate; +import io.flutter.plugins.firebasemlvision.live.CameraPreviewImageProvider; import io.flutter.plugins.firebasemlvision.live.LegacyCamera; + import java.io.File; import java.io.IOException; import java.util.HashMap; @@ -20,7 +26,7 @@ import java.util.Map; /** FirebaseMlVisionPlugin */ -public class FirebaseMlVisionPlugin implements MethodCallHandler { +public class FirebaseMlVisionPlugin implements MethodCallHandler, PreviewImageDelegate { public static final int CAMERA_REQUEST_ID = 928291720; private final Registrar registrar; private final Activity activity; @@ -30,6 +36,10 @@ public class FirebaseMlVisionPlugin implements MethodCallHandler { private FirebaseMlVisionPlugin(Registrar registrar) { this.registrar = registrar; this.activity = registrar.activity(); + if (activity instanceof CameraPreviewImageProvider) { + Log.d("ML", "the activity is a CameraPreviewImageProvider, setting self as a delegate"); + ((CameraPreviewImageProvider)activity).setImageDelegate(this); + } registrar.addRequestPermissionsResultListener(new CameraRequestPermissionsListener()); @@ -212,6 +222,11 @@ public void onOpened(long textureId, int width, int height) { } } + @Override + public void onImageAvailable(Image image) { + Log.d("ML", "got an image"); + } + private Detector.OperationFinishedCallback handleDetection(final Result result) { return new Detector.OperationFinishedCallback() { @Override diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraPreviewImageProvider.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraPreviewImageProvider.java new file mode 100644 index 000000000000..5424c0d99a0b --- /dev/null +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/CameraPreviewImageProvider.java @@ -0,0 +1,7 @@ +package io.flutter.plugins.firebasemlvision.live; + +import io.flutter.plugins.camera.PreviewImageDelegate; + +public interface CameraPreviewImageProvider { + void setImageDelegate(PreviewImageDelegate delegate); +} diff --git a/packages/firebase_ml_vision/example/android/app/build.gradle b/packages/firebase_ml_vision/example/android/app/build.gradle index 9383e31cc780..2c8a7e9fcb16 100644 --- a/packages/firebase_ml_vision/example/android/app/build.gradle +++ b/packages/firebase_ml_vision/example/android/app/build.gradle @@ -23,7 +23,7 @@ android { defaultConfig { applicationId "io.flutter.plugins.firebasemlvisionexample" - minSdkVersion 16 + minSdkVersion 21 } buildTypes { diff --git a/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java b/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java index f5bfd378a945..4dd4d41340c1 100644 --- a/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java +++ b/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java @@ -1,13 +1,38 @@ package io.flutter.plugins.firebasemlvisionexample; +import android.media.Image; import android.os.Bundle; +import android.support.annotation.Nullable; +import android.util.Log; + import io.flutter.app.FlutterActivity; import io.flutter.plugins.GeneratedPluginRegistrant; +import io.flutter.plugins.camera.PreviewImageDelegate; +import io.flutter.plugins.firebasemlvision.live.CameraPreviewImageProvider; + +public class MainActivity extends FlutterActivity implements PreviewImageDelegate, CameraPreviewImageProvider { + @Nullable + private PreviewImageDelegate previewImageDelegate; -public class MainActivity extends FlutterActivity { @Override protected void onCreate(Bundle savedInstanceState) { + Log.d("ML", "MainActivity created"); super.onCreate(savedInstanceState); GeneratedPluginRegistrant.registerWith(this); } + + @Override + public void onImageAvailable(Image image) { + Log.d("ML", "got a preview image"); + if (previewImageDelegate != null) { + Log.d("ML", "the delegate was not null, sending image to ml for processing"); + previewImageDelegate.onImageAvailable(image); + } + } + + @Override + public void setImageDelegate(PreviewImageDelegate delegate) { + Log.d("ML", "setting image delegate"); + previewImageDelegate = delegate; + } } diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 5dbdb4673831..2d0a91e6fae1 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:firebase_ml_vision/firebase_ml_vision.dart'; import 'package:firebase_ml_vision_example/detector_painters.dart'; import 'package:flutter/material.dart'; +import 'package:camera/camera.dart' as camera; class LivePreview extends StatefulWidget { final FirebaseVisionDetectorType detector; @@ -27,17 +28,17 @@ class LivePreviewState extends State { yield _readyLoadState; } else { yield new LiveViewCameraLoadStateLoading(); - final List cameras = await availableCameras(); - final LiveViewCameraDescription backCamera = cameras.firstWhere( - (LiveViewCameraDescription cameraDescription) => + final List cameras = await camera.availableCameras(); + final camera.CameraDescription backCamera = cameras.firstWhere( + (camera.CameraDescription cameraDescription) => cameraDescription.lensDirection == - LiveViewCameraLensDirection.back); + camera.CameraLensDirection.back); if (backCamera != null) { yield new LiveViewCameraLoadStateLoaded(backCamera); try { - final LiveViewCameraController controller = - new LiveViewCameraController( - backCamera, LiveViewResolutionPreset.high); + final camera.CameraController controller = + new camera.CameraController( + backCamera, camera.ResolutionPreset.high); await controller.initialize(); await setLiveViewDetector(); yield new LiveViewCameraLoadStateReady(controller); @@ -58,7 +59,7 @@ class LivePreviewState extends State { } Future setLiveViewDetector() async { - return _readyLoadState?.controller?.setDetector(widget.detector); + FirebaseVision.instance.setLiveViewDetector(widget.detector); } @override @@ -87,15 +88,7 @@ class LivePreviewState extends State { } return new AspectRatio( aspectRatio: _readyLoadState.controller.value.aspectRatio, - child: new LiveView( - controller: _readyLoadState.controller, - overlayBuilder: (BuildContext context, Size previewSize, - LiveViewDetectionList data) { - return data == null - ? new Container() - : customPaintForResults(previewSize, data); - }, - ), + child: new camera.CameraPreview(loadState.controller), ); } else if (loadState is LiveViewCameraLoadStateFailed) { return new Text("error loading camera ${loadState.errorMessage}"); @@ -115,13 +108,13 @@ abstract class LiveViewCameraLoadState {} class LiveViewCameraLoadStateLoading extends LiveViewCameraLoadState {} class LiveViewCameraLoadStateLoaded extends LiveViewCameraLoadState { - final LiveViewCameraDescription cameraDescription; + final camera.CameraDescription cameraDescription; LiveViewCameraLoadStateLoaded(this.cameraDescription); } class LiveViewCameraLoadStateReady extends LiveViewCameraLoadState { - final LiveViewCameraController controller; + final camera.CameraController controller; LiveViewCameraLoadStateReady(this.controller); diff --git a/packages/firebase_ml_vision/example/pubspec.yaml b/packages/firebase_ml_vision/example/pubspec.yaml index 5bf287f73e79..51a3c58a641b 100644 --- a/packages/firebase_ml_vision/example/pubspec.yaml +++ b/packages/firebase_ml_vision/example/pubspec.yaml @@ -13,6 +13,8 @@ dependencies: dev_dependencies: firebase_ml_vision: path: ../ + camera: + path: ../../camera flutter: uses-material-design: true diff --git a/packages/firebase_ml_vision/pubspec.yaml b/packages/firebase_ml_vision/pubspec.yaml index 7e19f091f73b..156d15694d27 100644 --- a/packages/firebase_ml_vision/pubspec.yaml +++ b/packages/firebase_ml_vision/pubspec.yaml @@ -8,10 +8,13 @@ homepage: https://github.com/flutter/plugins/tree/master/packages/firebase_ml_vi dependencies: flutter: sdk: flutter + video_player: ^0.6.4 dev_dependencies: flutter_test: sdk: flutter + camera: + path: ../camera flutter: plugin: From 543b6d786f544d296462ef68e9549ff70a4ca905 Mon Sep 17 00:00:00 2001 From: dustin Date: Fri, 27 Jul 2018 00:26:46 -0700 Subject: [PATCH 31/34] wIP: send detected data back to flutter from live feed. need to add code to handle delivery of detector data to the UI. --- .../flutter/plugins/camera/CameraPlugin.java | 52 +++- .../plugins/camera/PreviewImageDelegate.java | 2 +- .../FirebaseMlVisionPlugin.java | 293 ++++++++++-------- .../plugins/firebasemlvision/live/Camera.java | 1 - .../firebasemlvisionexample/MainActivity.java | 8 +- .../lib/src/firebase_vision.dart | 14 +- 6 files changed, 216 insertions(+), 154 deletions(-) diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 693f96d01a1a..5e832154a00e 100644 --- a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -29,6 +29,7 @@ import android.util.Size; import android.util.SparseIntArray; import android.view.Surface; +import android.view.WindowManager; import java.io.File; import java.io.FileOutputStream; @@ -74,14 +75,13 @@ public class CameraPlugin implements MethodCallHandler { private Runnable cameraPermissionContinuation; private boolean requestingPermission; @Nullable private PreviewImageDelegate previewImageDelegate; + private WindowManager windowManager; private CameraPlugin(Registrar registrar, FlutterView view, Activity activity) { - Log.d("ML", "registering camera plugin"); this.registrar = registrar; this.view = view; this.activity = activity; if (activity instanceof PreviewImageDelegate) { - Log.d("ML", "the activity is a PreviewImageDelegate, assigning the image delegate now"); this.previewImageDelegate = (PreviewImageDelegate) activity; } @@ -438,12 +438,10 @@ private void prepareMediaRecorder(String outputFilePath) throws IOException { new ImageReader.OnImageAvailableListener() { @Override public void onImageAvailable(ImageReader reader) { - Log.d("ML", "ImageReader image available..."); Image image = reader.acquireLatestImage(); if (image != null) { - // Log.d("ML", "image was not null"); if (previewImageDelegate != null) { - previewImageDelegate.onImageAvailable(image); + previewImageDelegate.onImageAvailable(image, getRotation()); } image.close(); } @@ -476,14 +474,12 @@ private void open(@Nullable final Result result) { if (result != null) result.error("cameraPermission", "Camera permission not granted", null); } else { try { - Log.d("ML", "opening camera"); startBackgroundThread(); imageReader = ImageReader.newInstance( - captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2); + captureSize.getWidth(), captureSize.getHeight(), ImageFormat.YUV_420_888, 2); imageReaderSurface = imageReader.getSurface(); imageReader.setOnImageAvailableListener(imageAvailable, mBackgroundHandler); - Log.d("ML", "assigned image available listener, boo"); cameraManager.openCamera( cameraName, new CameraDevice.StateCallback() { @@ -576,6 +572,7 @@ private void takePicture(String filePath, @NonNull final Result result) { return; } + // TODO: figure out how we'll use both image readers? do we need two? Log.d("ML", "not setting the take picture listener"); // imageReader.setOnImageAvailableListener( // new ImageReader.OnImageAvailableListener() { @@ -800,4 +797,43 @@ private void dispose() { textureEntry.release(); } } + + private int getRotation() { + if (windowManager == null) { + windowManager = (WindowManager) activity.getSystemService(Context.WINDOW_SERVICE); + } + int degrees = 0; + int rotation = windowManager.getDefaultDisplay().getRotation(); + switch (rotation) { + case Surface.ROTATION_0: + degrees = 0; + break; + case Surface.ROTATION_90: + degrees = 90; + break; + case Surface.ROTATION_180: + degrees = 180; + break; + case Surface.ROTATION_270: + degrees = 270; + break; + default: + Log.e("ML", "Bad rotation value: $rotation"); + } + + try { + int angle; + int displayAngle; // TODO? setDisplayOrientation? + CameraCharacteristics cameraCharacteristics = + cameraManager.getCameraCharacteristics(camera.cameraName); + Integer orientation = cameraCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); + // back-facing + angle = (orientation - degrees + 360) % 360; + displayAngle = angle; + int translatedAngle = angle / 90; + return translatedAngle; // this corresponds to the rotation constants + } catch (CameraAccessException e) { + return 0; + } + } } diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java index f0d3291eb499..ba53aa9fc21b 100644 --- a/packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java @@ -3,5 +3,5 @@ import android.media.Image; public interface PreviewImageDelegate { - void onImageAvailable(Image image); + void onImageAvailable(Image image, int rotation); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index bf802445f3ae..3c37cf04b9a1 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -1,14 +1,22 @@ package io.flutter.plugins.firebasemlvision; import android.app.Activity; -import android.app.Application; import android.media.Image; import android.net.Uri; -import android.os.Bundle; import android.support.annotation.Nullable; import android.util.Log; import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugin.common.MethodChannel.MethodCallHandler; @@ -17,83 +25,58 @@ import io.flutter.plugin.common.PluginRegistry.Registrar; import io.flutter.plugins.camera.PreviewImageDelegate; import io.flutter.plugins.firebasemlvision.live.CameraPreviewImageProvider; -import io.flutter.plugins.firebasemlvision.live.LegacyCamera; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; /** FirebaseMlVisionPlugin */ public class FirebaseMlVisionPlugin implements MethodCallHandler, PreviewImageDelegate { public static final int CAMERA_REQUEST_ID = 928291720; private final Registrar registrar; private final Activity activity; + private EventChannel.EventSink eventSink; + @Nullable + private Detector liveViewDetector; + + private final Detector.OperationFinishedCallback liveDetectorFinishedCallback = + new Detector.OperationFinishedCallback() { + @Override + public void success(Detector detector, Object data) { + Log.d("ML", "detector finished"); + shouldThrottle.set(false); + Map event = new HashMap<>(); + event.put("eventType", "recognized"); + String dataType; + String dataLabel; + if (detector instanceof BarcodeDetector) { + dataType = "barcode"; + dataLabel = "barcodeData"; + } else if (detector instanceof TextDetector) { + dataType = "text"; + dataLabel = "textData"; + } else { + // unsupported live detector + return; + } + event.put("recognitionType", dataType); + event.put(dataLabel, data); + eventSink.success(event); + } + + @Override + public void error(DetectorException e) { + Log.d("ML", "detector error"); + shouldThrottle.set(false); + e.sendError(eventSink); + } + }; - @Nullable private LegacyCamera camera; +// @Nullable private LegacyCamera camera; private FirebaseMlVisionPlugin(Registrar registrar) { this.registrar = registrar; this.activity = registrar.activity(); + registerEventChannel(); if (activity instanceof CameraPreviewImageProvider) { - Log.d("ML", "the activity is a CameraPreviewImageProvider, setting self as a delegate"); ((CameraPreviewImageProvider)activity).setImageDelegate(this); } - - registrar.addRequestPermissionsResultListener(new CameraRequestPermissionsListener()); - - activity - .getApplication() - .registerActivityLifecycleCallbacks( - new Application.ActivityLifecycleCallbacks() { - @Override - public void onActivityCreated(Activity activity, Bundle savedInstanceState) {} - - @Override - public void onActivityStarted(Activity activity) {} - - @Override - public void onActivityResumed(Activity activity) { - // TODO: handle camera permission requesting - // if (camera != null && camera.getRequestingPermission()) { - // camera.setRequestingPermission(false); - // return; - // } - if (activity == FirebaseMlVisionPlugin.this.activity) { - if (camera != null) { - try { - camera.start(null); - } catch (IOException ignored) { - } - } - } - } - - @Override - public void onActivityPaused(Activity activity) { - if (activity == FirebaseMlVisionPlugin.this.activity) { - if (camera != null) { - camera.stop(); - } - } - } - - @Override - public void onActivityStopped(Activity activity) { - if (activity == FirebaseMlVisionPlugin.this.activity) { - if (camera != null) { - camera.stop(); - } - } - } - - @Override - public void onActivitySaveInstanceState(Activity activity, Bundle outState) {} - - @Override - public void onActivityDestroyed(Activity activity) {} - }); } /** Plugin registration. */ @@ -103,78 +86,97 @@ public static void registerWith(Registrar registrar) { channel.setMethodCallHandler(new FirebaseMlVisionPlugin(registrar)); } + private void registerEventChannel() { + new EventChannel( + registrar.messenger(), + "plugins.flutter.io/firebase_ml_vision/liveViewEvents") + .setStreamHandler( + new EventChannel.StreamHandler() { + @Override + public void onListen(Object arguments, EventChannel.EventSink eventSink) { + FirebaseMlVisionPlugin.this.eventSink = eventSink; + } + + @Override + public void onCancel(Object arguments) { + FirebaseMlVisionPlugin.this.eventSink = null; + } + }); + } + @Override public void onMethodCall(MethodCall call, final Result result) { Map options = call.argument("options"); FirebaseVisionImage image; switch (call.method) { - case "init": - if (camera != null) { - camera.stop(); - } - result.success(null); - break; - case "availableCameras": - List> cameras = LegacyCamera.listAvailableCameraDetails(); - result.success(cameras); - break; - case "initialize": - String cameraName = call.argument("cameraName"); - String resolutionPreset = call.argument("resolutionPreset"); - if (camera != null) { - camera.stop(); - } - camera = - new LegacyCamera( - registrar, - resolutionPreset, - Integer.parseInt( - cameraName)); // new Camera(registrar, cameraName, resolutionPreset, result); - camera.setMachineLearningFrameProcessor(TextDetector.instance, options); - try { - camera.start( - new LegacyCamera.OnCameraOpenedCallback() { - @Override - public void onOpened(long textureId, int width, int height) { - Map reply = new HashMap<>(); - reply.put("textureId", textureId); - reply.put("previewWidth", width); - reply.put("previewHeight", height); - result.success(reply); - } - }); - } catch (IOException e) { - result.error("CameraInitializationError", e.getLocalizedMessage(), null); - } - break; - case "dispose": - { - if (camera != null) { - camera.release(); - camera = null; - } - result.success(null); - break; - } +// case "init": +// if (camera != null) { +// camera.stop(); +// } +// result.success(null); +// break; +// case "availableCameras": +// List> cameras = LegacyCamera.listAvailableCameraDetails(); +// result.success(cameras); +// break; +// case "initialize": +// String cameraName = call.argument("cameraName"); +// String resolutionPreset = call.argument("resolutionPreset"); +// if (camera != null) { +// camera.stop(); +// } +// camera = +// new LegacyCamera( +// registrar, +// resolutionPreset, +// Integer.parseInt( +// cameraName)); // new Camera(registrar, cameraName, resolutionPreset, result); +// camera.setMachineLearningFrameProcessor(TextDetector.instance, options); +// try { +// camera.start( +// new LegacyCamera.OnCameraOpenedCallback() { +// @Override +// public void onOpened(long textureId, int width, int height) { +// Map reply = new HashMap<>(); +// reply.put("textureId", textureId); +// reply.put("previewWidth", width); +// reply.put("previewHeight", height); +// result.success(reply); +// } +// }); +// } catch (IOException e) { +// result.error("CameraInitializationError", e.getLocalizedMessage(), null); +// } +// break; +// case "dispose": +// { +// if (camera != null) { +// camera.release(); +// camera = null; +// } +// result.success(null); +// break; +// } case "LiveView#setDetector": - if (camera != null) { +// if (camera != null) { String detectorType = call.argument("detectorType"); - Detector detector; switch (detectorType) { case "text": - detector = TextDetector.instance; + liveViewDetector = TextDetector.instance; break; case "barcode": - detector = BarcodeDetector.instance; + liveViewDetector = BarcodeDetector.instance; break; case "face": - detector = FaceDetector.instance; + liveViewDetector = FaceDetector.instance; break; + case "label": + liveViewDetector = LabelDetector.instance; default: - detector = TextDetector.instance; + liveViewDetector = TextDetector.instance; } - camera.setMachineLearningFrameProcessor(detector, options); - } +// camera.setMachineLearningFrameProcessor(detector, options); +// } result.success(null); break; case "BarcodeDetector#detectInImage": @@ -222,9 +224,40 @@ public void onOpened(long textureId, int width, int height) { } } + private final AtomicBoolean shouldThrottle = new AtomicBoolean(false); + @Override - public void onImageAvailable(Image image) { - Log.d("ML", "got an image"); + public void onImageAvailable(Image image, int rotation) { + if (eventSink == null) return; + if (liveViewDetector == null) return; + if (shouldThrottle.get()) return; + shouldThrottle.set(true); + ByteBuffer imageBuffer = YUV_420_888toNV21(image); + FirebaseVisionImageMetadata metadata = + new FirebaseVisionImageMetadata.Builder() + .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(rotation) + .build(); + FirebaseVisionImage firebaseVisionImage = + FirebaseVisionImage.fromByteBuffer(imageBuffer, metadata); + + liveViewDetector.handleDetection( + firebaseVisionImage, new HashMap(), liveDetectorFinishedCallback); + } + + private static ByteBuffer YUV_420_888toNV21(Image image) { + byte[] nv21; + ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); + ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); + ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); + + int ySize = yBuffer.remaining(); + int uSize = uBuffer.remaining(); + int vSize = vBuffer.remaining(); + + return ByteBuffer.allocate(ySize + uSize + vSize).put(yBuffer).put(vBuffer).put(uBuffer); } private Detector.OperationFinishedCallback handleDetection(final Result result) { @@ -245,18 +278,4 @@ private FirebaseVisionImage filePathToVisionImage(String path) throws IOExceptio File file = new File(path); return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file)); } - - private class CameraRequestPermissionsListener - implements PluginRegistry.RequestPermissionsResultListener { - @Override - public boolean onRequestPermissionsResult(int id, String[] permissions, int[] grantResults) { - // if (id == CAMERA_REQUEST_ID) { - // if (camera != null) { - // camera.continueRequestingPermissions(); - // } - // return true; - // } - return false; - } - } } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index 57a8f91a313f..2b6d636eb0b5 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -366,7 +366,6 @@ private void processImage(Image image) { public void onImageAvailable(ImageReader reader) { Image image = reader.acquireLatestImage(); if (image != null) { - // Log.d("ML", "image was not null"); processImage(image); image.close(); } diff --git a/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java b/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java index 4dd4d41340c1..e53efb693066 100644 --- a/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java +++ b/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java @@ -16,23 +16,19 @@ public class MainActivity extends FlutterActivity implements PreviewImageDelegat @Override protected void onCreate(Bundle savedInstanceState) { - Log.d("ML", "MainActivity created"); super.onCreate(savedInstanceState); GeneratedPluginRegistrant.registerWith(this); } @Override - public void onImageAvailable(Image image) { - Log.d("ML", "got a preview image"); + public void onImageAvailable(Image image, int rotation) { if (previewImageDelegate != null) { - Log.d("ML", "the delegate was not null, sending image to ml for processing"); - previewImageDelegate.onImageAvailable(image); + previewImageDelegate.onImageAvailable(image, rotation); } } @Override public void setImageDelegate(PreviewImageDelegate delegate) { - Log.d("ML", "setting image delegate"); previewImageDelegate = delegate; } } diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index fb9158f5ebad..0b8376b7b14d 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -13,7 +13,19 @@ part of firebase_ml_vision; /// TextDetector textDetector = FirebaseVision.instance.getTextDetector(); /// ``` class FirebaseVision { - FirebaseVision._(); + StreamSubscription _eventSubscription; + + FirebaseVision._() { + _eventSubscription = const EventChannel( + 'plugins.flutter.io/firebase_ml_vision/liveViewEvents') + .receiveBroadcastStream() + .listen(_listener); + } + + void _listener(dynamic event) { + //TODO: handle presentation of recognized items + print(event); + } @visibleForTesting static final MethodChannel channel = From 22e10c2ec2582e5169f3275ffa3a960a6f76b2e3 Mon Sep 17 00:00:00 2001 From: dustin Date: Fri, 27 Jul 2018 00:37:23 -0700 Subject: [PATCH 32/34] fix formatting issues. --- .../flutter/plugins/camera/CameraPlugin.java | 56 ++-- .../FirebaseMlVisionPlugin.java | 257 +++++++++--------- .../firebasemlvisionexample/MainActivity.java | 8 +- .../example/lib/live_preview.dart | 5 +- .../lib/src/firebase_vision.dart | 2 +- 5 files changed, 159 insertions(+), 169 deletions(-) diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 5e832154a00e..59b9a2e7136b 100644 --- a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -30,7 +30,14 @@ import android.util.SparseIntArray; import android.view.Surface; import android.view.WindowManager; - +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugin.common.MethodCall; +import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugin.common.MethodChannel.MethodCallHandler; +import io.flutter.plugin.common.MethodChannel.Result; +import io.flutter.plugin.common.PluginRegistry; +import io.flutter.plugin.common.PluginRegistry.Registrar; +import io.flutter.view.FlutterView; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -43,15 +50,6 @@ import java.util.List; import java.util.Map; -import io.flutter.plugin.common.EventChannel; -import io.flutter.plugin.common.MethodCall; -import io.flutter.plugin.common.MethodChannel; -import io.flutter.plugin.common.MethodChannel.MethodCallHandler; -import io.flutter.plugin.common.MethodChannel.Result; -import io.flutter.plugin.common.PluginRegistry; -import io.flutter.plugin.common.PluginRegistry.Registrar; -import io.flutter.view.FlutterView; - public class CameraPlugin implements MethodCallHandler { private static final int CAMERA_REQUEST_ID = 513469796; @@ -574,24 +572,24 @@ private void takePicture(String filePath, @NonNull final Result result) { // TODO: figure out how we'll use both image readers? do we need two? Log.d("ML", "not setting the take picture listener"); -// imageReader.setOnImageAvailableListener( -// new ImageReader.OnImageAvailableListener() { -// @Override -// public void onImageAvailable(ImageReader reader) { -// try (Image image = reader.acquireLatestImage()) { -// if (previewImageDelegate != null) { -// Log.d("ML", "the preview image delegate is not null, sending the image"); -// previewImageDelegate.onImageAvailable(image); -// } -// ByteBuffer buffer = image.getPlanes()[0].getBuffer(); -// writeToFile(buffer, file); -// result.success(null); -// } catch (IOException e) { -// result.error("IOError", "Failed saving image", null); -// } -// } -// }, -// null); + // imageReader.setOnImageAvailableListener( + // new ImageReader.OnImageAvailableListener() { + // @Override + // public void onImageAvailable(ImageReader reader) { + // try (Image image = reader.acquireLatestImage()) { + // if (previewImageDelegate != null) { + // Log.d("ML", "the preview image delegate is not null, sending the image"); + // previewImageDelegate.onImageAvailable(image); + // } + // ByteBuffer buffer = image.getPlanes()[0].getBuffer(); + // writeToFile(buffer, file); + // result.success(null); + // } catch (IOException e) { + // result.error("IOError", "Failed saving image", null); + // } + // } + // }, + // null); try { final CaptureRequest.Builder captureBuilder = @@ -825,7 +823,7 @@ private int getRotation() { int angle; int displayAngle; // TODO? setDisplayOrientation? CameraCharacteristics cameraCharacteristics = - cameraManager.getCameraCharacteristics(camera.cameraName); + cameraManager.getCameraCharacteristics(camera.cameraName); Integer orientation = cameraCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); // back-facing angle = (orientation - degrees + 360) % 360; diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 3c37cf04b9a1..d4cb4bc1e4ed 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -5,26 +5,22 @@ import android.net.Uri; import android.support.annotation.Nullable; import android.util.Log; - import com.google.firebase.ml.vision.common.FirebaseVisionImage; import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugin.common.MethodChannel.MethodCallHandler; import io.flutter.plugin.common.MethodChannel.Result; -import io.flutter.plugin.common.PluginRegistry; import io.flutter.plugin.common.PluginRegistry.Registrar; import io.flutter.plugins.camera.PreviewImageDelegate; import io.flutter.plugins.firebasemlvision.live.CameraPreviewImageProvider; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; /** FirebaseMlVisionPlugin */ public class FirebaseMlVisionPlugin implements MethodCallHandler, PreviewImageDelegate { @@ -32,50 +28,49 @@ public class FirebaseMlVisionPlugin implements MethodCallHandler, PreviewImageDe private final Registrar registrar; private final Activity activity; private EventChannel.EventSink eventSink; - @Nullable - private Detector liveViewDetector; + @Nullable private Detector liveViewDetector; private final Detector.OperationFinishedCallback liveDetectorFinishedCallback = - new Detector.OperationFinishedCallback() { - @Override - public void success(Detector detector, Object data) { - Log.d("ML", "detector finished"); - shouldThrottle.set(false); - Map event = new HashMap<>(); - event.put("eventType", "recognized"); - String dataType; - String dataLabel; - if (detector instanceof BarcodeDetector) { - dataType = "barcode"; - dataLabel = "barcodeData"; - } else if (detector instanceof TextDetector) { - dataType = "text"; - dataLabel = "textData"; - } else { - // unsupported live detector - return; + new Detector.OperationFinishedCallback() { + @Override + public void success(Detector detector, Object data) { + Log.d("ML", "detector finished"); + shouldThrottle.set(false); + Map event = new HashMap<>(); + event.put("eventType", "recognized"); + String dataType; + String dataLabel; + if (detector instanceof BarcodeDetector) { + dataType = "barcode"; + dataLabel = "barcodeData"; + } else if (detector instanceof TextDetector) { + dataType = "text"; + dataLabel = "textData"; + } else { + // unsupported live detector + return; + } + event.put("recognitionType", dataType); + event.put(dataLabel, data); + eventSink.success(event); } - event.put("recognitionType", dataType); - event.put(dataLabel, data); - eventSink.success(event); - } - @Override - public void error(DetectorException e) { - Log.d("ML", "detector error"); - shouldThrottle.set(false); - e.sendError(eventSink); - } - }; + @Override + public void error(DetectorException e) { + Log.d("ML", "detector error"); + shouldThrottle.set(false); + e.sendError(eventSink); + } + }; -// @Nullable private LegacyCamera camera; + // @Nullable private LegacyCamera camera; private FirebaseMlVisionPlugin(Registrar registrar) { this.registrar = registrar; this.activity = registrar.activity(); registerEventChannel(); if (activity instanceof CameraPreviewImageProvider) { - ((CameraPreviewImageProvider)activity).setImageDelegate(this); + ((CameraPreviewImageProvider) activity).setImageDelegate(this); } } @@ -87,21 +82,19 @@ public static void registerWith(Registrar registrar) { } private void registerEventChannel() { - new EventChannel( - registrar.messenger(), - "plugins.flutter.io/firebase_ml_vision/liveViewEvents") - .setStreamHandler( - new EventChannel.StreamHandler() { - @Override - public void onListen(Object arguments, EventChannel.EventSink eventSink) { - FirebaseMlVisionPlugin.this.eventSink = eventSink; - } + new EventChannel(registrar.messenger(), "plugins.flutter.io/firebase_ml_vision/liveViewEvents") + .setStreamHandler( + new EventChannel.StreamHandler() { + @Override + public void onListen(Object arguments, EventChannel.EventSink eventSink) { + FirebaseMlVisionPlugin.this.eventSink = eventSink; + } - @Override - public void onCancel(Object arguments) { - FirebaseMlVisionPlugin.this.eventSink = null; - } - }); + @Override + public void onCancel(Object arguments) { + FirebaseMlVisionPlugin.this.eventSink = null; + } + }); } @Override @@ -109,74 +102,74 @@ public void onMethodCall(MethodCall call, final Result result) { Map options = call.argument("options"); FirebaseVisionImage image; switch (call.method) { -// case "init": -// if (camera != null) { -// camera.stop(); -// } -// result.success(null); -// break; -// case "availableCameras": -// List> cameras = LegacyCamera.listAvailableCameraDetails(); -// result.success(cameras); -// break; -// case "initialize": -// String cameraName = call.argument("cameraName"); -// String resolutionPreset = call.argument("resolutionPreset"); -// if (camera != null) { -// camera.stop(); -// } -// camera = -// new LegacyCamera( -// registrar, -// resolutionPreset, -// Integer.parseInt( -// cameraName)); // new Camera(registrar, cameraName, resolutionPreset, result); -// camera.setMachineLearningFrameProcessor(TextDetector.instance, options); -// try { -// camera.start( -// new LegacyCamera.OnCameraOpenedCallback() { -// @Override -// public void onOpened(long textureId, int width, int height) { -// Map reply = new HashMap<>(); -// reply.put("textureId", textureId); -// reply.put("previewWidth", width); -// reply.put("previewHeight", height); -// result.success(reply); -// } -// }); -// } catch (IOException e) { -// result.error("CameraInitializationError", e.getLocalizedMessage(), null); -// } -// break; -// case "dispose": -// { -// if (camera != null) { -// camera.release(); -// camera = null; -// } -// result.success(null); -// break; -// } + // case "init": + // if (camera != null) { + // camera.stop(); + // } + // result.success(null); + // break; + // case "availableCameras": + // List> cameras = LegacyCamera.listAvailableCameraDetails(); + // result.success(cameras); + // break; + // case "initialize": + // String cameraName = call.argument("cameraName"); + // String resolutionPreset = call.argument("resolutionPreset"); + // if (camera != null) { + // camera.stop(); + // } + // camera = + // new LegacyCamera( + // registrar, + // resolutionPreset, + // Integer.parseInt( + // cameraName)); // new Camera(registrar, cameraName, resolutionPreset, result); + // camera.setMachineLearningFrameProcessor(TextDetector.instance, options); + // try { + // camera.start( + // new LegacyCamera.OnCameraOpenedCallback() { + // @Override + // public void onOpened(long textureId, int width, int height) { + // Map reply = new HashMap<>(); + // reply.put("textureId", textureId); + // reply.put("previewWidth", width); + // reply.put("previewHeight", height); + // result.success(reply); + // } + // }); + // } catch (IOException e) { + // result.error("CameraInitializationError", e.getLocalizedMessage(), null); + // } + // break; + // case "dispose": + // { + // if (camera != null) { + // camera.release(); + // camera = null; + // } + // result.success(null); + // break; + // } case "LiveView#setDetector": -// if (camera != null) { - String detectorType = call.argument("detectorType"); - switch (detectorType) { - case "text": - liveViewDetector = TextDetector.instance; - break; - case "barcode": - liveViewDetector = BarcodeDetector.instance; - break; - case "face": - liveViewDetector = FaceDetector.instance; - break; - case "label": - liveViewDetector = LabelDetector.instance; - default: - liveViewDetector = TextDetector.instance; - } -// camera.setMachineLearningFrameProcessor(detector, options); -// } + // if (camera != null) { + String detectorType = call.argument("detectorType"); + switch (detectorType) { + case "text": + liveViewDetector = TextDetector.instance; + break; + case "barcode": + liveViewDetector = BarcodeDetector.instance; + break; + case "face": + liveViewDetector = FaceDetector.instance; + break; + case "label": + liveViewDetector = LabelDetector.instance; + default: + liveViewDetector = TextDetector.instance; + } + // camera.setMachineLearningFrameProcessor(detector, options); + // } result.success(null); break; case "BarcodeDetector#detectInImage": @@ -234,17 +227,17 @@ public void onImageAvailable(Image image, int rotation) { shouldThrottle.set(true); ByteBuffer imageBuffer = YUV_420_888toNV21(image); FirebaseVisionImageMetadata metadata = - new FirebaseVisionImageMetadata.Builder() - .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) - .setWidth(image.getWidth()) - .setHeight(image.getHeight()) - .setRotation(rotation) - .build(); + new FirebaseVisionImageMetadata.Builder() + .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(rotation) + .build(); FirebaseVisionImage firebaseVisionImage = - FirebaseVisionImage.fromByteBuffer(imageBuffer, metadata); + FirebaseVisionImage.fromByteBuffer(imageBuffer, metadata); liveViewDetector.handleDetection( - firebaseVisionImage, new HashMap(), liveDetectorFinishedCallback); + firebaseVisionImage, new HashMap(), liveDetectorFinishedCallback); } private static ByteBuffer YUV_420_888toNV21(Image image) { diff --git a/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java b/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java index e53efb693066..86577c43582f 100644 --- a/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java +++ b/packages/firebase_ml_vision/example/android/app/src/main/java/io/flutter/plugins/firebasemlvisionexample/MainActivity.java @@ -3,16 +3,14 @@ import android.media.Image; import android.os.Bundle; import android.support.annotation.Nullable; -import android.util.Log; - import io.flutter.app.FlutterActivity; import io.flutter.plugins.GeneratedPluginRegistrant; import io.flutter.plugins.camera.PreviewImageDelegate; import io.flutter.plugins.firebasemlvision.live.CameraPreviewImageProvider; -public class MainActivity extends FlutterActivity implements PreviewImageDelegate, CameraPreviewImageProvider { - @Nullable - private PreviewImageDelegate previewImageDelegate; +public class MainActivity extends FlutterActivity + implements PreviewImageDelegate, CameraPreviewImageProvider { + @Nullable private PreviewImageDelegate previewImageDelegate; @Override protected void onCreate(Bundle savedInstanceState) { diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 2d0a91e6fae1..3da24e59ef59 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -28,11 +28,12 @@ class LivePreviewState extends State { yield _readyLoadState; } else { yield new LiveViewCameraLoadStateLoading(); - final List cameras = await camera.availableCameras(); + final List cameras = + await camera.availableCameras(); final camera.CameraDescription backCamera = cameras.firstWhere( (camera.CameraDescription cameraDescription) => cameraDescription.lensDirection == - camera.CameraLensDirection.back); + camera.CameraLensDirection.back); if (backCamera != null) { yield new LiveViewCameraLoadStateLoaded(backCamera); try { diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 0b8376b7b14d..b0eb42ff1184 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -17,7 +17,7 @@ class FirebaseVision { FirebaseVision._() { _eventSubscription = const EventChannel( - 'plugins.flutter.io/firebase_ml_vision/liveViewEvents') + 'plugins.flutter.io/firebase_ml_vision/liveViewEvents') .receiveBroadcastStream() .listen(_listener); } From 858b2aab603dbe3d62ac281ffdf0dfecb0f1cad2 Mon Sep 17 00:00:00 2001 From: dustin Date: Fri, 27 Jul 2018 10:24:23 -0700 Subject: [PATCH 33/34] restore normal camera functionality. --- .../flutter/plugins/camera/CameraPlugin.java | 83 ++++++++++--------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 59b9a2e7136b..17d80727a45d 100644 --- a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -30,14 +30,7 @@ import android.util.SparseIntArray; import android.view.Surface; import android.view.WindowManager; -import io.flutter.plugin.common.EventChannel; -import io.flutter.plugin.common.MethodCall; -import io.flutter.plugin.common.MethodChannel; -import io.flutter.plugin.common.MethodChannel.MethodCallHandler; -import io.flutter.plugin.common.MethodChannel.Result; -import io.flutter.plugin.common.PluginRegistry; -import io.flutter.plugin.common.PluginRegistry.Registrar; -import io.flutter.view.FlutterView; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -50,6 +43,15 @@ import java.util.List; import java.util.Map; +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugin.common.MethodCall; +import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugin.common.MethodChannel.MethodCallHandler; +import io.flutter.plugin.common.MethodChannel.Result; +import io.flutter.plugin.common.PluginRegistry; +import io.flutter.plugin.common.PluginRegistry.Registrar; +import io.flutter.view.FlutterView; + public class CameraPlugin implements MethodCallHandler { private static final int CAMERA_REQUEST_ID = 513469796; @@ -248,7 +250,8 @@ private class Camera { private CameraDevice cameraDevice; private CameraCaptureSession cameraCaptureSession; private EventChannel.EventSink eventSink; - private ImageReader imageReader; + private ImageReader previewImageReader; + private ImageReader captureImageReader; private int sensorOrientation; private boolean isFrontFacing; private String cameraName; @@ -431,7 +434,6 @@ private void prepareMediaRecorder(String outputFilePath) throws IOException { private Handler mBackgroundHandler; private HandlerThread mBackgroundThread; - private Surface imageReaderSurface; private final ImageReader.OnImageAvailableListener imageAvailable = new ImageReader.OnImageAvailableListener() { @Override @@ -473,11 +475,14 @@ private void open(@Nullable final Result result) { } else { try { startBackgroundThread(); - imageReader = + // this image reader is used for sending frame data to other packages that need it, such as firebase_ml_vision + previewImageReader = + ImageReader.newInstance( + previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 4); + captureImageReader = ImageReader.newInstance( - captureSize.getWidth(), captureSize.getHeight(), ImageFormat.YUV_420_888, 2); - imageReaderSurface = imageReader.getSurface(); - imageReader.setOnImageAvailableListener(imageAvailable, mBackgroundHandler); + captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2); + previewImageReader.setOnImageAvailableListener(imageAvailable, mBackgroundHandler); cameraManager.openCamera( cameraName, new CameraDevice.StateCallback() { @@ -570,31 +575,25 @@ private void takePicture(String filePath, @NonNull final Result result) { return; } - // TODO: figure out how we'll use both image readers? do we need two? - Log.d("ML", "not setting the take picture listener"); - // imageReader.setOnImageAvailableListener( - // new ImageReader.OnImageAvailableListener() { - // @Override - // public void onImageAvailable(ImageReader reader) { - // try (Image image = reader.acquireLatestImage()) { - // if (previewImageDelegate != null) { - // Log.d("ML", "the preview image delegate is not null, sending the image"); - // previewImageDelegate.onImageAvailable(image); - // } - // ByteBuffer buffer = image.getPlanes()[0].getBuffer(); - // writeToFile(buffer, file); - // result.success(null); - // } catch (IOException e) { - // result.error("IOError", "Failed saving image", null); - // } - // } - // }, - // null); + captureImageReader.setOnImageAvailableListener( + new ImageReader.OnImageAvailableListener() { + @Override + public void onImageAvailable(ImageReader reader) { + try (Image image = reader.acquireLatestImage()) { + ByteBuffer buffer = image.getPlanes()[0].getBuffer(); + writeToFile(buffer, file); + result.success(null); + } catch (IOException e) { + result.error("IOError", "Failed saving image", null); + } + } + }, + null); try { final CaptureRequest.Builder captureBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE); - captureBuilder.addTarget(imageReader.getSurface()); + captureBuilder.addTarget(captureImageReader.getSurface()); int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation(); int displayOrientation = ORIENTATIONS.get(displayRotation); if (isFrontFacing) displayOrientation = -displayOrientation; @@ -724,8 +723,12 @@ private void startPreview() throws CameraAccessException { surfaces.add(previewSurface); captureRequestBuilder.addTarget(previewSurface); - surfaces.add(imageReaderSurface); - captureRequestBuilder.addTarget(imageReaderSurface); + surfaces.add(captureImageReader.getSurface()); + + // This is so we can send sample frames out to other plugins that need a live feed of frames + Surface previewImageReaderSurface = previewImageReader.getSurface(); + surfaces.add(previewImageReaderSurface); + captureRequestBuilder.addTarget(previewImageReaderSurface); cameraDevice.createCaptureSession( surfaces, @@ -778,9 +781,9 @@ private void close() { cameraDevice.close(); cameraDevice = null; } - if (imageReader != null) { - imageReader.close(); - imageReader = null; + if (previewImageReader != null) { + previewImageReader.close(); + previewImageReader = null; } if (mediaRecorder != null) { mediaRecorder.reset(); From e91181f009a3dbf9ff74f1566afaef80030f6c5d Mon Sep 17 00:00:00 2001 From: dustin Date: Thu, 9 Aug 2018 12:01:55 -0700 Subject: [PATCH 34/34] present detection boundaries in Flutter from camera plugin integration --- .../firebasemlvision/BarcodeDetector.java | 12 +- .../plugins/firebasemlvision/Detector.java | 17 ++- .../firebasemlvision/DetectorException.java | 5 +- .../firebasemlvision/FaceDetector.java | 11 +- .../FirebaseMlVisionPlugin.java | 116 +++++++----------- .../firebasemlvision/LabelDetector.java | 11 +- .../firebasemlvision/TextDetector.java | 11 +- .../plugins/firebasemlvision/live/Camera.java | 34 +++-- .../firebasemlvision/live/LegacyCamera.java | 4 +- .../example/lib/detector_painters.dart | 21 ++-- .../example/lib/live_preview.dart | 32 ++++- .../lib/firebase_ml_vision.dart | 3 + .../lib/src/barcode_detector.dart | 13 +- .../lib/src/face_detector.dart | 7 +- .../lib/src/firebase_vision.dart | 55 +++++++-- .../lib/src/label_detector.dart | 13 +- .../firebase_ml_vision/lib/src/live_view.dart | 4 +- .../lib/src/live_view_detection_result.dart | 42 +++++++ .../lib/src/vision_options.dart | 5 + 19 files changed, 271 insertions(+), 145 deletions(-) create mode 100644 packages/firebase_ml_vision/lib/src/live_view_detection_result.dart create mode 100644 packages/firebase_ml_vision/lib/src/vision_options.dart diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java index 226e1340a72d..060c96b09810 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java @@ -3,6 +3,8 @@ import android.graphics.Point; import android.graphics.Rect; import android.support.annotation.NonNull; +import android.util.Size; + import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; @@ -10,6 +12,7 @@ import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector; import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetectorOptions; import com.google.firebase.ml.vision.common.FirebaseVisionImage; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -22,9 +25,10 @@ private BarcodeDetector() {} @Override void processImage( - FirebaseVisionImage image, - Map options, - final OperationFinishedCallback finishedCallback) { + final FirebaseVisionImage image, + final Size imageSize, + Map options, + final OperationFinishedCallback finishedCallback) { FirebaseVisionBarcodeDetector detector = FirebaseVision.getInstance().getVisionBarcodeDetector(parseOptions(options)); @@ -212,7 +216,7 @@ public void onSuccess(List firebaseVisionBarcodes) { barcodes.add(barcodeMap); } - finishedCallback.success(BarcodeDetector.this, barcodes); + finishedCallback.success(BarcodeDetector.this, barcodes, imageSize); } }) .addOnFailureListener( diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java index eb94a6ce26b7..a450dcac0e02 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java @@ -1,5 +1,7 @@ package io.flutter.plugins.firebasemlvision; +import android.util.Size; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -7,7 +9,7 @@ public abstract class Detector { public interface OperationFinishedCallback { - void success(Detector detector, Object data); + void success(Detector detector, Object data, Size size); void error(DetectorException e); } @@ -15,20 +17,22 @@ public interface OperationFinishedCallback { private final AtomicBoolean shouldThrottle = new AtomicBoolean(false); public void handleDetection( - FirebaseVisionImage image, - Map options, - final OperationFinishedCallback finishedCallback) { + final FirebaseVisionImage image, + final Size imageSize, + Map options, + final OperationFinishedCallback finishedCallback) { if (shouldThrottle.get()) { return; } processImage( image, + imageSize, options, new OperationFinishedCallback() { @Override - public void success(Detector detector, Object data) { + public void success(Detector detector, Object data, Size size) { shouldThrottle.set(false); - finishedCallback.success(detector, data); + finishedCallback.success(detector, data, size); } @Override @@ -45,6 +49,7 @@ public void error(DetectorException e) { abstract void processImage( FirebaseVisionImage image, + Size imageSize, Map options, OperationFinishedCallback finishedCallback); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/DetectorException.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/DetectorException.java index fd2779b07eb1..9b2b3f47c71c 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/DetectorException.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/DetectorException.java @@ -1,5 +1,8 @@ package io.flutter.plugins.firebasemlvision; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; @@ -16,7 +19,7 @@ public DetectorException( this.exceptionData = exceptionData; } - public void sendError(EventChannel.EventSink eventSink) { + public void sendError(@NonNull EventChannel.EventSink eventSink) { eventSink.error(detectorExceptionType, detectorExceptionDescription, exceptionData); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java index f102007d92e8..8ae9bb69528a 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java @@ -1,6 +1,8 @@ package io.flutter.plugins.firebasemlvision; import android.support.annotation.NonNull; +import android.util.Size; + import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; @@ -21,9 +23,10 @@ private FaceDetector() {} @Override void processImage( - FirebaseVisionImage image, - Map options, - final OperationFinishedCallback finishedCallback) { + FirebaseVisionImage image, + final Size imageSize, + Map options, + final OperationFinishedCallback finishedCallback) { FirebaseVisionFaceDetector detector; if (options == null) { detector = FirebaseVision.getInstance().getVisionFaceDetector(); @@ -72,7 +75,7 @@ public void onSuccess(List firebaseVisionFaces) { faces.add(faceData); } - finishedCallback.success(FaceDetector.this, faces); + finishedCallback.success(FaceDetector.this, faces, imageSize); } }) .addOnFailureListener( diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index d4cb4bc1e4ed..29d438662da2 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -5,8 +5,18 @@ import android.net.Uri; import android.support.annotation.Nullable; import android.util.Log; +import android.util.Size; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; @@ -15,43 +25,45 @@ import io.flutter.plugin.common.PluginRegistry.Registrar; import io.flutter.plugins.camera.PreviewImageDelegate; import io.flutter.plugins.firebasemlvision.live.CameraPreviewImageProvider; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; /** FirebaseMlVisionPlugin */ public class FirebaseMlVisionPlugin implements MethodCallHandler, PreviewImageDelegate { public static final int CAMERA_REQUEST_ID = 928291720; private final Registrar registrar; private final Activity activity; - private EventChannel.EventSink eventSink; + @Nullable private EventChannel.EventSink eventSink; @Nullable private Detector liveViewDetector; + @Nullable private Map liveViewOptions; private final Detector.OperationFinishedCallback liveDetectorFinishedCallback = new Detector.OperationFinishedCallback() { @Override - public void success(Detector detector, Object data) { + public void success(Detector detector, Object data, Size imageSize) { + if (eventSink == null) return; Log.d("ML", "detector finished"); shouldThrottle.set(false); Map event = new HashMap<>(); - event.put("eventType", "recognized"); + event.put("eventType", "detection"); String dataType; String dataLabel; if (detector instanceof BarcodeDetector) { dataType = "barcode"; - dataLabel = "barcodeData"; } else if (detector instanceof TextDetector) { dataType = "text"; - dataLabel = "textData"; + } else if (detector instanceof FaceDetector) { + dataType = "face"; + } else if (detector instanceof LabelDetector) { + dataType = "label"; } else { - // unsupported live detector + // unsupported detector return; } - event.put("recognitionType", dataType); - event.put(dataLabel, data); + event.put("detectionType", dataType); + event.put("data", data); + Map sizeMap = new HashMap<>(); + sizeMap.put("width", imageSize.getWidth()); + sizeMap.put("height", imageSize.getHeight()); + event.put("imageSize", sizeMap); eventSink.success(event); } @@ -59,7 +71,9 @@ public void success(Detector detector, Object data) { public void error(DetectorException e) { Log.d("ML", "detector error"); shouldThrottle.set(false); - e.sendError(eventSink); + if (eventSink != null) { + e.sendError(eventSink); + } } }; @@ -102,56 +116,8 @@ public void onMethodCall(MethodCall call, final Result result) { Map options = call.argument("options"); FirebaseVisionImage image; switch (call.method) { - // case "init": - // if (camera != null) { - // camera.stop(); - // } - // result.success(null); - // break; - // case "availableCameras": - // List> cameras = LegacyCamera.listAvailableCameraDetails(); - // result.success(cameras); - // break; - // case "initialize": - // String cameraName = call.argument("cameraName"); - // String resolutionPreset = call.argument("resolutionPreset"); - // if (camera != null) { - // camera.stop(); - // } - // camera = - // new LegacyCamera( - // registrar, - // resolutionPreset, - // Integer.parseInt( - // cameraName)); // new Camera(registrar, cameraName, resolutionPreset, result); - // camera.setMachineLearningFrameProcessor(TextDetector.instance, options); - // try { - // camera.start( - // new LegacyCamera.OnCameraOpenedCallback() { - // @Override - // public void onOpened(long textureId, int width, int height) { - // Map reply = new HashMap<>(); - // reply.put("textureId", textureId); - // reply.put("previewWidth", width); - // reply.put("previewHeight", height); - // result.success(reply); - // } - // }); - // } catch (IOException e) { - // result.error("CameraInitializationError", e.getLocalizedMessage(), null); - // } - // break; - // case "dispose": - // { - // if (camera != null) { - // camera.release(); - // camera = null; - // } - // result.success(null); - // break; - // } case "LiveView#setDetector": - // if (camera != null) { + liveViewOptions = options; String detectorType = call.argument("detectorType"); switch (detectorType) { case "text": @@ -168,14 +134,13 @@ public void onMethodCall(MethodCall call, final Result result) { default: liveViewDetector = TextDetector.instance; } - // camera.setMachineLearningFrameProcessor(detector, options); - // } result.success(null); break; case "BarcodeDetector#detectInImage": try { image = filePathToVisionImage((String) call.argument("path")); - BarcodeDetector.instance.handleDetection(image, options, handleDetection(result)); + BarcodeDetector.instance.handleDetection( + image, new Size(0, 0), options, handleDetection(result)); } catch (IOException e) { result.error("barcodeDetectorIOError", e.getLocalizedMessage(), null); } catch (Exception e) { @@ -185,7 +150,8 @@ public void onMethodCall(MethodCall call, final Result result) { case "FaceDetector#detectInImage": try { image = filePathToVisionImage((String) call.argument("path")); - FaceDetector.instance.handleDetection(image, options, handleDetection(result)); + FaceDetector.instance.handleDetection( + image, new Size(0, 0), options, handleDetection(result)); } catch (IOException e) { result.error("faceDetectorIOError", e.getLocalizedMessage(), null); } catch (Exception e) { @@ -195,7 +161,8 @@ public void onMethodCall(MethodCall call, final Result result) { case "LabelDetector#detectInImage": try { image = filePathToVisionImage((String) call.argument("path")); - LabelDetector.instance.handleDetection(image, options, handleDetection(result)); + LabelDetector.instance.handleDetection( + image, new Size(0, 0), options, handleDetection(result)); } catch (IOException e) { result.error("labelDetectorIOError", e.getLocalizedMessage(), null); } catch (Exception e) { @@ -205,7 +172,8 @@ public void onMethodCall(MethodCall call, final Result result) { case "TextDetector#detectInImage": try { image = filePathToVisionImage((String) call.argument("path")); - TextDetector.instance.handleDetection(image, options, handleDetection(result)); + TextDetector.instance.handleDetection( + image, new Size(0, 0), options, handleDetection(result)); } catch (IOException e) { result.error("textDetectorIOError", e.getLocalizedMessage(), null); } catch (Exception e) { @@ -237,7 +205,10 @@ public void onImageAvailable(Image image, int rotation) { FirebaseVisionImage.fromByteBuffer(imageBuffer, metadata); liveViewDetector.handleDetection( - firebaseVisionImage, new HashMap(), liveDetectorFinishedCallback); + firebaseVisionImage, + new Size(image.getWidth(), image.getHeight()), + liveViewOptions, + liveDetectorFinishedCallback); } private static ByteBuffer YUV_420_888toNV21(Image image) { @@ -256,7 +227,8 @@ private static ByteBuffer YUV_420_888toNV21(Image image) { private Detector.OperationFinishedCallback handleDetection(final Result result) { return new Detector.OperationFinishedCallback() { @Override - public void success(Detector detector, Object data) { + public void success( + Detector detector, Object data, Size imageSize /*ignore size for file image detection*/) { result.success(data); } diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java index 632ed9dc90b4..59c7da84636a 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/LabelDetector.java @@ -1,6 +1,8 @@ package io.flutter.plugins.firebasemlvision; import android.support.annotation.NonNull; +import android.util.Size; + import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; @@ -25,9 +27,10 @@ private FirebaseVisionLabelDetectorOptions parseOptions(Map opti @Override void processImage( - FirebaseVisionImage image, - Map options, - final OperationFinishedCallback finishedCallback) { + FirebaseVisionImage image, + final Size imageSize, + Map options, + final OperationFinishedCallback finishedCallback) { FirebaseVisionLabelDetector detector = FirebaseVision.getInstance().getVisionLabelDetector(parseOptions(options)); detector @@ -46,7 +49,7 @@ public void onSuccess(List firebaseVisionLabels) { labels.add(labelData); } - finishedCallback.success(LabelDetector.this, labels); + finishedCallback.success(LabelDetector.this, labels, imageSize); } }) .addOnFailureListener( diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java index aa72e95a49ba..0a5b9733504a 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextDetector.java @@ -3,6 +3,8 @@ import android.graphics.Point; import android.graphics.Rect; import android.support.annotation.NonNull; +import android.util.Size; + import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; @@ -22,9 +24,10 @@ private TextDetector() {} @Override void processImage( - FirebaseVisionImage image, - Map options, - final OperationFinishedCallback finishedCallback) { + FirebaseVisionImage image, + final Size imageSize, + Map options, + final OperationFinishedCallback finishedCallback) { if (textDetector == null) textDetector = FirebaseVision.getInstance().getVisionTextDetector(); textDetector .detectInImage(image) @@ -60,7 +63,7 @@ public void onSuccess(FirebaseVisionText firebaseVisionText) { blockData.put("lines", lines); blocks.add(blockData); } - finishedCallback.success(TextDetector.this, blocks); + finishedCallback.success(TextDetector.this, blocks, imageSize); } }) .addOnFailureListener( diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java index 2b6d636eb0b5..6984f2f972f1 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/Camera.java @@ -1,7 +1,5 @@ package io.flutter.plugins.firebasemlvision.live; -import static io.flutter.plugins.firebasemlvision.FirebaseMlVisionPlugin.CAMERA_REQUEST_ID; - import android.Manifest; import android.annotation.TargetApi; import android.app.Activity; @@ -31,16 +29,10 @@ import android.util.SparseIntArray; import android.view.Surface; import android.view.WindowManager; + import com.google.firebase.ml.vision.common.FirebaseVisionImage; import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; -import io.flutter.plugin.common.EventChannel; -import io.flutter.plugin.common.MethodChannel; -import io.flutter.plugin.common.PluginRegistry; -import io.flutter.plugins.firebasemlvision.BarcodeDetector; -import io.flutter.plugins.firebasemlvision.Detector; -import io.flutter.plugins.firebasemlvision.DetectorException; -import io.flutter.plugins.firebasemlvision.TextDetector; -import io.flutter.view.FlutterView; + import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -51,6 +43,17 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugin.common.PluginRegistry; +import io.flutter.plugins.firebasemlvision.BarcodeDetector; +import io.flutter.plugins.firebasemlvision.Detector; +import io.flutter.plugins.firebasemlvision.DetectorException; +import io.flutter.plugins.firebasemlvision.TextDetector; +import io.flutter.view.FlutterView; + +import static io.flutter.plugins.firebasemlvision.FirebaseMlVisionPlugin.CAMERA_REQUEST_ID; + @SuppressWarnings("WeakerAccess") @TargetApi(Build.VERSION_CODES.LOLLIPOP) class Camera { @@ -87,7 +90,7 @@ class Camera { private final Detector.OperationFinishedCallback liveDetectorFinishedCallback = new Detector.OperationFinishedCallback() { @Override - public void success(Detector detector, Object data) { + public void success(Detector detector, Object data, Size imageSize) { shouldThrottle.set(false); Map event = new HashMap<>(); event.put("eventType", "recognized"); @@ -104,6 +107,10 @@ public void success(Detector detector, Object data) { return; } event.put("recognitionType", dataType); + Map sizeMap = new HashMap<>(); + sizeMap.put("width", imageSize.getWidth()); + sizeMap.put("height", imageSize.getHeight()); + event.put("imageSize", sizeMap); event.put(dataLabel, data); eventSink.success(event); } @@ -357,7 +364,10 @@ private void processImage(Image image) { FirebaseVisionImage.fromByteBuffer(imageBuffer, metadata); currentDetector.handleDetection( - firebaseVisionImage, new HashMap(), liveDetectorFinishedCallback); + firebaseVisionImage, + new Size(image.getWidth(), image.getHeight()), + new HashMap(), + liveDetectorFinishedCallback); } private final ImageReader.OnImageAvailableListener imageAvailable = diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java index d087b6fbcb5a..23577492c585 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/live/LegacyCamera.java @@ -134,7 +134,7 @@ public interface OnCameraOpenedCallback { private final Detector.OperationFinishedCallback liveDetectorFinishedCallback = new Detector.OperationFinishedCallback() { @Override - public void success(Detector detector, Object data) { + public void success(Detector detector, Object data, android.util.Size size) { Map event = new HashMap<>(); event.put("eventType", "detection"); String dataType; @@ -791,7 +791,7 @@ public void run() { .setRotation(rotation) .build(); FirebaseVisionImage image = FirebaseVisionImage.fromByteBuffer(data, metadata); - detector.handleDetection(image, detectorOptions, liveDetectorFinishedCallback); + detector.handleDetection(image, new android.util.Size(previewSize.getWidth(), previewSize.getHeight()), detectorOptions, liveDetectorFinishedCallback); } } catch (Throwable t) { Log.e(TAG, "Exception thrown from receiver.", t); diff --git a/packages/firebase_ml_vision/example/lib/detector_painters.dart b/packages/firebase_ml_vision/example/lib/detector_painters.dart index 36ae3297a05c..b8a89d8214eb 100644 --- a/packages/firebase_ml_vision/example/lib/detector_painters.dart +++ b/packages/firebase_ml_vision/example/lib/detector_painters.dart @@ -7,18 +7,19 @@ import 'dart:ui' as ui; import 'package:firebase_ml_vision/firebase_ml_vision.dart'; import 'package:flutter/material.dart'; -CustomPaint customPaintForResults( - Size imageSize, LiveViewDetectionList results) { +CustomPaint customPaintForResults(LiveViewDetectionResult result) { CustomPainter painter; - if (results is LiveViewBarcodeDetectionList) { - painter = new BarcodeDetectorPainter(imageSize, results.data); - } else if (results is LiveViewTextDetectionList) { - painter = new TextDetectorPainter(imageSize, results.data); - } else if (results is LiveViewFaceDetectionList) { - painter = new FaceDetectorPainter(imageSize, results.data); - } else if (results is LiveViewLabelDetectionList) { - painter = new LabelDetectorPainter(imageSize, results.data); + if (result is LiveViewBarcodeDetectionResult) { + painter = new BarcodeDetectorPainter(result.size, result.data); + } else if (result is LiveViewTextDetectionResult) { + print("painting text"); + painter = new TextDetectorPainter(result.size, result.data); + } else if (result is LiveViewFaceDetectionResult) { + painter = new FaceDetectorPainter(result.size, result.data); + } else if (result is LiveViewLabelDetectionResult) { + painter = new LabelDetectorPainter(result.size, result.data); } else { + print("unknown painter"); painter = null; } diff --git a/packages/firebase_ml_vision/example/lib/live_preview.dart b/packages/firebase_ml_vision/example/lib/live_preview.dart index 3da24e59ef59..427bd08a02f8 100644 --- a/packages/firebase_ml_vision/example/lib/live_preview.dart +++ b/packages/firebase_ml_vision/example/lib/live_preview.dart @@ -1,9 +1,9 @@ import 'dart:async'; +import 'package:camera/camera.dart' as camera; import 'package:firebase_ml_vision/firebase_ml_vision.dart'; import 'package:firebase_ml_vision_example/detector_painters.dart'; import 'package:flutter/material.dart'; -import 'package:camera/camera.dart' as camera; class LivePreview extends StatefulWidget { final FirebaseVisionDetectorType detector; @@ -60,7 +60,15 @@ class LivePreviewState extends State { } Future setLiveViewDetector() async { - FirebaseVision.instance.setLiveViewDetector(widget.detector); + VisionOptions options; + if (widget.detector == FirebaseVisionDetectorType.barcode) { + options = const BarcodeDetectorOptions(); + } else if (widget.detector == FirebaseVisionDetectorType.label) { + options = const LabelDetectorOptions(); + } else if (widget.detector == FirebaseVisionDetectorType.face) { + options = const FaceDetectorOptions(); + } + FirebaseVision.instance.setLiveViewDetector(widget.detector, options); } @override @@ -89,7 +97,25 @@ class LivePreviewState extends State { } return new AspectRatio( aspectRatio: _readyLoadState.controller.value.aspectRatio, - child: new camera.CameraPreview(loadState.controller), + child: Stack(children: [ + new camera.CameraPreview(loadState.controller), + Container( + constraints: const BoxConstraints.expand(), + child: StreamBuilder( + builder: (BuildContext context, + AsyncSnapshot snapshot) { + print("update: ${snapshot}"); + if (snapshot == null || snapshot.data == null) { + return Text("No DATA!!!"); + } + print("size: ${snapshot.data.size}"); + print("data: ${snapshot.data.data}"); + return customPaintForResults(snapshot.data); + }, + stream: FirebaseVision.instance.liveViewStream, + ), + ) + ]), ); } else if (loadState is LiveViewCameraLoadStateFailed) { return new Text("error loading camera ${loadState.errorMessage}"); diff --git a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart index 6627bd24c2b7..a9852bfeb7f0 100644 --- a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart +++ b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart @@ -7,6 +7,7 @@ library firebase_ml_vision; import 'dart:async'; import 'dart:io'; import 'dart:math'; +import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; @@ -18,3 +19,5 @@ part 'src/firebase_vision.dart'; part 'src/label_detector.dart'; part 'src/text_detector.dart'; part 'src/live_view.dart'; +part 'src/live_view_detection_result.dart'; +part 'src/vision_options.dart'; diff --git a/packages/firebase_ml_vision/lib/src/barcode_detector.dart b/packages/firebase_ml_vision/lib/src/barcode_detector.dart index b0a8725be077..c739be72cc7d 100644 --- a/packages/firebase_ml_vision/lib/src/barcode_detector.dart +++ b/packages/firebase_ml_vision/lib/src/barcode_detector.dart @@ -190,9 +190,7 @@ class BarcodeDetector extends FirebaseVisionDetector { 'BarcodeDetector#detectInImage', { 'path': visionImage.imageFile.path, - 'options': { - 'barcodeFormats': options.barcodeFormats.value, - }, + 'options': options.toMap(), }, ); @@ -215,10 +213,17 @@ class BarcodeDetector extends FirebaseVisionDetector { /// final BarcodeDetectorOptions options = /// BarcodeDetectorOptions(barcodeFormats: BarcodeFormat.aztec | BarcodeFormat.ean8); /// ``` -class BarcodeDetectorOptions { +class BarcodeDetectorOptions implements VisionOptions { const BarcodeDetectorOptions({this.barcodeFormats = BarcodeFormat.all}); final BarcodeFormat barcodeFormats; + + @override + Map toMap() { + return { + 'barcodeFormats': barcodeFormats.value, + }; + } } /// Represents a single recognized barcode and its value. diff --git a/packages/firebase_ml_vision/lib/src/face_detector.dart b/packages/firebase_ml_vision/lib/src/face_detector.dart index 942161653dab..db626190ad06 100644 --- a/packages/firebase_ml_vision/lib/src/face_detector.dart +++ b/packages/firebase_ml_vision/lib/src/face_detector.dart @@ -45,7 +45,7 @@ class FaceDetector extends FirebaseVisionDetector { 'FaceDetector#detectInImage', { 'path': visionImage.imageFile.path, - 'options': options.optionsMap, + 'options': options.toMap(), }, ); @@ -62,7 +62,7 @@ class FaceDetector extends FirebaseVisionDetector { /// /// Used to configure features such as classification, face tracking, speed, /// etc. -class FaceDetectorOptions { +class FaceDetectorOptions implements VisionOptions { /// Constructor for [FaceDetectorOptions]. /// /// The parameter minFaceValue must be between 0.0 and 1.0, inclusive. @@ -99,7 +99,8 @@ class FaceDetectorOptions { /// Option for controlling additional accuracy / speed trade-offs. final FaceDetectorMode mode; - Map get optionsMap { + @override + Map toMap() { return { 'enableClassification': enableClassification, 'enableLandmarks': enableLandmarks, diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 37c9b98aca16..3fd085ef1bae 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -13,18 +13,53 @@ part of firebase_ml_vision; /// TextDetector textDetector = FirebaseVision.instance.getTextDetector(); /// ``` class FirebaseVision { - StreamSubscription _eventSubscription; + Stream liveViewStream; FirebaseVision._() { - _eventSubscription = const EventChannel( + liveViewStream = const EventChannel( 'plugins.flutter.io/firebase_ml_vision/liveViewEvents') .receiveBroadcastStream() - .listen(_listener); - } - - void _listener(dynamic event) { - //TODO: handle presentation of recognized items - print(event); + .where((dynamic event) => event['eventType'] == 'detection') + .map((dynamic event) { + print("mapping!"); + + // get the image size + final Map sizeMap = event['imageSize']; + int width = sizeMap['width']; + int height = sizeMap['height']; + final imageSize = Size(width.toDouble(), height.toDouble()); + + // get the data + final List reply = event['data']; + // get the data type + final String detectionType = event['detectionType']; + if (detectionType == "barcode") { + final List barcodes = []; + reply.forEach((dynamic barcodeMap) { + barcodes.add(new Barcode(barcodeMap)); + }); + return new LiveViewBarcodeDetectionResult(barcodes, imageSize); + } else if (detectionType == "text") { + final List texts = []; + reply.forEach((dynamic block) { + texts.add(TextBlock.fromBlockData(block)); + }); + return new LiveViewTextDetectionResult(texts, imageSize); + } else if (detectionType == "face") { + final List faces = []; + reply.forEach((dynamic f) { + faces.add(new Face(f)); + }); + return new LiveViewFaceDetectionResult(faces, imageSize); + } else if (detectionType == "label") { + final List