WIP: Live MLKit vision detection using the camera plugin

packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java

@@ -21,19 +21,16 @@
 import android.media.MediaRecorder;
 import android.os.Build;
 import android.os.Bundle;
+import android.os.Handler;
+import android.os.HandlerThread;
 import android.support.annotation.NonNull;
 import android.support.annotation.Nullable;
+import android.util.Log;
 import android.util.Size;
 import android.util.SparseIntArray;
 import android.view.Surface;
-import io.flutter.plugin.common.EventChannel;
-import io.flutter.plugin.common.MethodCall;
-import io.flutter.plugin.common.MethodChannel;
-import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
-import io.flutter.plugin.common.MethodChannel.Result;
-import io.flutter.plugin.common.PluginRegistry;
-import io.flutter.plugin.common.PluginRegistry.Registrar;
-import io.flutter.view.FlutterView;
+import android.view.WindowManager;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -46,6 +43,15 @@
 import java.util.List;
 import java.util.Map;
 
+import io.flutter.plugin.common.EventChannel;
+import io.flutter.plugin.common.MethodCall;
+import io.flutter.plugin.common.MethodChannel;
+import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
+import io.flutter.plugin.common.MethodChannel.Result;
+import io.flutter.plugin.common.PluginRegistry;
+import io.flutter.plugin.common.PluginRegistry.Registrar;
+import io.flutter.view.FlutterView;
+
 public class CameraPlugin implements MethodCallHandler {
 
   private static final int CAMERA_REQUEST_ID = 513469796;
@@ -68,11 +74,16 @@ public class CameraPlugin implements MethodCallHandler {
   // The code to run after requesting camera permissions.
   private Runnable cameraPermissionContinuation;
   private boolean requestingPermission;
+  @Nullable private PreviewImageDelegate previewImageDelegate;
+  private WindowManager windowManager;
 
   private CameraPlugin(Registrar registrar, FlutterView view, Activity activity) {
     this.registrar = registrar;
     this.view = view;
     this.activity = activity;
+    if (activity instanceof PreviewImageDelegate) {
+      this.previewImageDelegate = (PreviewImageDelegate) activity;
+    }
 
     registrar.addRequestPermissionsResultListener(new CameraRequestPermissionsListener());
 
@@ -239,7 +250,8 @@ private class Camera {
     private CameraDevice cameraDevice;
     private CameraCaptureSession cameraCaptureSession;
     private EventChannel.EventSink eventSink;
-    private ImageReader imageReader;
+    private ImageReader previewImageReader;
+    private ImageReader captureImageReader;
     private int sensorOrientation;
     private boolean isFrontFacing;
     private String cameraName;
@@ -376,7 +388,8 @@ private void computeBestPreviewAndRecordingSize(
       } else {
         previewSize = goodEnough.get(0);
 
-        // Video capture size should not be greater than 1080 because MediaRecorder cannot handle higher resolutions.
+        // Video capture size should not be greater than 1080 because MediaRecorder cannot handle
+        // higher resolutions.
         videoSize = goodEnough.get(0);
         for (int i = goodEnough.size() - 1; i >= 0; i--) {
           if (goodEnough.get(i).getHeight() <= 1080) {
@@ -419,14 +432,57 @@ private void prepareMediaRecorder(String outputFilePath) throws IOException {
       mediaRecorder.prepare();
     }
 
+    private Handler mBackgroundHandler;
+    private HandlerThread mBackgroundThread;
+    private final ImageReader.OnImageAvailableListener imageAvailable =
+        new ImageReader.OnImageAvailableListener() {
+          @Override
+          public void onImageAvailable(ImageReader reader) {
+            Image image = reader.acquireLatestImage();
+            if (image != null) {
+              if (previewImageDelegate != null) {
+                previewImageDelegate.onImageAvailable(image, getRotation());
+              }
+              image.close();
+            }
+          }
+        };
+
+    /** Starts a background thread and its {@link Handler}. */
+    private void startBackgroundThread() {
+      mBackgroundThread = new HandlerThread("CameraBackground");
+      mBackgroundThread.start();
+      mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
+    }
+
+    /** Stops the background thread and its {@link Handler}. */
+    private void stopBackgroundThread() {
+      if (mBackgroundThread != null) {
+        mBackgroundThread.quitSafely();
+        try {
+          mBackgroundThread.join();
+          mBackgroundThread = null;
+          mBackgroundHandler = null;
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+      }
+    }
+
     private void open(@Nullable final Result result) {
       if (!hasCameraPermission()) {
         if (result != null) result.error("cameraPermission", "Camera permission not granted", null);
       } else {
         try {
-          imageReader =
+          startBackgroundThread();
+          // this image reader is used for sending frame data to other packages that need it, such as firebase_ml_vision
+          previewImageReader =
+              ImageReader.newInstance(
+                  previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 4);
+          captureImageReader =
               ImageReader.newInstance(
                   captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2);
+          previewImageReader.setOnImageAvailableListener(imageAvailable, mBackgroundHandler);
           cameraManager.openCamera(
               cameraName,
               new CameraDevice.StateCallback() {
@@ -519,7 +575,7 @@ private void takePicture(String filePath, @NonNull final Result result) {
         return;
       }
 
-      imageReader.setOnImageAvailableListener(
+      captureImageReader.setOnImageAvailableListener(
           new ImageReader.OnImageAvailableListener() {
             @Override
             public void onImageAvailable(ImageReader reader) {
@@ -537,7 +593,7 @@ public void onImageAvailable(ImageReader reader) {
       try {
         final CaptureRequest.Builder captureBuilder =
             cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE);
-        captureBuilder.addTarget(imageReader.getSurface());
+        captureBuilder.addTarget(captureImageReader.getSurface());
         int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation();
         int displayOrientation = ORIENTATIONS.get(displayRotation);
         if (isFrontFacing) displayOrientation = -displayOrientation;
@@ -667,7 +723,12 @@ private void startPreview() throws CameraAccessException {
       surfaces.add(previewSurface);
       captureRequestBuilder.addTarget(previewSurface);
 
-      surfaces.add(imageReader.getSurface());
+      surfaces.add(captureImageReader.getSurface());
+
+      // This is so we can send sample frames out to other plugins that need a live feed of frames
+      Surface previewImageReaderSurface = previewImageReader.getSurface();
+      surfaces.add(previewImageReaderSurface);
+      captureRequestBuilder.addTarget(previewImageReaderSurface);
 
       cameraDevice.createCaptureSession(
           surfaces,
@@ -720,20 +781,60 @@ private void close() {
         cameraDevice.close();
         cameraDevice = null;
       }
-      if (imageReader != null) {
-        imageReader.close();
-        imageReader = null;
+      if (previewImageReader != null) {
+        previewImageReader.close();
+        previewImageReader = null;
       }
       if (mediaRecorder != null) {
         mediaRecorder.reset();
         mediaRecorder.release();
         mediaRecorder = null;
       }
+      stopBackgroundThread();
     }
 
     private void dispose() {
       close();
       textureEntry.release();
     }
   }
+
+  private int getRotation() {
+    if (windowManager == null) {
+      windowManager = (WindowManager) activity.getSystemService(Context.WINDOW_SERVICE);
+    }
+    int degrees = 0;
+    int rotation = windowManager.getDefaultDisplay().getRotation();
+    switch (rotation) {
+      case Surface.ROTATION_0:
+        degrees = 0;
+        break;
+      case Surface.ROTATION_90:
+        degrees = 90;
+        break;
+      case Surface.ROTATION_180:
+        degrees = 180;
+        break;
+      case Surface.ROTATION_270:
+        degrees = 270;
+        break;
+      default:
+        Log.e("ML", "Bad rotation value: $rotation");
+    }
+
+    try {
+      int angle;
+      int displayAngle; // TODO? setDisplayOrientation?
+      CameraCharacteristics cameraCharacteristics =
+          cameraManager.getCameraCharacteristics(camera.cameraName);
+      Integer orientation = cameraCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION);
+      // back-facing
+      angle = (orientation - degrees + 360) % 360;
+      displayAngle = angle;
+      int translatedAngle = angle / 90;
+      return translatedAngle; // this corresponds to the rotation constants
+    } catch (CameraAccessException e) {
+      return 0;
+    }
+  }
 }

packages/camera/android/src/main/java/io/flutter/plugins/camera/PreviewImageDelegate.java

@@ -0,0 +1,7 @@
+package io.flutter.plugins.camera;
+
+import android.media.Image;
+
+public interface PreviewImageDelegate {
+  void onImageAvailable(Image image, int rotation);
+}

packages/camera/example/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

packages/firebase_ml_vision/android/build.gradle

@@ -25,7 +25,7 @@ android {
     compileSdkVersion 27
 
     defaultConfig {
-        minSdkVersion 16
+        minSdkVersion 21
         testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
     }
     lintOptions {
@@ -34,5 +34,6 @@ android {
     dependencies {
         api 'com.google.firebase:firebase-ml-vision:16.0.0'
         api 'com.google.firebase:firebase-ml-vision-image-label-model:15.0.0'
+        implementation project(':camera')
     }
 }

packages/firebase_ml_vision/android/src/main/AndroidManifest.xml

@@ -1,3 +1,5 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
   package="io.flutter.plugins.firebasemlvision">
+
+  <uses-permission android:name="android.permission.CAMERA" />
 </manifest>

packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/BarcodeDetector.java

@@ -3,27 +3,32 @@
 import android.graphics.Point;
 import android.graphics.Rect;
 import android.support.annotation.NonNull;
+import android.util.Size;
+
 import com.google.android.gms.tasks.OnFailureListener;
 import com.google.android.gms.tasks.OnSuccessListener;
 import com.google.firebase.ml.vision.FirebaseVision;
 import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode;
 import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector;
 import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetectorOptions;
 import com.google.firebase.ml.vision.common.FirebaseVisionImage;
-import io.flutter.plugin.common.MethodChannel;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-class BarcodeDetector implements Detector {
+public class BarcodeDetector extends Detector {
   public static final BarcodeDetector instance = new BarcodeDetector();
 
   private BarcodeDetector() {}
 
   @Override
-  public void handleDetection(
-      FirebaseVisionImage image, Map<String, Object> options, final MethodChannel.Result result) {
+  void processImage(
+    final FirebaseVisionImage image,
+    final Size imageSize,
+    Map<String, Object> options,
+    final OperationFinishedCallback finishedCallback) {
 
     FirebaseVisionBarcodeDetector detector =
         FirebaseVision.getInstance().getVisionBarcodeDetector(parseOptions(options));
@@ -32,6 +37,7 @@ public void handleDetection(
         .detectInImage(image)
         .addOnSuccessListener(
             new OnSuccessListener<List<FirebaseVisionBarcode>>() {
+              @SuppressWarnings("ConstantConditions")
               @Override
               public void onSuccess(List<FirebaseVisionBarcode> firebaseVisionBarcodes) {
                 List<Map<String, Object>> barcodes = new ArrayList<>();
@@ -210,14 +216,16 @@ public void onSuccess(List<FirebaseVisionBarcode> firebaseVisionBarcodes) {
 
                   barcodes.add(barcodeMap);
                 }
-                result.success(barcodes);
+                finishedCallback.success(BarcodeDetector.this, barcodes, imageSize);
               }
             })
         .addOnFailureListener(
             new OnFailureListener() {
               @Override
               public void onFailure(@NonNull Exception exception) {
-                result.error("barcodeDetectorError", exception.getLocalizedMessage(), null);
+                finishedCallback.error(
+                    new DetectorException(
+                        "barcodeDetectorError", exception.getLocalizedMessage(), null));
               }
             });
   }

packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/Detector.java

@@ -1,10 +1,55 @@
 package io.flutter.plugins.firebasemlvision;
 
+import android.util.Size;
+
 import com.google.firebase.ml.vision.common.FirebaseVisionImage;
-import io.flutter.plugin.common.MethodChannel;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public abstract class Detector {
+
+  public interface OperationFinishedCallback {
+    void success(Detector detector, Object data, Size size);
+
+    void error(DetectorException e);
+  }
+
+  private final AtomicBoolean shouldThrottle = new AtomicBoolean(false);
+
+  public void handleDetection(
+    final FirebaseVisionImage image,
+    final Size imageSize,
+    Map<String, Object> options,
+    final OperationFinishedCallback finishedCallback) {
+    if (shouldThrottle.get()) {
+      return;
+    }
+    processImage(
+        image,
+        imageSize,
+        options,
+        new OperationFinishedCallback() {
+          @Override
+          public void success(Detector detector, Object data, Size size) {
+            shouldThrottle.set(false);
+            finishedCallback.success(detector, data, size);
+          }
+
+          @Override
+          public void error(DetectorException e) {
+            shouldThrottle.set(false);
+            finishedCallback.error(e);
+          }
+        });
+
+    // Begin throttling until this frame of input has been processed, either in onSuccess or
+    // onFailure.
+    shouldThrottle.set(true);
+  }
 
-interface Detector {
-  void handleDetection(
-      FirebaseVisionImage image, Map<String, Object> options, final MethodChannel.Result result);
+  abstract void processImage(
+      FirebaseVisionImage image,
+      Size imageSize,
+      Map<String, Object> options,
+      OperationFinishedCallback finishedCallback);
 }