[camerax] Implement Image Streaming (#3454)

Implements image streaming with all required image data, see [here](flutter/flutter#113792).

Fixes flutter/flutter#120463.
Fixes flutter/flutter#113792 along the way.

Special shoutout to @reidbaker for pair programming on this!

Author: camsim99

packages/camera/camera_android_camerax/CHANGELOG.md

@@ -18,3 +18,4 @@
 * Implements image capture.
 * Fixes cast of CameraInfo to fix integration test failure.
 * Updates internal Java InstanceManager to only stop finalization callbacks when stopped.
+* Implements image streaming.

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/AnalyzerFlutterApiImpl.java

@@ -0,0 +1,74 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.VisibleForTesting;
+import androidx.camera.core.ImageAnalysis;
+import androidx.camera.core.ImageProxy;
+import io.flutter.plugin.common.BinaryMessenger;
+import io.flutter.plugins.camerax.GeneratedCameraXLibrary.AnalyzerFlutterApi;
+import java.util.Objects;
+
+/**
+ * Flutter API implementation for {@link ImageAnalysis.Analyzer}.
+ *
+ * <p>This class may handle adding native instances that are attached to a Dart instance or passing
+ * arguments of callbacks methods to a Dart instance.
+ */
+public class AnalyzerFlutterApiImpl {
+  private final BinaryMessenger binaryMessenger;
+  private final InstanceManager instanceManager;
+  private AnalyzerFlutterApi api;
+
+  /**
+   * Constructs a {@link AnalyzerFlutterApiImpl}.
+   *
+   * @param binaryMessenger used to communicate with Dart over asynchronous messages
+   * @param instanceManager maintains instances stored to communicate with attached Dart objects
+   */
+  public AnalyzerFlutterApiImpl(
+      @NonNull BinaryMessenger binaryMessenger, @NonNull InstanceManager instanceManager) {
+    this.binaryMessenger = binaryMessenger;
+    this.instanceManager = instanceManager;
+    api = new AnalyzerFlutterApi(binaryMessenger);
+  }
+
+  /**
+   * Stores the {@link ImageAnalysis.Analyzer} instance and notifies Dart to create and store a new
+   * {@code Analyzer} instance that is attached to this one. If {@code instance} has already been
+   * added, this method does nothing.
+   */
+  public void create(
+      @NonNull ImageAnalysis.Analyzer instance, @NonNull AnalyzerFlutterApi.Reply<Void> callback) {
+    if (!instanceManager.containsInstance(instance)) {
+      api.create(instanceManager.addHostCreatedInstance(instance), callback);
+    }
+  }
+
+  /**
+   * Sends a message to Dart to call {@code Analyzer.analyze} on the Dart object representing
+   * `instance`.
+   */
+  public void analyze(
+      @NonNull ImageAnalysis.Analyzer analyzerInstance,
+      @NonNull ImageProxy imageProxyInstance,
+      @NonNull AnalyzerFlutterApi.Reply<Void> callback) {
+    api.analyze(
+        Objects.requireNonNull(instanceManager.getIdentifierForStrongReference(analyzerInstance)),
+        Objects.requireNonNull(instanceManager.getIdentifierForStrongReference(imageProxyInstance)),
+        callback);
+  }
+
+  /**
+   * Sets the Flutter API used to send messages to Dart.
+   *
+   * <p>This is only visible for testing.
+   */
+  @VisibleForTesting
+  void setApi(@NonNull AnalyzerFlutterApi api) {
+    this.api = api;
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/AnalyzerHostApiImpl.java

@@ -0,0 +1,119 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.VisibleForTesting;
+import androidx.camera.core.ImageAnalysis;
+import androidx.camera.core.ImageProxy;
+import io.flutter.plugin.common.BinaryMessenger;
+import io.flutter.plugins.camerax.GeneratedCameraXLibrary.AnalyzerHostApi;
+
+/**
+ * Host API implementation for {@link ImageAnalysis.Analyzer}.
+ *
+ * <p>This class may handle instantiating and adding native object instances that are attached to a
+ * Dart instance or handle method calls on the associated native class or an instance of the class.
+ */
+public class AnalyzerHostApiImpl implements AnalyzerHostApi {
+  private final BinaryMessenger binaryMessenger;
+  private final InstanceManager instanceManager;
+  private final AnalyzerProxy proxy;
+
+  /** Proxy for constructors and static method of {@link ImageAnalysis.Analyzer}. */
+  @VisibleForTesting
+  public static class AnalyzerProxy {
+
+    /** Creates an instance of {@link AnalyzerImpl}. */
+    @NonNull
+    public AnalyzerImpl create(
+        @NonNull BinaryMessenger binaryMessenger, @NonNull InstanceManager instanceManager) {
+      return new AnalyzerImpl(binaryMessenger, instanceManager);
+    }
+  }
+
+  /**
+   * Implementation of {@link ImageAnalysis.Analyzer} that passes arguments of callback methods to
+   * Dart.
+   */
+  public static class AnalyzerImpl implements ImageAnalysis.Analyzer {
+    private BinaryMessenger binaryMessenger;
+    private InstanceManager instanceManager;
+    private AnalyzerFlutterApiImpl api;
+
+    @VisibleForTesting @NonNull public ImageProxyFlutterApiImpl imageProxyApi;
+
+    /**
+     * Constructs an instance of {@link ImageAnalysis.Analyzer} that passes arguments of callbacks
+     * methods to Dart.
+     */
+    public AnalyzerImpl(
+        @NonNull BinaryMessenger binaryMessenger, @NonNull InstanceManager instanceManager) {
+      super();
+      this.binaryMessenger = binaryMessenger;
+      this.instanceManager = instanceManager;
+      api = new AnalyzerFlutterApiImpl(binaryMessenger, instanceManager);
+      imageProxyApi = new ImageProxyFlutterApiImpl(binaryMessenger, instanceManager);
+    }
+
+    @Override
+    public void analyze(@NonNull ImageProxy imageProxy) {
+      Long imageFormat = Long.valueOf(imageProxy.getFormat());
+      Long imageHeight = Long.valueOf(imageProxy.getHeight());
+      Long imageWidth = Long.valueOf(imageProxy.getWidth());
+      imageProxyApi.create(imageProxy, imageFormat, imageHeight, imageWidth, reply -> {});
+
+      api.analyze(this, imageProxy, reply -> {});
+    }
+
+    /**
+     * Flutter API used to send messages back to Dart.
+     *
+     * <p>This is only visible for testing.
+     */
+    @VisibleForTesting
+    void setApi(@NonNull AnalyzerFlutterApiImpl api) {
+      this.api = api;
+    }
+  }
+
+  /**
+   * Constructs a {@link AnalyzerHostApiImpl}.
+   *
+   * @param binaryMessenger used to communicate with Dart over asynchronous messages
+   * @param instanceManager maintains instances stored to communicate with attached Dart objects
+   */
+  public AnalyzerHostApiImpl(
+      @NonNull BinaryMessenger binaryMessenger, @NonNull InstanceManager instanceManager) {
+    this(binaryMessenger, instanceManager, new AnalyzerProxy());
+  }
+
+  /**
+   * Constructs a {@link AnalyzerHostApiImpl}.
+   *
+   * @param binaryMessenger used to communicate with Dart over asynchronous messages
+   * @param instanceManager maintains instances stored to communicate with attached Dart objects
+   * @param proxy proxy for constructors and static method of {@link ImageAnalysis.Analyzer}
+   */
+  @VisibleForTesting
+  AnalyzerHostApiImpl(
+      @NonNull BinaryMessenger binaryMessenger,
+      @NonNull InstanceManager instanceManager,
+      @NonNull AnalyzerProxy proxy) {
+    this.binaryMessenger = binaryMessenger;
+    this.instanceManager = instanceManager;
+    this.proxy = proxy;
+  }
+
+  /**
+   * Creates an {@link AnalyzerProxy} that represents an {@link ImageAnalysis.Analyzer} instance
+   * with the specified identifier.
+   */
+  @Override
+  public void create(@NonNull Long identifier) {
+    instanceManager.addDartCreatedInstance(
+        proxy.create(binaryMessenger, instanceManager), identifier);
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraAndroidCameraxPlugin.java

@@ -18,6 +18,7 @@ public final class CameraAndroidCameraxPlugin implements FlutterPlugin, Activity
   private InstanceManager instanceManager;
   private FlutterPluginBinding pluginBinding;
   private ProcessCameraProviderHostApiImpl processCameraProviderHostApi;
+  private ImageAnalysisHostApiImpl imageAnalysisHostApiImpl;
   private ImageCaptureHostApiImpl imageCaptureHostApi;
   public SystemServicesHostApiImpl systemServicesHostApi;
 
@@ -56,6 +57,12 @@ void setUp(BinaryMessenger binaryMessenger, Context context, TextureRegistry tex
         binaryMessenger, new PreviewHostApiImpl(binaryMessenger, instanceManager, textureRegistry));
     imageCaptureHostApi = new ImageCaptureHostApiImpl(binaryMessenger, instanceManager, context);
     GeneratedCameraXLibrary.ImageCaptureHostApi.setup(binaryMessenger, imageCaptureHostApi);
+    imageAnalysisHostApiImpl = new ImageAnalysisHostApiImpl(binaryMessenger, instanceManager);
+    GeneratedCameraXLibrary.ImageAnalysisHostApi.setup(binaryMessenger, imageAnalysisHostApiImpl);
+    GeneratedCameraXLibrary.AnalyzerHostApi.setup(
+        binaryMessenger, new AnalyzerHostApiImpl(binaryMessenger, instanceManager));
+    GeneratedCameraXLibrary.ImageProxyHostApi.setup(
+        binaryMessenger, new ImageProxyHostApiImpl(binaryMessenger, instanceManager));
   }
 
   @Override
@@ -113,5 +120,8 @@ public void updateContext(Context context) {
     if (imageCaptureHostApi != null) {
       processCameraProviderHostApi.setContext(context);
     }
+    if (imageAnalysisHostApiImpl != null) {
+      imageAnalysisHostApiImpl.setContext(context);
+    }
   }
 }

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXProxy.java

@@ -6,16 +6,27 @@
 
 import android.app.Activity;
 import android.graphics.SurfaceTexture;
+import android.util.Size;
 import android.view.Surface;
 import androidx.annotation.NonNull;
 import androidx.camera.core.CameraSelector;
+import androidx.camera.core.ImageAnalysis;
 import androidx.camera.core.ImageCapture;
 import androidx.camera.core.Preview;
 import io.flutter.plugin.common.BinaryMessenger;
+import io.flutter.plugins.camerax.GeneratedCameraXLibrary.ResolutionInfo;
 import java.io.File;
 
 /** Utility class used to create CameraX-related objects primarily for testing purposes. */
 public class CameraXProxy {
+  /**
+   * Converts a {@link ResolutionInfo} instance to a {@link Size} for setting the target resolution
+   * of {@link UseCase}s.
+   */
+  public static Size sizeFromResolution(@NonNull ResolutionInfo resolutionInfo) {
+    return new Size(resolutionInfo.getWidth().intValue(), resolutionInfo.getHeight().intValue());
+  }
+
   public CameraSelector.Builder createCameraSelectorBuilder() {
     return new CameraSelector.Builder();
   }
@@ -58,7 +69,20 @@ public ImageCapture.Builder createImageCaptureBuilder() {
   /**
    * Creates an {@link ImageCapture.OutputFileOptions} to configure where to save a captured image.
    */
+  @NonNull
   public ImageCapture.OutputFileOptions createImageCaptureOutputFileOptions(@NonNull File file) {
     return new ImageCapture.OutputFileOptions.Builder(file).build();
   }
+
+  /** Creates an instance of {@link ImageAnalysis.Builder}. */
+  @NonNull
+  public ImageAnalysis.Builder createImageAnalysisBuilder() {
+    return new ImageAnalysis.Builder();
+  }
+
+  /** Creates an array of {@code byte}s with the size provided. */
+  @NonNull
+  public byte[] getBytesFromBuffer(int size) {
+    return new byte[size];
+  }
 }