[camera_windows] Support image streams on Windows platform (#7067)

Adds support to the windows camera implementation to allow streaming of frames.

Fixes flutter/flutter#97542

Author: jlundOverlay

packages/camera/camera_windows/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.5
+
+* Adds support for streaming frames.
+
 ## 0.2.4+1
 
 * Updates to pigeon 21.

packages/camera/camera_windows/lib/camera_windows.dart

@@ -11,6 +11,7 @@ import 'package:flutter/widgets.dart';
 import 'package:stream_transform/stream_transform.dart';
 
 import 'src/messages.g.dart';
+import 'type_conversion.dart';
 
 /// An implementation of [CameraPlatform] for Windows.
 class CameraWindows extends CameraPlatform {
@@ -29,6 +30,12 @@ class CameraWindows extends CameraPlatform {
   /// Camera specific method channels to allow communicating with specific cameras.
   final Map<int, MethodChannel> _cameraChannels = <int, MethodChannel>{};
 
+  // The stream to receive frames from the native code.
+  StreamSubscription<dynamic>? _platformImageStreamSubscription;
+
+  // The stream for vending frames to platform interface clients.
+  StreamController<CameraImageData>? _frameStreamController;
+
   /// The controller that broadcasts events coming from handleCameraMethodCall
   ///
   /// It is a `broadcast` because multiple controllers will connect to
@@ -242,6 +249,57 @@ class CameraWindows extends CameraPlatform {
         'resumeVideoRecording() is not supported due to Win32 API limitations.');
   }
 
+  @override
+  Stream<CameraImageData> onStreamedFrameAvailable(int cameraId,
+      {CameraImageStreamOptions? options}) {
+    _installStreamController(
+        onListen: () => _onFrameStreamListen(cameraId),
+        onCancel: () => _onFrameStreamCancel(cameraId));
+    return _frameStreamController!.stream;
+  }
+
+  StreamController<CameraImageData> _installStreamController(
+      {void Function()? onListen, void Function()? onCancel}) {
+    _frameStreamController = StreamController<CameraImageData>(
+      onListen: onListen ?? () {},
+      onPause: _onFrameStreamPauseResume,
+      onResume: _onFrameStreamPauseResume,
+      onCancel: onCancel ?? () {},
+    );
+    return _frameStreamController!;
+  }
+
+  void _onFrameStreamListen(int cameraId) {
+    _startPlatformStream(cameraId);
+  }
+
+  Future<void> _startPlatformStream(int cameraId) async {
+    _startStreamListener();
+    await _hostApi.startImageStream(cameraId);
+  }
+
+  void _startStreamListener() {
+    const EventChannel cameraEventChannel =
+        EventChannel('plugins.flutter.io/camera_android/imageStream');
+    _platformImageStreamSubscription =
+        cameraEventChannel.receiveBroadcastStream().listen((dynamic imageData) {
+      _frameStreamController!
+          .add(cameraImageFromPlatformData(imageData as Map<dynamic, dynamic>));
+    });
+  }
+
+  FutureOr<void> _onFrameStreamCancel(int cameraId) async {
+    await _hostApi.stopImageStream(cameraId);
+    await _platformImageStreamSubscription?.cancel();
+    _platformImageStreamSubscription = null;
+    _frameStreamController = null;
+  }
+
+  void _onFrameStreamPauseResume() {
+    throw CameraException('InvalidCall',
+        'Pause and resume are not supported for onStreamedFrameAvailable');
+  }
+
   @override
   Future<void> setFlashMode(int cameraId, FlashMode mode) async {
     // TODO(jokerttu): Implement flash mode support, https://github.com/flutter/flutter/issues/97537.

packages/camera/camera_windows/lib/src/messages.g.dart

@@ -348,6 +348,56 @@ class CameraApi {
     }
   }
 
+  /// Starts the image stream for the given camera.
+  Future<void> startImageStream(int cameraId) async {
+    final String __pigeon_channelName =
+        'dev.flutter.pigeon.camera_windows.CameraApi.startImageStream$__pigeon_messageChannelSuffix';
+    final BasicMessageChannel<Object?> __pigeon_channel =
+        BasicMessageChannel<Object?>(
+      __pigeon_channelName,
+      pigeonChannelCodec,
+      binaryMessenger: __pigeon_binaryMessenger,
+    );
+    final List<Object?>? __pigeon_replyList =
+        await __pigeon_channel.send(<Object?>[cameraId]) as List<Object?>?;
+    if (__pigeon_replyList == null) {
+      throw _createConnectionError(__pigeon_channelName);
+    } else if (__pigeon_replyList.length > 1) {
+      throw PlatformException(
+        code: __pigeon_replyList[0]! as String,
+        message: __pigeon_replyList[1] as String?,
+        details: __pigeon_replyList[2],
+      );
+    } else {
+      return;
+    }
+  }
+
+  /// Stops the image stream for the given camera.
+  Future<void> stopImageStream(int cameraId) async {
+    final String __pigeon_channelName =
+        'dev.flutter.pigeon.camera_windows.CameraApi.stopImageStream$__pigeon_messageChannelSuffix';
+    final BasicMessageChannel<Object?> __pigeon_channel =
+        BasicMessageChannel<Object?>(
+      __pigeon_channelName,
+      pigeonChannelCodec,
+      binaryMessenger: __pigeon_binaryMessenger,
+    );
+    final List<Object?>? __pigeon_replyList =
+        await __pigeon_channel.send(<Object?>[cameraId]) as List<Object?>?;
+    if (__pigeon_replyList == null) {
+      throw _createConnectionError(__pigeon_channelName);
+    } else if (__pigeon_replyList.length > 1) {
+      throw PlatformException(
+        code: __pigeon_replyList[0]! as String,
+        message: __pigeon_replyList[1] as String?,
+        details: __pigeon_replyList[2],
+      );
+    } else {
+      return;
+    }
+  }
+
   /// Starts the preview stream for the given camera.
   Future<void> pausePreview(int cameraId) async {
     final String __pigeon_channelName =

packages/camera/camera_windows/lib/type_conversion.dart

@@ -0,0 +1,25 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:typed_data';
+
+import 'package:camera_platform_interface/camera_platform_interface.dart';
+
+/// Converts method channel call [data] for `receivedImageStreamData` to a
+/// [CameraImageData].
+CameraImageData cameraImageFromPlatformData(Map<dynamic, dynamic> data) {
+  return CameraImageData(
+      format: const CameraImageFormat(ImageFormatGroup.bgra8888, raw: 0),
+      height: data['height'] as int,
+      width: data['width'] as int,
+      lensAperture: data['lensAperture'] as double?,
+      sensorExposureTime: data['sensorExposureTime'] as int?,
+      sensorSensitivity: data['sensorSensitivity'] as double?,
+      planes: <CameraImagePlane>[
+        CameraImagePlane(
+          bytes: data['data'] as Uint8List,
+          bytesPerRow: (data['width'] as int) * 4,
+        )
+      ]);
+}

packages/camera/camera_windows/pigeons/messages.dart

@@ -70,6 +70,14 @@ abstract class CameraApi {
   @async
   String stopVideoRecording(int cameraId);
 
+  /// Starts the image stream for the given camera.
+  @async
+  void startImageStream(int cameraId);
+
+  /// Stops the image stream for the given camera.
+  @async
+  void stopImageStream(int cameraId);
+
   /// Starts the preview stream for the given camera.
   @async
   void pausePreview(int cameraId);

packages/camera/camera_windows/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_windows
 description: A Flutter plugin for getting information about and controlling the camera on Windows.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_windows
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.2.4+1
+version: 0.2.5
 
 environment:
   sdk: ^3.2.0

packages/camera/camera_windows/windows/camera.h

@@ -28,6 +28,8 @@ enum class PendingResultType {
   kTakePicture,
   kStartRecord,
   kStopRecord,
+  kStartStream,
+  kStopStream,
   kPausePreview,
   kResumePreview,
 };

packages/camera/camera_windows/windows/camera_plugin.cpp

@@ -4,6 +4,8 @@
 
 #include "camera_plugin.h"
 
+#include <flutter/event_channel.h>
+#include <flutter/event_stream_handler_functions.h>
 #include <flutter/flutter_view.h>
 #include <flutter/method_channel.h>
 #include <flutter/plugin_registrar_windows.h>
@@ -32,6 +34,10 @@ namespace {
 
 const std::string kPictureCaptureExtension = "jpeg";
 const std::string kVideoCaptureExtension = "mp4";
+constexpr char kFrameEventChannelName[] =
+    "plugins.flutter.io/camera_android/imageStream";
+
+std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> event_sink;
 
 // Builds CaptureDeviceInfo object from given device holding device name and id.
 std::unique_ptr<CaptureDeviceInfo> GetDeviceInfo(IMFActivate* device) {
@@ -116,12 +122,34 @@ std::optional<std::string> GetFilePathForVideo() {
 }
 }  // namespace
 
+// a setter for the event sink helpful for testing.
+void CameraPlugin::SetEventSink(
+    std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> events) {
+  event_sink = std::move(events);
+}
+
 // static
 void CameraPlugin::RegisterWithRegistrar(
     flutter::PluginRegistrarWindows* registrar) {
   std::unique_ptr<CameraPlugin> plugin = std::make_unique<CameraPlugin>(
       registrar->texture_registrar(), registrar->messenger());
 
+  auto frameEventchannel = std::make_unique<flutter::EventChannel<>>(
+      registrar->messenger(), kFrameEventChannelName,
+      &flutter::StandardMethodCodec::GetInstance());
+
+  auto event_channel_handler =
+      std::make_unique<flutter::StreamHandlerFunctions<>>(
+          [plugin = plugin.get()](auto arguments, auto events) {
+            plugin->SetEventSink(std::move(events));
+            return nullptr;
+          },
+          [](auto arguments) {
+            event_sink.reset();
+            return nullptr;
+          });
+  frameEventchannel->SetStreamHandler(std::move(event_channel_handler));
+
   CameraApi::SetUp(registrar->messenger(), plugin.get());
 
   registrar->AddPlugin(std::move(plugin));
@@ -341,6 +369,53 @@ void CameraPlugin::StopVideoRecording(
   }
 }
 
+void CameraPlugin::StartImageStream(
+    int64_t camera_id,
+    std::function<void(std::optional<FlutterError> reply)> result) {
+  // check if request already exists
+  Camera* camera = GetCameraByCameraId(camera_id);
+  if (!camera) {
+    return result(FlutterError("camera_error", "Camera not created"));
+  }
+  if (camera->HasPendingResultByType(PendingResultType::kStartStream)) {
+    return result(
+        FlutterError("camera_error", "Pending start stream request exists"));
+  }
+
+  if (!event_sink) {
+    return result(FlutterError("camera_error",
+                               "Unable to make event channel from windows"));
+  }
+
+  if (camera->AddPendingVoidResult(PendingResultType::kStartStream,
+                                   std::move(result))) {
+    CaptureController* cc = camera->GetCaptureController();
+    assert(cc);
+    cc->StartImageStream(std::move(event_sink));
+  }
+}
+
+void CameraPlugin::StopImageStream(
+    int64_t camera_id,
+    std::function<void(std::optional<FlutterError> reply)> result) {
+  // check if request already exists
+  Camera* camera = GetCameraByCameraId(camera_id);
+  if (!camera) {
+    return result(FlutterError("camera_error", "Camera not created"));
+  }
+  if (camera->HasPendingResultByType(PendingResultType::kStopStream)) {
+    return result(
+        FlutterError("camera_error", "Pending stop stream request exists"));
+  }
+
+  if (camera->AddPendingVoidResult(PendingResultType::kStopStream,
+                                   std::move(result))) {
+    CaptureController* cc = camera->GetCaptureController();
+    assert(cc);
+    cc->StopImageStream();
+  }
+}
+
 void CameraPlugin::TakePicture(
     int64_t camera_id, std::function<void(ErrorOr<std::string> reply)> result) {
   auto camera = GetCameraByCameraId(camera_id);

packages/camera/camera_windows/windows/camera_plugin.h

@@ -31,6 +31,8 @@ class CameraPlugin : public flutter::Plugin,
                      public CameraApi,
                      public VideoCaptureDeviceEnumerator {
  public:
+  void SetEventSink(
+      std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> events);
   static void RegisterWithRegistrar(flutter::PluginRegistrarWindows* registrar);
 
   CameraPlugin(flutter::TextureRegistrar* texture_registrar,
@@ -68,6 +70,12 @@ class CameraPlugin : public flutter::Plugin,
   void StopVideoRecording(
       int64_t camera_id,
       std::function<void(ErrorOr<std::string> reply)> result) override;
+  void StartImageStream(
+      int64_t camera_id,
+      std::function<void(std::optional<FlutterError> reply)> result) override;
+  void StopImageStream(
+      int64_t camera_id,
+      std::function<void(std::optional<FlutterError> reply)> result) override;
   void TakePicture(
       int64_t camera_id,
       std::function<void(ErrorOr<std::string> reply)> result) override;

packages/camera/camera_windows/windows/capture_controller.cpp

@@ -5,11 +5,14 @@
 #include "capture_controller.h"
 
 #include <comdef.h>
+#include <flutter/event_stream_handler_functions.h>
+#include <flutter/standard_method_codec.h>
 #include <wincodec.h>
 #include <wrl/client.h>
 
 #include <cassert>
 #include <chrono>
+#include <iostream>
 
 #include "com_heap_ptr.h"
 #include "photo_handler.h"
@@ -550,6 +553,16 @@ void CaptureControllerImpl::StopRecord() {
                            "Failed to stop video recording");
   }
 }
+void CaptureControllerImpl::StartImageStream(
+    std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> sink) {
+  assert(capture_controller_listener_);
+  image_stream_sink_ = std::move(sink);
+}
+
+void CaptureControllerImpl::StopImageStream() {
+  assert(capture_controller_listener_);
+  image_stream_sink_.reset();
+}
 
 // Starts capturing preview frames using preview handler
 // After first frame is captured, OnPreviewStarted is called
@@ -843,6 +856,32 @@ bool CaptureControllerImpl::UpdateBuffer(uint8_t* buffer,
   if (!texture_handler_) {
     return false;
   }
+  if (image_stream_sink_) {
+    // Convert the buffer data to a std::vector<uint8_t>.
+    std::vector<uint8_t> buffer_data(buffer, buffer + data_length);
+
+    // Ensure preview_frame_height_ and preview_frame_width_ are of supported
+    // types.
+    int preview_frame_height = static_cast<int>(preview_frame_height_);
+    int preview_frame_width = static_cast<int>(preview_frame_width_);
+
+    // Create a map to hold the buffer data and data length.
+    flutter::EncodableMap data_map;
+    data_map[flutter::EncodableValue("data")] =
+        flutter::EncodableValue(buffer_data);
+    data_map[flutter::EncodableValue("height")] =
+        flutter::EncodableValue(preview_frame_height);
+    data_map[flutter::EncodableValue("width")] =
+        flutter::EncodableValue(preview_frame_width);
+    data_map[flutter::EncodableValue("length")] =
+        flutter::EncodableValue(static_cast<int>(data_length));
+
+    // Wrap the map in a flutter::EncodableValue.
+    flutter::EncodableValue encoded_value(data_map);
+
+    // Send the encoded value through the image_stream_sink_.
+    image_stream_sink_->Success(encoded_value);
+  }
   return texture_handler_->UpdateBuffer(buffer, data_length);
 }