model memory troubleshooting

Author: craiglabenz

packages/mediapipe-core/lib/src/task_options.dart

@@ -21,40 +21,83 @@ import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart'
 ///    classifier's desired behavior.
 class BaseOptions extends Equatable {
   /// Generative constructor that creates a [BaseOptions] instance.
-  const BaseOptions({this.modelAssetBuffer, this.modelAssetPath})
-      : assert(
+  const BaseOptions._({
+    this.modelAssetBuffer,
+    this.modelAssetPath,
+    this.modelAssetBufferCount,
+    required _BaseOptionsType type,
+  })  : assert(
           !(modelAssetBuffer == null && modelAssetPath == null),
           'You must supply either `modelAssetBuffer` or `modelAssetPath`',
         ),
         assert(
           !(modelAssetBuffer != null && modelAssetPath != null),
           'You must only supply one of `modelAssetBuffer` and `modelAssetPath`',
-        );
+        ),
+        assert(
+          (modelAssetBuffer == null) == (modelAssetBufferCount == null),
+          'modelAssetBuffer and modelAssetBufferCount must only be submitted '
+          'together',
+        ),
+        _type = type;
+
+  /// Constructor for [BaseOptions] classes using a file system path.
+  ///
+  /// In practice, this is unsupported, as assets in Flutter are bundled into
+  /// the build output and not available on disk. However, it can potentially
+  /// be helpful for testing / development purposes.
+  factory BaseOptions.path(String path) => BaseOptions._(
+        modelAssetPath: path,
+        type: _BaseOptionsType.path,
+      );
+
+  /// Constructor for [BaseOptions] classes using an in-memory pointer to the
+  /// MediaPipe SDK.
+  ///
+  /// In practice, this is the only option supported for production builds.
+  factory BaseOptions.memory(Uint8List buffer) {
+    return BaseOptions._(
+      modelAssetBuffer: buffer,
+      modelAssetBufferCount: buffer.lengthInBytes,
+      type: _BaseOptionsType.memory,
+    );
+  }
 
   /// The model asset file contents as bytes;
   final Uint8List? modelAssetBuffer;
 
+  /// The size of the model assets buffer (or `0` if not set).
+  final int? modelAssetBufferCount;
+
   /// Path to the model asset file.
   final String? modelAssetPath;
 
+  final _BaseOptionsType _type;
+
   /// Converts this pure-Dart representation into C-memory suitable for the
   /// MediaPipe SDK to instantiate various classifiers.
   Pointer<bindings.BaseOptions> toStruct() {
     final struct = calloc<bindings.BaseOptions>();
 
-    if (modelAssetPath != null) {
+    if (_type == _BaseOptionsType.path) {
       struct.ref.model_asset_path = prepareString(modelAssetPath!);
     }
-    if (modelAssetBuffer != null) {
+    if (_type == _BaseOptionsType.memory) {
       struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!);
     }
     return struct;
   }
 
   @override
-  List<Object?> get props => [modelAssetBuffer, modelAssetPath];
+  List<Object?> get props => [
+        modelAssetBuffer,
+        modelAssetPath,
+        modelAssetBufferCount,
+      ];
 }
 
+enum _BaseOptionsType { path, memory }
+
 /// Dart representation of MediaPipe's "ClassifierOptions" concept.
 ///
 /// Classifier options shared across MediaPipe classification tasks.

packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart

@@ -23,6 +23,9 @@ final class BaseOptions extends ffi.Struct {
   external ffi.Pointer<ffi.Char> model_asset_buffer;
 
   external ffi.Pointer<ffi.Char> model_asset_path;
+
+  @ffi.Int()
+  external int model_asset_buffer_count;
 }
 
 final class __mbstate_t extends ffi.Union {

packages/mediapipe-core/test/task_options_test.dart

@@ -9,32 +9,16 @@ import 'package:test/test.dart';
 import 'package:mediapipe_core/mediapipe_core.dart';
 
 void main() {
-  group('BaseOptions constructor should', () {
-    test('enforce exactly one of modelPath and modelBuffer', () {
-      expect(
-        () => BaseOptions(
-          modelAssetPath: 'abc',
-          modelAssetBuffer: Uint8List.fromList([1, 2, 3]),
-        ),
-        throwsA(TypeMatcher<AssertionError>()),
-      );
-
-      expect(BaseOptions.new, throwsA(TypeMatcher<AssertionError>()));
-    });
-  });
-
   group('BaseOptions.toStruct/fromStruct should', () {
     test('allocate memory in C for a modelAssetPath', () {
-      final options = BaseOptions(modelAssetPath: 'abc');
+      final options = BaseOptions.path('abc');
       final struct = options.toStruct();
       expect(toDartString(struct.ref.model_asset_path), 'abc');
       expectNullPtr(struct.ref.model_asset_buffer);
     });
 
     test('allocate memory in C for a modelAssetBuffer', () {
-      final options = BaseOptions(
-        modelAssetBuffer: Uint8List.fromList([1, 2, 3]),
-      );
+      final options = BaseOptions.memory(Uint8List.fromList([1, 2, 3]));
       final struct = options.toStruct();
       expect(
         toUint8List(struct.ref.model_asset_buffer),
@@ -44,9 +28,7 @@ void main() {
     });
 
     test('allocate memory in C for a modelAssetBuffer containing 0', () {
-      final options = BaseOptions(
-        modelAssetBuffer: Uint8List.fromList([1, 2, 0, 3]),
-      );
+      final options = BaseOptions.memory(Uint8List.fromList([1, 2, 0, 3]));
       final struct = options.toStruct();
       expect(
         toUint8List(struct.ref.model_asset_buffer),

packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h

@@ -20,14 +20,17 @@ limitations under the License.
 extern "C" {
 #endif
 
-// Base options for MediaPipe C Tasks.
-struct BaseOptions {
-  // The model asset file contents as a string.
-  char* model_asset_buffer;
-
-  // The path to the model asset to open and mmap in memory.
-  char* model_asset_path;
-};
+  // Base options for MediaPipe C Tasks.
+  struct BaseOptions {
+    // The model asset file contents as a string.
+    const char *model_asset_buffer;
+
+    // The path to the model asset to open and mmap in memory.
+    const char *model_asset_path;
+
+    // The size of the model assets buffer (or `0` if not set).
+    const size_t model_asset_buffer_count;
+  };
 
 #ifdef __cplusplus
 }  // extern C

packages/mediapipe-task-text/example/lib/main.dart

@@ -1,10 +1,8 @@
 import 'dart:io' as io;
 import 'dart:typed_data';
 import 'package:logging/logging.dart';
-import 'package:path/path.dart' as path;
 import 'package:flutter/material.dart';
 import 'package:mediapipe_text/mediapipe_text.dart';
-import 'package:path_provider/path_provider.dart';
 
 final _log = Logger('TextClassificationExample');
 
@@ -33,54 +31,24 @@ class _MainAppState extends State<MainApp> {
   late final TextClassifier _classifier;
   final TextEditingController _controller = TextEditingController();
   String? results;
+  late final ByteData classifierBytes;
 
   @override
   void initState() {
     super.initState();
     _controller.text = 'Hello, world!';
     _initClassifier();
+    Future.delayed(const Duration(milliseconds: 500)).then((_) => _classify());
   }
 
   Future<void> _initClassifier() async {
-    // getApplicationDocumentsDirectory().then((dir) {
-    //   print(dir.absolute.path);
-    // });
-    // final dir = await getApplicationSupportDirectory();
-    // print('app support: ${dir.absolute.path}');
-
-    // DefaultAssetBundle.of(context).
-
-    final ByteData classifierBytes = await DefaultAssetBundle.of(context)
+    classifierBytes = await DefaultAssetBundle.of(context)
         .load('assets/bert_classifier.tflite');
 
-    // final dir = io.Directory(path.current);
-    // final modelPath = path.joinAll(
-    //   [dir.absolute.path, 'assets/bert_classifier.tflite'],
-    // );
-    // _log.finest('modelPath: $modelPath');
-    // if (io.File(modelPath).existsSync()) {
-    //   _log.fine('Successfully found model.');
-    // } else {
-    //   _log.severe('Invalid model path \n\t$modelPath.\n\nModel not found.');
-    //   io.exit(1);
-    // }
-
-    // final sdkPath = path.joinAll(
-    //   [dir.absolute.path, 'assets/libtext_classifier.dylib'],
-    // );
-    // _log.finest('sdkPath: $sdkPath');
-    // if (io.File(sdkPath).existsSync()) {
-    //   _log.fine('Successfully found SDK.');
-    // } else {
-    //   _log.severe('Invalid SDK path $sdkPath. SDK not found.');
-    //   io.exit(1);
-    // }
-
     _classifier = TextClassifier(
-      // options: TextClassifierOptions.fromAssetPath(modelPath),
       options: TextClassifierOptions.fromAssetBuffer(
-          Uint8List.view(classifierBytes.buffer)),
-      // sdkPath: sdkPath,
+        classifierBytes.buffer.asUint8List(),
+      ),
     );
   }
 

packages/mediapipe-task-text/lib/src/tasks/text_classification/containers/text_classifier_options.dart

@@ -27,7 +27,7 @@ class TextClassifierOptions {
   }) {
     assert(!kIsWeb, 'fromAssetPath cannot be used on the web');
     return TextClassifierOptions(
-      baseOptions: BaseOptions(modelAssetPath: assetPath),
+      baseOptions: BaseOptions.path(assetPath),
       classifierOptions: classifierOptions,
     );
   }
@@ -40,7 +40,7 @@ class TextClassifierOptions {
     ClassifierOptions? classifierOptions,
   }) =>
       TextClassifierOptions(
-        baseOptions: BaseOptions(modelAssetBuffer: assetBuffer),
+        baseOptions: BaseOptions.memory(assetBuffer),
         classifierOptions: classifierOptions,
       );