diff --git a/packages/mediapipe-core/lib/generated/core_symbols.yaml b/packages/mediapipe-core/lib/generated/core_symbols.yaml index 19786c8b..531b258c 100644 --- a/packages/mediapipe-core/lib/generated/core_symbols.yaml +++ b/packages/mediapipe-core/lib/generated/core_symbols.yaml @@ -1,7 +1,7 @@ format_version: 1.0.0 files: - package:mediapipe_core/src/io/third_party/mediapipe/generated/mediapipe_common_bindings.dart: - used-config: + ? package:mediapipe_core/src/io/third_party/mediapipe/generated/mediapipe_common_bindings.dart + : used-config: ffi-native: false symbols: c:@S@BaseOptions: diff --git a/packages/mediapipe-core/lib/src/containers.dart b/packages/mediapipe-core/lib/src/containers.dart new file mode 100644 index 00000000..fbd7e0ef --- /dev/null +++ b/packages/mediapipe-core/lib/src/containers.dart @@ -0,0 +1,162 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi'; +import 'package:ffi/ffi.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; +import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as bindings; + +/// Dart representation of MediaPipe's "Category" concept. +/// +/// Category is a util class, that contains a category name, its display name, +/// a float value as score, and the index of the label in the corresponding +/// label file. Typically it's used as result of classification or detection +/// tasks. +/// +/// See more: +/// * [MediaPipe's Category documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/containers/Category) +class Category { + /// Generative constructor that creates a [Category] instance. + const Category({ + required this.index, + required this.score, + required this.categoryName, + required this.displayName, + }); + + /// The index of the label in the corresponding label file. + final int index; + + /// The probability score of this label category. + final double score; + + /// The label of this category object. + final String? categoryName; + + /// The display name of the label, which may be translated for different locales. + final String? displayName; + + /// Accepts a pointer to a list of structs, and a count representing the length + /// of the list, and returns a list of pure-Dart [Category] instances. + static List fromStructs( + Pointer structs, + int count, + ) { + final categories = []; + for (int i = 0; i < count; i++) { + categories.add(fromStruct(structs[i])); + } + return categories; + } + + /// Accepts a pointer to a single struct and returns a pure-Dart [Category] instance. + static Category fromStruct(bindings.Category struct) { + return Category( + index: struct.index, + score: struct.score, + categoryName: toDartString(struct.category_name), + displayName: toDartString(struct.display_name), + ); + } + + /// Releases all C memory associated with a list of [bindings.Category] pointers. + /// This method is important to call after calling [Category.fromStructs] to + /// convert that C memory into pure-Dart objects. + static void freeStructs(Pointer structs, int count) { + int index = 0; + while (index < count) { + bindings.Category obj = structs[index]; + calloc.free(obj.category_name); + calloc.free(obj.display_name); + index++; + } + calloc.free(structs); + } + + @override + String toString() => 'Category(index=$index, score=$score, ' + 'categoryName=$categoryName, displayName=$displayName)'; +} + +/// Dart representation of MediaPipe's "Classifications" concept. +/// +/// Represents the list of classification for a given classifier head. +/// Typically used as a result for classification tasks. +/// +/// See also: +/// * [MediaPipe's Classifications documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/containers/Classifications) +class Classifications { + /// Generative constructor that creates a [Classifications] instance. + const Classifications({ + required this.categories, + required this.headIndex, + required this.headName, + }); + + /// A list of Category objects which contain the actual classification + /// information, including human-readable labels and probability scores. + final List categories; + + /// The index of the classifier head these entries refer to. + final int headIndex; + + /// The optional name of the classifier head, which is the corresponding + /// tensor metadata name. + final String? headName; + + /// Accepts a pointer to a list of structs, and a count representing the length + /// of the list, and returns a list of pure-Dart [Classifications] instances. + static List fromStructs( + Pointer structs, + int count, + ) { + final classifications = []; + for (int i = 0; i < count; i++) { + classifications.add(fromStruct(structs[i])); + } + return classifications; + } + + /// Accepts a pointer to a single struct and returns a pure-Dart [Classifications] + /// instance. + static Classifications fromStruct(bindings.Classifications struct) { + return Classifications( + categories: Category.fromStructs( + struct.categories, + struct.categories_count, + ), + headIndex: struct.head_index, + headName: toDartString(struct.head_name), + ); + } + + /// Releases all C memory associated with a list of [bindings.Classifications] + /// pointers. This method is important to call after calling [Classifications.fromStructs] + /// to convert that C memory into pure-Dart objects. + static void freeStructs( + Pointer structs, + int count, + ) { + int index = 0; + while (index < count) { + bindings.Classifications obj = structs[index]; + Category.freeStructs(obj.categories, obj.categories_count); + calloc.free(obj.head_name); + index++; + } + calloc.free(structs); + } + + /// Convenience getter for the first [Category] out of the [categories] list. + Category? get firstCategory => + categories.isNotEmpty ? categories.first : null; + + @override + String toString() { + final categoryStrings = categories.map((cat) => cat.toString()).join(', '); + return 'Classification(categories=[$categoryStrings], ' + 'headIndex=$headIndex, headName=$headName)'; + } +} diff --git a/packages/mediapipe-core/lib/src/ffi_utils.dart b/packages/mediapipe-core/lib/src/ffi_utils.dart new file mode 100644 index 00000000..9410e13a --- /dev/null +++ b/packages/mediapipe-core/lib/src/ffi_utils.dart @@ -0,0 +1,95 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; +import 'package:ffi/ffi.dart'; + +/// Converts a list of Dart strings into their C memory equivalent. +/// +/// See also: +/// * [prepareString] +Pointer> prepareListOfStrings(List values) { + final ptrArray = calloc>(values.length); + for (var i = 0; i < values.length; i++) { + ptrArray[i] = prepareString(values[i]); + } + return ptrArray; +} + +/// Converts a single Dart string into its C memory equivalent. +/// +/// See also: +/// * [prepareListOfStrings] +Pointer prepareString(String val) => val.toNativeUtf8().cast(); + +/// Converts the C memory representation of a string into a Dart [String]. If the +/// supplied pointer is a null pointer, this function returns `null`. +/// +/// See also: +/// * [toDartStrings] +String? toDartString(Pointer val) { + if (val.address == 0) return null; + return val.cast().toDartString(); +} + +/// Converts a list of C memory representations of strings into a list of Dart +/// [String]s. +/// +/// See also: +/// * [toDartString] +List toDartStrings(Pointer> val, [int? length]) => + length != null + ? _toDartStringsWithCount(val, length) + : _toStartStringsUntilNull(val); + +List _toStartStringsUntilNull(Pointer> val) { + final dartStrings = []; + int counter = 0; + while (true) { + if (val[counter].address == 0) break; + dartStrings.add(toDartString(val[counter])); + counter++; + } + return dartStrings; +} + +List _toDartStringsWithCount(Pointer> val, int length) { + final dartStrings = []; + int counter = 0; + while (counter < length) { + dartStrings.add(toDartString(val[counter])); + counter++; + } + return dartStrings; +} + +/// Converts Dart's representation for binary data, a [Uint8List], into its C +/// memory representation. +Pointer prepareUint8List(Uint8List ints) { + final Pointer ptr = calloc(ints.length); + ptr.asTypedList(ints.length).setAll(0, ints); + return ptr.cast(); +} + +/// Converts a pointer to binary data in C memory to Dart's representation for +/// binary data, a [Uint8List]. +Uint8List toUint8List(Pointer val, {int? length}) { + final codeUnits = val.cast(); + if (length != null) { + RangeError.checkNotNegative(length, 'length'); + } else { + length = _length(codeUnits); + } + return codeUnits.asTypedList(length); +} + +// Counts the non-null bytes in a string to determine its length +int _length(Pointer codeUnits) { + var length = 0; + while (codeUnits[length] != 0) { + length++; + } + return length; +} diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart new file mode 100644 index 00000000..18dd0281 --- /dev/null +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -0,0 +1,158 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; +import 'package:equatable/equatable.dart'; +import 'package:ffi/ffi.dart'; +import 'ffi_utils.dart'; +import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as bindings; + +/// Dart representation of MediaPipe's "BaseOptions" concept. +/// +/// Used to configure various classifiers by specifying the model they will use +/// for computation. +/// +/// See also: +/// * [MediaPipe's BaseOptions documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/core/BaseOptions) +/// * [ClassifierOptions], which is often used in conjunction to specify a +/// classifier's desired behavior. +class BaseOptions extends Equatable { + /// Generative constructor that creates a [BaseOptions] instance. + const BaseOptions({this.modelAssetBuffer, this.modelAssetPath}) + : assert( + !(modelAssetBuffer == null && modelAssetPath == null), + 'You must supply either `modelAssetBuffer` or `modelAssetPath`', + ), + assert( + !(modelAssetBuffer != null && modelAssetPath != null), + 'You must only supply one of `modelAssetBuffer` and `modelAssetPath`', + ); + + /// The model asset file contents as bytes; + final Uint8List? modelAssetBuffer; + + /// Path to the model asset file. + final String? modelAssetPath; + + /// Converts this pure-Dart representation into C-memory suitable for the + /// MediaPipe SDK to instantiate various classifiers. + Pointer toStruct() { + final struct = calloc(); + + if (modelAssetPath != null) { + struct.ref.model_asset_path = prepareString(modelAssetPath!); + } + if (modelAssetBuffer != null) { + struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!); + } + return struct; + } + + @override + List get props => [modelAssetBuffer, modelAssetPath]; +} + +/// Dart representation of MediaPipe's "ClassifierOptions" concept. +/// +/// Classifier options shared across MediaPipe classification tasks. +/// +/// See also: +/// * [MediaPipe's ClassifierOptions documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/processors/ClassifierOptions) +/// * [BaseOptions], which is often used in conjunction to specify a +/// classifier's desired behavior. +class ClassifierOptions extends Equatable { + /// Generative constructor that creates a [ClassifierOptions] instance. + const ClassifierOptions({ + this.displayNamesLocale, + this.maxResults, + this.scoreThreshold, + this.categoryAllowlist, + this.categoryDenylist, + }); + + /// The locale to use for display names specified through the TFLite Model + /// Metadata. + final String? displayNamesLocale; + + /// The maximum number of top-scored classification results to return. + final int? maxResults; + + /// If set, establishes a minimum `score` and leads to the rejection of any + /// categories with lower `score` values. + final double? scoreThreshold; + + /// Allowlist of category names. + /// + /// If non-empty, classification results whose category name is not in + /// this set will be discarded. Duplicate or unknown category names + /// are ignored. Mutually exclusive with `categoryDenylist`. + /// + /// See also: + /// * [Category.categoryName] + final List? categoryAllowlist; + + /// Denylist of category names. + /// + /// If non-empty, classification results whose category name is in this set + /// will be discarded. Duplicate or unknown category names are ignored. + /// Mutually exclusive with `categoryAllowList`. + /// + /// See also: + /// * [Category.categoryName] + final List? categoryDenylist; + + /// Converts this pure-Dart representation into C-memory suitable for the + /// MediaPipe SDK to instantiate various classifiers. + Pointer toStruct() { + final struct = calloc(); + _setDisplayNamesLocale(struct.ref); + _setMaxResults(struct.ref); + _setScoreThreshold(struct.ref); + _setAllowlist(struct.ref); + _setDenylist(struct.ref); + return struct; + } + + void _setDisplayNamesLocale(bindings.ClassifierOptions struct) { + if (displayNamesLocale != null) { + struct.display_names_locale = prepareString(displayNamesLocale!); + } + } + + void _setMaxResults(bindings.ClassifierOptions struct) { + // This value must not be zero, and -1 implies no limit. + struct.max_results = maxResults ?? -1; + } + + void _setScoreThreshold(bindings.ClassifierOptions struct) { + if (scoreThreshold != null) { + struct.score_threshold = scoreThreshold!; + } + } + + void _setAllowlist(bindings.ClassifierOptions struct) { + if (categoryAllowlist != null) { + struct.category_allowlist = prepareListOfStrings(categoryAllowlist!); + struct.category_allowlist_count = categoryAllowlist!.length; + } + } + + void _setDenylist(bindings.ClassifierOptions struct) { + if (categoryDenylist != null) { + struct.category_denylist = prepareListOfStrings(categoryDenylist!); + struct.category_denylist_count = categoryDenylist!.length; + } + } + + @override + List get props => [ + displayNamesLocale, + maxResults, + scoreThreshold, + ...(categoryAllowlist ?? []), + ...(categoryDenylist ?? []), + ]; +} diff --git a/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart new file mode 100644 index 00000000..13bf3d24 --- /dev/null +++ b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart @@ -0,0 +1,347 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// AUTO GENERATED FILE, DO NOT EDIT. +// +// Generated by `package:ffigen`. +// ignore_for_file: type=lint +import 'dart:ffi' as ffi; + +final class BaseOptions extends ffi.Struct { + external ffi.Pointer model_asset_buffer; + + external ffi.Pointer model_asset_path; +} + +final class __mbstate_t extends ffi.Union { + @ffi.Array.multi([128]) + external ffi.Array __mbstate8; + + @ffi.LongLong() + external int _mbstateL; +} + +final class __darwin_pthread_handler_rec extends ffi.Struct { + external ffi + .Pointer)>> + __routine; + + external ffi.Pointer __arg; + + external ffi.Pointer<__darwin_pthread_handler_rec> __next; +} + +final class _opaque_pthread_attr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_cond_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([40]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_condattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutex_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutexattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_once_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlock_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([192]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlockattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([16]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + external ffi.Pointer<__darwin_pthread_handler_rec> __cleanup_stack; + + @ffi.Array.multi([8176]) + external ffi.Array __opaque; +} + +final class ClassifierOptions extends ffi.Struct { + external ffi.Pointer display_names_locale; + + @ffi.Int() + external int max_results; + + @ffi.Float() + external double score_threshold; + + external ffi.Pointer> category_allowlist; + + @ffi.Uint32() + external int category_allowlist_count; + + external ffi.Pointer> category_denylist; + + @ffi.Uint32() + external int category_denylist_count; +} + +final class Category extends ffi.Struct { + @ffi.Int() + external int index; + + @ffi.Float() + external double score; + + external ffi.Pointer category_name; + + external ffi.Pointer display_name; +} + +final class Classifications extends ffi.Struct { + external ffi.Pointer categories; + + @ffi.Uint32() + external int categories_count; + + @ffi.Int() + external int head_index; + + external ffi.Pointer head_name; +} + +final class ClassificationResult extends ffi.Struct { + external ffi.Pointer classifications; + + @ffi.Uint32() + external int classifications_count; + + @ffi.Int64() + external int timestamp_ms; + + @ffi.Bool() + external bool has_timestamp_ms; +} + +const int __WORDSIZE = 64; + +const int __DARWIN_ONLY_64_BIT_INO_T = 1; + +const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; + +const int __DARWIN_ONLY_VERS_1050 = 1; + +const int __DARWIN_UNIX03 = 1; + +const int __DARWIN_64_BIT_INO_T = 1; + +const int __DARWIN_VERS_1050 = 1; + +const int __DARWIN_NON_CANCELABLE = 0; + +const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; + +const int __DARWIN_C_ANSI = 4096; + +const int __DARWIN_C_FULL = 900000; + +const int __DARWIN_C_LEVEL = 900000; + +const int __STDC_WANT_LIB_EXT1__ = 1; + +const int __DARWIN_NO_LONG_LONG = 0; + +const int _DARWIN_FEATURE_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; + +const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; + +const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; + +const int __has_ptrcheck = 0; + +const int __DARWIN_NULL = 0; + +const int __PTHREAD_SIZE__ = 8176; + +const int __PTHREAD_ATTR_SIZE__ = 56; + +const int __PTHREAD_MUTEXATTR_SIZE__ = 8; + +const int __PTHREAD_MUTEX_SIZE__ = 56; + +const int __PTHREAD_CONDATTR_SIZE__ = 8; + +const int __PTHREAD_COND_SIZE__ = 40; + +const int __PTHREAD_ONCE_SIZE__ = 8; + +const int __PTHREAD_RWLOCK_SIZE__ = 192; + +const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; + +const int USER_ADDR_NULL = 0; + +const int INT8_MAX = 127; + +const int INT16_MAX = 32767; + +const int INT32_MAX = 2147483647; + +const int INT64_MAX = 9223372036854775807; + +const int INT8_MIN = -128; + +const int INT16_MIN = -32768; + +const int INT32_MIN = -2147483648; + +const int INT64_MIN = -9223372036854775808; + +const int UINT8_MAX = 255; + +const int UINT16_MAX = 65535; + +const int UINT32_MAX = 4294967295; + +const int UINT64_MAX = -1; + +const int INT_LEAST8_MIN = -128; + +const int INT_LEAST16_MIN = -32768; + +const int INT_LEAST32_MIN = -2147483648; + +const int INT_LEAST64_MIN = -9223372036854775808; + +const int INT_LEAST8_MAX = 127; + +const int INT_LEAST16_MAX = 32767; + +const int INT_LEAST32_MAX = 2147483647; + +const int INT_LEAST64_MAX = 9223372036854775807; + +const int UINT_LEAST8_MAX = 255; + +const int UINT_LEAST16_MAX = 65535; + +const int UINT_LEAST32_MAX = 4294967295; + +const int UINT_LEAST64_MAX = -1; + +const int INT_FAST8_MIN = -128; + +const int INT_FAST16_MIN = -32768; + +const int INT_FAST32_MIN = -2147483648; + +const int INT_FAST64_MIN = -9223372036854775808; + +const int INT_FAST8_MAX = 127; + +const int INT_FAST16_MAX = 32767; + +const int INT_FAST32_MAX = 2147483647; + +const int INT_FAST64_MAX = 9223372036854775807; + +const int UINT_FAST8_MAX = 255; + +const int UINT_FAST16_MAX = 65535; + +const int UINT_FAST32_MAX = 4294967295; + +const int UINT_FAST64_MAX = -1; + +const int INTPTR_MAX = 9223372036854775807; + +const int INTPTR_MIN = -9223372036854775808; + +const int UINTPTR_MAX = -1; + +const int INTMAX_MAX = 9223372036854775807; + +const int UINTMAX_MAX = -1; + +const int INTMAX_MIN = -9223372036854775808; + +const int PTRDIFF_MIN = -9223372036854775808; + +const int PTRDIFF_MAX = 9223372036854775807; + +const int SIZE_MAX = -1; + +const int RSIZE_MAX = 9223372036854775807; + +const int WCHAR_MAX = 2147483647; + +const int WCHAR_MIN = -2147483648; + +const int WINT_MIN = -2147483648; + +const int WINT_MAX = 2147483647; + +const int SIG_ATOMIC_MIN = -2147483648; + +const int SIG_ATOMIC_MAX = 2147483647; + +const int __bool_true_false_are_defined = 1; + +const int true1 = 1; + +const int false1 = 0; diff --git a/packages/mediapipe-core/test/task_options_test.dart b/packages/mediapipe-core/test/task_options_test.dart new file mode 100644 index 00000000..61ad88f7 --- /dev/null +++ b/packages/mediapipe-core/test/task_options_test.dart @@ -0,0 +1,115 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:test/test.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; + +void main() { + group('BaseOptions constructor should', () { + test('enforce exactly one of modelPath and modelBuffer', () { + expect( + () => BaseOptions( + modelAssetPath: 'abc', + modelAssetBuffer: Uint8List.fromList([1, 2, 3]), + ), + throwsA(TypeMatcher()), + ); + + expect(BaseOptions.new, throwsA(TypeMatcher())); + }); + }); + + group('BaseOptions.toStruct/fromStruct should', () { + test('allocate memory in C for a modelAssetPath', () { + final options = BaseOptions(modelAssetPath: 'abc'); + final struct = options.toStruct(); + expect(toDartString(struct.ref.model_asset_path), 'abc'); + expectNullPtr(struct.ref.model_asset_buffer); + }); + + test('allocate memory in C for a modelAssetBuffer', () { + final options = BaseOptions( + modelAssetBuffer: Uint8List.fromList([1, 2, 3]), + ); + final struct = options.toStruct(); + expect( + toUint8List(struct.ref.model_asset_buffer), + Uint8List.fromList([1, 2, 3]), + ); + expectNullPtr(struct.ref.model_asset_path); + }); + + test('allocate memory in C for a modelAssetBuffer containing 0', () { + final options = BaseOptions( + modelAssetBuffer: Uint8List.fromList([1, 2, 0, 3]), + ); + final struct = options.toStruct(); + expect( + toUint8List(struct.ref.model_asset_buffer), + Uint8List.fromList([1, 2]), + ); + expectNullPtr(struct.ref.model_asset_path); + }); + }); + + group('ClassOptions should', () { + test('allocate memory for empty fields', () { + final options = ClassifierOptions(); + + final struct = options.toStruct(); + + expect(struct.ref.max_results, -1); + expect(struct.ref.score_threshold, 0.0); + expect(struct.ref.category_allowlist, isA>()); + expect(struct.ref.category_allowlist_count, 0); + expect(struct.ref.category_denylist, isA>()); + expect(struct.ref.category_denylist_count, 0); + expectNullPtr(struct.ref.display_names_locale); + + // TODO: Could we do something like this? + // (right now, this segfaults) + // expect(struct.ref.display_names_locale.cast().asTypedList(1), + // Uint8List.fromList([0])); + }); + + test('allocate memory for full fields', () { + final options = ClassifierOptions( + displayNamesLocale: 'en', + maxResults: 5, + scoreThreshold: 0.9, + categoryAllowlist: ['good', 'great', 'best'], + categoryDenylist: ['bad', 'terrible', 'worst', 'honestly come on'], + ); + + final struct = options.toStruct(); + + expect(toDartString(struct.ref.display_names_locale), 'en'); + expect(struct.ref.max_results, 5); + expect(struct.ref.score_threshold, greaterThan(0.8999)); + expect(struct.ref.score_threshold, lessThan(0.90001)); + expect( + toDartStrings( + struct.ref.category_allowlist, + struct.ref.category_allowlist_count, + ), + ['good', 'great', 'best'], + ); + expect(struct.ref.category_allowlist_count, 3); + + expect( + toDartStrings( + struct.ref.category_denylist, + struct.ref.category_denylist_count, + ), + ['bad', 'terrible', 'worst', 'honestly come on'], + ); + expect(struct.ref.category_denylist_count, 4); + }); + }); +} + +void expectNullPtr(Pointer ptr) => expect(ptr.address, equals(0)); diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h index d03fe05e..ce91d71c 100644 --- a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h @@ -23,43 +23,43 @@ limitations under the License. extern "C" { #endif -// Defines classification results for a given classifier head. -struct Classifications { - // The array of predicted categories, usually sorted by descending scores, - // e.g. from high to low probability. - struct Category* categories; - // The number of elements in the categories array. - uint32_t categories_count; + // Defines classification results for a given classifier head. + struct Classifications { + // The array of predicted categories, usually sorted by descending scores, + // e.g. from high to low probability. + struct Category* categories; + // The number of elements in the categories array. + uint32_t categories_count; - // The index of the classifier head (i.e. output tensor) these categories - // refer to. This is useful for multi-head models. - int head_index; + // The index of the classifier head (i.e. output tensor) these categories + // refer to. This is useful for multi-head models. + int head_index; - // The optional name of the classifier head, as provided in the TFLite Model - // Metadata [1] if present. This is useful for multi-head models. - // - // [1]: https://www.tensorflow.org/lite/convert/metadata - char* head_name; -}; + // The optional name of the classifier head, as provided in the TFLite Model + // Metadata [1] if present. This is useful for multi-head models. + // + // [1]: https://www.tensorflow.org/lite/convert/metadata + char* head_name; + }; -// Defines classification results of a model. -struct ClassificationResult { - // The classification results for each head of the model. - struct Classifications* classifications; - // The number of classifications in the classifications array. - uint32_t classifications_count; + // Defines classification results of a model. + struct ClassificationResult { + // The classification results for each head of the model. + struct Classifications* classifications; + // The number of classifications in the classifications array. + uint32_t classifications_count; - // The optional timestamp (in milliseconds) of the start of the chunk of data - // corresponding to these results. - // - // This is only used for classification on time series (e.g. audio - // classification). In these use cases, the amount of data to process might - // exceed the maximum size that the model can process: to solve this, the - // input data is split into multiple chunks starting at different timestamps. - int64_t timestamp_ms; - // Specifies whether the timestamp contains a valid value. - bool has_timestamp_ms; -}; + // The optional timestamp (in milliseconds) of the start of the chunk of data + // corresponding to these results. + // + // This is only used for classification on time series (e.g. audio + // classification). In these use cases, the amount of data to process might + // exceed the maximum size that the model can process: to solve this, the + // input data is split into multiple chunks starting at different timestamps. + int64_t timestamp_ms; + // Specifies whether the timestamp contains a valid value. + bool has_timestamp_ms; + }; #ifdef __cplusplus } // extern C diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h index 32ad22b0..99f0edf1 100644 --- a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h @@ -22,35 +22,35 @@ limitations under the License. extern "C" { #endif -// Classifier options for MediaPipe C classification Tasks. -struct ClassifierOptions { - // The locale to use for display names specified through the TFLite Model - // Metadata, if any. Defaults to English. - const char* display_names_locale; - - // The maximum number of top-scored classification results to return. If < 0, - // all available results will be returned. If 0, an invalid argument error is - // returned. - int max_results; - - // Score threshold to override the one provided in the model metadata (if - // any). Results below this value are rejected. - float score_threshold; - - // The allowlist of category names. If non-empty, detection results whose - // category name is not in this set will be filtered out. Duplicate or unknown - // category names are ignored. Mutually exclusive with category_denylist. - const char** category_allowlist; - // The number of elements in the category allowlist. - uint32_t category_allowlist_count; - - // The denylist of category names. If non-empty, detection results whose - // category name is in this set will be filtered out. Duplicate or unknown - // category names are ignored. Mutually exclusive with category_allowlist. - const char** category_denylist; - // The number of elements in the category denylist. - uint32_t category_denylist_count; -}; + // Classifier options for MediaPipe C classification Tasks. + struct ClassifierOptions { + // The locale to use for display names specified through the TFLite Model + // Metadata, if any. Defaults to English. + const char* display_names_locale; + + // The maximum number of top-scored classification results to return. If < 0, + // all available results will be returned. If 0, an invalid argument error is + // returned. + int max_results; + + // Score threshold to override the one provided in the model metadata (if + // any). Results below this value are rejected. + float score_threshold; + + // The allowlist of category names. If non-empty, detection results whose + // category name is not in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_denylist. + const char** category_allowlist; + // The number of elements in the category allowlist. + uint32_t category_allowlist_count; + + // The denylist of category names. If non-empty, detection results whose + // category name is in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_allowlist. + const char** category_denylist; + // The number of elements in the category denylist. + uint32_t category_denylist_count; + }; #ifdef __cplusplus } // extern C diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h index 20c068a8..60f807bf 100644 --- a/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h @@ -20,17 +20,17 @@ limitations under the License. extern "C" { #endif -// Base options for MediaPipe C Tasks. -struct BaseOptions { - // The model asset file contents as bytes. - const char* model_asset_buffer; + // Base options for MediaPipe C Tasks. + struct BaseOptions { + // The model asset file contents as bytes. + const char* model_asset_buffer; - // The size of the model assets buffer (or `0` if not set). - unsigned int model_asset_buffer_count; + // The size of the model assets buffer (or `0` if not set). + unsigned int model_asset_buffer_count; - // The path to the model asset to open and mmap in memory. - const char* model_asset_path; -}; + // The path to the model asset to open and mmap in memory. + const char* model_asset_path; + }; #ifdef __cplusplus } // extern C diff --git a/packages/mediapipe-task-vision/.gitignore b/packages/mediapipe-task-vision/.gitignore new file mode 100644 index 00000000..ac5aa989 --- /dev/null +++ b/packages/mediapipe-task-vision/.gitignore @@ -0,0 +1,29 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.buildlog/ +.history +.svn/ +migrate_working_dir/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# The .vscode folder contains launch configuration and tasks you configure in +# VS Code which you may wish to be included in version control, so this line +# is commented out by default. +#.vscode/ + +# Flutter/Dart/Pub related +# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock. +/pubspec.lock +**/doc/api/ +.dart_tool/ +build/ diff --git a/packages/mediapipe-task-vision/.metadata b/packages/mediapipe-task-vision/.metadata new file mode 100644 index 00000000..a4e117e8 --- /dev/null +++ b/packages/mediapipe-task-vision/.metadata @@ -0,0 +1,10 @@ +# This file tracks properties of this Flutter project. +# Used by Flutter tool to assess capabilities and perform upgrades etc. +# +# This file should be version controlled and should not be manually edited. + +version: + revision: "7cfd3d47fc2e9a9266795e512d03ffd6f7f451fc" + channel: "master" + +project_type: package diff --git a/packages/mediapipe-task-vision/CHANGELOG.md b/packages/mediapipe-task-vision/CHANGELOG.md new file mode 100644 index 00000000..41cc7d81 --- /dev/null +++ b/packages/mediapipe-task-vision/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +* TODO: Describe initial release. diff --git a/packages/mediapipe-task-vision/analysis_options.yaml b/packages/mediapipe-task-vision/analysis_options.yaml new file mode 100644 index 00000000..ee7f274f --- /dev/null +++ b/packages/mediapipe-task-vision/analysis_options.yaml @@ -0,0 +1,5 @@ +include: ../analysis_options.yaml + +analyzer: + exclude: + - "**/mediapipe_vision_bindings.dart" diff --git a/packages/mediapipe-task-vision/ffigen.yaml b/packages/mediapipe-task-vision/ffigen.yaml new file mode 100644 index 00000000..309cbe03 --- /dev/null +++ b/packages/mediapipe-task-vision/ffigen.yaml @@ -0,0 +1,23 @@ +name: "MediaPipeVisionBindings" +description: "Bindings for MediaPipe structs for vision-related tasks" +output: + bindings: "lib/third_party/mediapipe/mediapipe_vision_bindings.dart" +headers: + entry-points: + - "third_party/mediapipe/tasks/c/**" +import: + symbol-files: + - "package:mediapipe_core/generated/core_symbols.yaml" +preamble: | + /* Copyright 2023 The MediaPipe Authors. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +ffi-native: diff --git a/packages/mediapipe-task-vision/lib/interface.dart b/packages/mediapipe-task-vision/lib/interface.dart new file mode 100644 index 00000000..110d7e37 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/interface.dart @@ -0,0 +1,5 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/interface/interface.dart'; \ No newline at end of file diff --git a/packages/mediapipe-task-vision/lib/io.dart b/packages/mediapipe-task-vision/lib/io.dart new file mode 100644 index 00000000..5a1b3ae7 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/io.dart @@ -0,0 +1,5 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/io/mediapipe_vision.dart'; diff --git a/packages/mediapipe-task-vision/lib/mediapipe_vision.dart b/packages/mediapipe-task-vision/lib/mediapipe_vision.dart new file mode 100644 index 00000000..d85d2554 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/mediapipe_vision.dart @@ -0,0 +1,10 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Package containing MediaPipe's vision-specific tasks. +library mediapipe_vision; + +export 'universal_mediapipe_vision.dart' + if (dart.library.html) 'src/web/mediapipe_vision.dart' + if (dart.library.io) 'src/io/mediapipe_vision.dart'; diff --git a/packages/mediapipe-task-vision/lib/src/interface/interface.dart b/packages/mediapipe-task-vision/lib/src/interface/interface.dart new file mode 100644 index 00000000..70736a95 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/interface/interface.dart @@ -0,0 +1,4 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + diff --git a/packages/mediapipe-task-vision/lib/src/interface/tasks/tasks.dart b/packages/mediapipe-task-vision/lib/src/interface/tasks/tasks.dart new file mode 100644 index 00000000..854d12e7 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/interface/tasks/tasks.dart @@ -0,0 +1,3 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. diff --git a/packages/mediapipe-task-vision/lib/src/io/mediapipe_vision.dart b/packages/mediapipe-task-vision/lib/src/io/mediapipe_vision.dart new file mode 100644 index 00000000..70736a95 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/io/mediapipe_vision.dart @@ -0,0 +1,4 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + diff --git a/packages/mediapipe-task-vision/lib/src/io/tasks/tasks.dart b/packages/mediapipe-task-vision/lib/src/io/tasks/tasks.dart new file mode 100644 index 00000000..854d12e7 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/io/tasks/tasks.dart @@ -0,0 +1,3 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. diff --git a/packages/mediapipe-task-vision/lib/third_party/mediapipe/mediapipe_vision_bindings.dart b/packages/mediapipe-task-vision/lib/third_party/mediapipe/mediapipe_vision_bindings.dart new file mode 100644 index 00000000..41a44458 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/third_party/mediapipe/mediapipe_vision_bindings.dart @@ -0,0 +1,298 @@ +/* Copyright 2023 The MediaPipe Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// AUTO GENERATED FILE, DO NOT EDIT. +// +// Generated by `package:ffigen`. +// ignore_for_file: type=lint +import 'dart:ffi' as ffi; +import 'package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as imp1; + +@ffi.Native< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer>)>(symbol: 'image_classifier_create') +external ffi.Pointer image_classifier_create( + ffi.Pointer options, + ffi.Pointer> error_msg, +); + +@ffi.Native< + ffi.Int Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>)>( + symbol: 'image_classifier_classify_image') +external int image_classifier_classify_image( + ffi.Pointer classifier, + ffi.Pointer image, + ffi.Pointer result, + ffi.Pointer> error_msg, +); + +@ffi.Native)>( + symbol: 'image_classifier_close_result') +external void image_classifier_close_result( + ffi.Pointer result, +); + +@ffi.Native< + ffi.Int Function(ffi.Pointer, + ffi.Pointer>)>(symbol: 'image_classifier_close') +external int image_classifier_close( + ffi.Pointer classifier, + ffi.Pointer> error_msg, +); + +abstract class ImageFormat { + static const int UNKNOWN = 0; + static const int SRGB = 1; + static const int SRGBA = 2; + static const int GRAY8 = 3; + static const int SBGRA = 11; +} + +abstract class RunningMode { + static const int IMAGE = 1; + static const int VIDEO = 2; + static const int LIVE_STREAM = 3; +} + +final class ImageFrame extends ffi.Struct { + @ffi.Int32() + external int format; + + external ffi.Pointer image_buffer; + + @ffi.Int() + external int width; + + @ffi.Int() + external int height; +} + +final class GpuBuffer extends ffi.Struct { + @ffi.Int() + external int height; + + @ffi.Int() + external int width; +} + +final class MpImage extends ffi.Struct { + @ffi.Int32() + external int type; + + external UnnamedUnion1 unnamed; +} + +final class UnnamedUnion1 extends ffi.Union { + external ImageFrame image_frame; + + external GpuBuffer gpu_buffer; +} + +final class ImageClassifierOptions extends ffi.Struct { + external imp1.BaseOptions base_options; + + @ffi.Int32() + external int running_mode; + + external imp1.ClassifierOptions classifier_options; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, + ffi.Pointer, ffi.Int64)>> result_callback_fn; + + @ffi.Int() + external int result_callback; +} + +typedef ImageClassifierResult = imp1.ClassificationResult; + +const int IMAGE_FRAME = 0; + +const int GPU_BUFFER = 1; + +const int __bool_true_false_are_defined = 1; + +const int true1 = 1; + +const int false1 = 0; + +const int __WORDSIZE = 64; + +const int __DARWIN_ONLY_64_BIT_INO_T = 1; + +const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; + +const int __DARWIN_ONLY_VERS_1050 = 1; + +const int __DARWIN_UNIX03 = 1; + +const int __DARWIN_64_BIT_INO_T = 1; + +const int __DARWIN_VERS_1050 = 1; + +const int __DARWIN_NON_CANCELABLE = 0; + +const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; + +const int __DARWIN_C_ANSI = 4096; + +const int __DARWIN_C_FULL = 900000; + +const int __DARWIN_C_LEVEL = 900000; + +const int __STDC_WANT_LIB_EXT1__ = 1; + +const int __DARWIN_NO_LONG_LONG = 0; + +const int _DARWIN_FEATURE_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; + +const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; + +const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; + +const int __has_ptrcheck = 0; + +const int __DARWIN_NULL = 0; + +const int __PTHREAD_SIZE__ = 8176; + +const int __PTHREAD_ATTR_SIZE__ = 56; + +const int __PTHREAD_MUTEXATTR_SIZE__ = 8; + +const int __PTHREAD_MUTEX_SIZE__ = 56; + +const int __PTHREAD_CONDATTR_SIZE__ = 8; + +const int __PTHREAD_COND_SIZE__ = 40; + +const int __PTHREAD_ONCE_SIZE__ = 8; + +const int __PTHREAD_RWLOCK_SIZE__ = 192; + +const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; + +const int USER_ADDR_NULL = 0; + +const int INT8_MAX = 127; + +const int INT16_MAX = 32767; + +const int INT32_MAX = 2147483647; + +const int INT64_MAX = 9223372036854775807; + +const int INT8_MIN = -128; + +const int INT16_MIN = -32768; + +const int INT32_MIN = -2147483648; + +const int INT64_MIN = -9223372036854775808; + +const int UINT8_MAX = 255; + +const int UINT16_MAX = 65535; + +const int UINT32_MAX = 4294967295; + +const int UINT64_MAX = -1; + +const int INT_LEAST8_MIN = -128; + +const int INT_LEAST16_MIN = -32768; + +const int INT_LEAST32_MIN = -2147483648; + +const int INT_LEAST64_MIN = -9223372036854775808; + +const int INT_LEAST8_MAX = 127; + +const int INT_LEAST16_MAX = 32767; + +const int INT_LEAST32_MAX = 2147483647; + +const int INT_LEAST64_MAX = 9223372036854775807; + +const int UINT_LEAST8_MAX = 255; + +const int UINT_LEAST16_MAX = 65535; + +const int UINT_LEAST32_MAX = 4294967295; + +const int UINT_LEAST64_MAX = -1; + +const int INT_FAST8_MIN = -128; + +const int INT_FAST16_MIN = -32768; + +const int INT_FAST32_MIN = -2147483648; + +const int INT_FAST64_MIN = -9223372036854775808; + +const int INT_FAST8_MAX = 127; + +const int INT_FAST16_MAX = 32767; + +const int INT_FAST32_MAX = 2147483647; + +const int INT_FAST64_MAX = 9223372036854775807; + +const int UINT_FAST8_MAX = 255; + +const int UINT_FAST16_MAX = 65535; + +const int UINT_FAST32_MAX = 4294967295; + +const int UINT_FAST64_MAX = -1; + +const int INTPTR_MAX = 9223372036854775807; + +const int INTPTR_MIN = -9223372036854775808; + +const int UINTPTR_MAX = -1; + +const int INTMAX_MAX = 9223372036854775807; + +const int UINTMAX_MAX = -1; + +const int INTMAX_MIN = -9223372036854775808; + +const int PTRDIFF_MIN = -9223372036854775808; + +const int PTRDIFF_MAX = 9223372036854775807; + +const int SIZE_MAX = -1; + +const int RSIZE_MAX = 9223372036854775807; + +const int WCHAR_MAX = 2147483647; + +const int WCHAR_MIN = -2147483648; + +const int WINT_MIN = -2147483648; + +const int WINT_MAX = 2147483647; + +const int SIG_ATOMIC_MIN = -2147483648; + +const int SIG_ATOMIC_MAX = 2147483647; diff --git a/packages/mediapipe-task-vision/lib/universal_mediapipe_vision.dart b/packages/mediapipe-task-vision/lib/universal_mediapipe_vision.dart new file mode 100644 index 00000000..854d12e7 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/universal_mediapipe_vision.dart @@ -0,0 +1,3 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. diff --git a/packages/mediapipe-task-vision/pubspec.yaml b/packages/mediapipe-task-vision/pubspec.yaml new file mode 100644 index 00000000..57da1cc7 --- /dev/null +++ b/packages/mediapipe-task-vision/pubspec.yaml @@ -0,0 +1,58 @@ +name: mediapipe_vision +description: "A new Flutter package project." +version: 0.0.1 +repository: https://github.com/google/flutter-mediapipe +issue_tracker: https://github.com/google/flutter-mediapipe/issues +publish_to: none + +environment: + sdk: ">=3.3.0-33.0.dev <4.0.0" + flutter: ">=1.17.0" + +dependencies: + ffi: ^2.1.0 + flutter: + sdk: flutter + logging: ^1.2.0 + mediapipe_core: + path: ../mediapipe-core + +dev_dependencies: + ffigen: ^9.0.1 + flutter_lints: ^2.0.0 + flutter_test: + sdk: flutter + +# For information on the generic Dart part of this file, see the +# following page: https://dart.dev/tools/pub/pubspec +# The following section is specific to Flutter packages. +flutter: null +# To add assets to your package, add an assets section, like this: +# assets: +# - images/a_dot_burr.jpeg +# - images/a_dot_ham.jpeg +# +# For details regarding assets in packages, see +# https://flutter.dev/assets-and-images/#from-packages +# +# An image asset can refer to one or more resolution-specific "variants", see +# https://flutter.dev/assets-and-images/#resolution-aware +# To add custom fonts to your package, add a fonts section here, +# in this "flutter" section. Each entry in this list should have a +# "family" key with the font family name, and a "fonts" key with a +# list giving the asset and other descriptors for the font. For +# example: +# fonts: +# - family: Schyler +# fonts: +# - asset: fonts/Schyler-Regular.ttf +# - asset: fonts/Schyler-Italic.ttf +# style: italic +# - family: Trajan Pro +# fonts: +# - asset: fonts/TrajanPro.ttf +# - asset: fonts/TrajanPro_Bold.ttf +# weight: 700 +# +# For details regarding fonts in packages, see +# https://flutter.dev/custom-fonts/#from-packages diff --git a/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart b/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart new file mode 100644 index 00000000..593cb08d --- /dev/null +++ b/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart @@ -0,0 +1,7 @@ +// import 'package:flutter_test/flutter_test.dart'; + +// import 'package:mediapipe_vision/mediapipe_vision.dart'; + +void main() { + // TODO(craiglabenz): Write tests +} diff --git a/packages/mediapipe-task-vision/third_party/mediapipe/tasks/c/vision/image_classifier.h b/packages/mediapipe-task-vision/third_party/mediapipe/tasks/c/vision/image_classifier.h new file mode 100644 index 00000000..7e57b7c0 --- /dev/null +++ b/packages/mediapipe-task-vision/third_party/mediapipe/tasks/c/vision/image_classifier.h @@ -0,0 +1,135 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_IMAGE_CLASSIFIER_IMAGE_CLASSIFIER_H_ +#define MEDIAPIPE_TASKS_C_VISION_IMAGE_CLASSIFIER_IMAGE_CLASSIFIER_H_ + +#include + +#include "../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h" +#include "../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h" +#include "../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + + typedef ClassificationResult ImageClassifierResult; + + // Supported image formats. + enum ImageFormat { + UNKNOWN = 0, + SRGB = 1, + SRGBA = 2, + GRAY8 = 3, + SBGRA = 11 // compatible with Flutter `bgra8888` format. + }; + + // Supported processing modes. + enum RunningMode { + IMAGE = 1, + VIDEO = 2, + LIVE_STREAM = 3, + }; + + // Structure to hold image frame. + struct ImageFrame { + enum ImageFormat format; + const uint8_t* image_buffer; + int width; + int height; + }; + + // TODO: Add GPU buffer declaration and proccessing logic for it. + struct GpuBuffer { + int height; + int width; + }; + + // The object to contain an image, realizes `OneOf` concept. + struct MpImage { + enum { IMAGE_FRAME, GPU_BUFFER } type; + union { + struct ImageFrame image_frame; + struct GpuBuffer gpu_buffer; + }; + }; + + // The options for configuring a Mediapipe image classifier task. + struct ImageClassifierOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + + // The running mode of the task. Default to the image mode. + // Image classifier has three running modes: + // 1) The image mode for classifying image on single image inputs. + // 2) The video mode for classifying image on the decoded frames of a video. + // 3) The live stream mode for classifying image on the live stream of input + // data, such as from camera. In this mode, the "result_callback" below must + // be specified to receive the segmentation results asynchronously. + RunningMode running_mode; + + // Options for configuring the classifier behavior, such as score threshold, + // number of results, etc. + struct ClassifierOptions classifier_options; + + // The user-defined result callback for processing live stream data. + // The result callback should only be specified when the running mode is set + // to RunningMode::LIVE_STREAM. + typedef void (*result_callback_fn)(ImageClassifierResult*, const MpImage*, + int64_t); + result_callback_fn result_callback; + }; + + // Creates an ImageClassifier from provided `options`. + // Returns a pointer to the image classifier on success. + // If an error occurs, returns `nullptr` and sets the error parameter to an + // an error message (if `error_msg` is not nullptr). You must free the memory + // allocated for the error message. + MP_EXPORT void* image_classifier_create(struct ImageClassifierOptions* options, + char** error_msg = nullptr); + + // Performs image classification on the input `image`. Returns `0` on success. + // If an error occurs, returns an error code and sets the error parameter to an + // an error message (if `error_msg` is not nullptr). You must free the memory + // allocated for the error message. + // + // TODO: Add API for video and live stream processing. + MP_EXPORT int image_classifier_classify_image(void* classifier, + const MpImage* image, + ImageClassifierResult* result, + char** error_msg = nullptr); + + // Frees the memory allocated inside a ImageClassifierResult result. + // Does not free the result pointer itself. + MP_EXPORT void image_classifier_close_result(ImageClassifierResult* result); + + // Frees image classifier. + // If an error occurs, returns an error code and sets the error parameter to an + // an error message (if `error_msg` is not nullptr). You must free the memory + // allocated for the error message. + MP_EXPORT int image_classifier_close(void* classifier, + char** error_msg = nullptr); + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_IMAGE_CLASSIFIER_IMAGE_CLASSIFIER_H_