Skip to content

Commit 5ffcffa

Browse files
panmaripanmari
committed
Add support for YUV_420_888 Image format.
Casting the bytes to a type directly is not possible, thus allocating a new texture is necessary (and costly). But on the bright side, we avoid the conversion inside the camera plugin [0]. Or any other custom conversion code in dart, which is likely more costly. [0] https://github.com/flutter/packages/blob/d1fd6232ec33cd5a25aa762e605c494afced812f/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java#L35
1 parent d430f02 commit 5ffcffa

File tree

11 files changed

+178
-59
lines changed
  • packages
    • example/lib/vision_detector_views
    • google_mlkit_barcode_scanning/android/src/main/java/com/google_mlkit_barcode_scanning
    • google_mlkit_commons/android/src/main/java/com/google_mlkit_commons
    • google_mlkit_face_detection/android/src/main/java/com/google_mlkit_face_detection
    • google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection
    • google_mlkit_image_labeling/android/src/main/java/com/google_mlkit_image_labeling
    • google_mlkit_object_detection/android/src/main/java/com/google_mlkit_object_detection
    • google_mlkit_pose_detection/android/src/main/java/com/google_mlkit_pose_detection
    • google_mlkit_selfie_segmentation/android/src/main/java/com/google_mlkit_selfie_segmentation
    • google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation
    • google_mlkit_text_recognition/android/src/main/java/com/google_mlkit_text_recognition

11 files changed

+178
-59
lines changed

‎packages/example/lib/vision_detector_views/camera_view.dart

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -358,29 +358,52 @@ class _CameraViewState extends State<CameraView> {
358358

359359
// get image format
360360
final format = InputImageFormatValue.fromRawValue(image.format.raw);
361-
// validate format depending on platform
362-
// only supported formats:
363-
// * nv21 for Android
364-
// * bgra8888 for iOS
365-
if (format == null ||
366-
(Platform.isAndroid && format != InputImageFormat.nv21) ||
361+
if (format == null) {
362+
print('could not find format from raw value: $image.format.raw');
363+
return null;
364+
}
365+
// Validate format depending on platform
366+
final androidSupportedFormats = [
367+
InputImageFormat.nv21,
368+
InputImageFormat.yv12,
369+
InputImageFormat.yuv_420_888
370+
];
371+
if ((Platform.isAndroid && !androidSupportedFormats.contains(format)) ||
367372
(Platform.isIOS && format != InputImageFormat.bgra8888)) {
373+
print('image format is not supported: $format');
368374
return null;
369375
}
370376

371-
// since format is constraint to nv21 or bgra8888, both only have one plane
372-
if (image.planes.length != 1) return null;
373-
final plane = image.planes.first;
377+
// Compile a flat list of all image data. For image formats with multiple planes,
378+
// takes some copying.
379+
final Uint8List bytes = image.planes.length == 1
380+
? image.planes.first.bytes
381+
: _concatenatePlanes(image);
374382

375383
// compose InputImage using bytes
376384
return InputImage.fromBytes(
377-
bytes: plane.bytes,
385+
bytes: bytes,
378386
metadata: InputImageMetadata(
379387
size: Size(image.width.toDouble(), image.height.toDouble()),
380388
rotation: rotation, // used only in Android
381-
format: format, // used only in iOS
382-
bytesPerRow: plane.bytesPerRow, // used only in iOS
389+
format: format,
390+
bytesPerRow: image.planes.first.bytesPerRow, // used only in iOS
383391
),
384392
);
385393
}
394+
395+
Uint8List _concatenatePlanes(CameraImage image) {
396+
int length = 0;
397+
for (final Plane p in image.planes) {
398+
length += p.bytes.length;
399+
}
400+
401+
final Uint8List bytes = Uint8List(length);
402+
int offset = 0;
403+
for (final Plane p in image.planes) {
404+
bytes.setRange(offset, offset + p.bytes.length, p.bytes);
405+
offset += p.bytes.length;
406+
}
407+
return bytes;
408+
}
386409
}

‎packages/google_mlkit_barcode_scanning/android/src/main/java/com/google_mlkit_barcode_scanning/BarcodeScanner.java

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import io.flutter.plugin.common.MethodChannel;
2424

2525
public class BarcodeScanner implements MethodChannel.MethodCallHandler {
26+
2627
private static final String START = "vision#startBarcodeScanner";
2728
private static final String CLOSE = "vision#closeBarcodeScanner";
2829

@@ -67,8 +68,11 @@ private com.google.mlkit.vision.barcode.BarcodeScanner initialize(MethodCall cal
6768

6869
private void handleDetection(MethodCall call, final MethodChannel.Result result) {
6970
Map<String, Object> imageData = call.argument("imageData");
70-
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
71-
if (inputImage == null) return;
71+
InputImageConverter converter = new InputImageConverter();
72+
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
73+
if (inputImage == null) {
74+
return;
75+
}
7276

7377
String id = call.argument("id");
7478
com.google.mlkit.vision.barcode.BarcodeScanner barcodeScanner = instances.get(id);
@@ -191,7 +195,9 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
191195
}
192196
result.success(barcodeList);
193197
})
194-
.addOnFailureListener(e -> result.error("BarcodeDetectorError", e.toString(), null));
198+
.addOnFailureListener(e -> result.error("BarcodeDetectorError", e.toString(), e))
199+
// Closing is necessary for both success and failure.
200+
.addOnCompleteListener(r -> converter.close());
195201
}
196202

197203
private void addPoints(Point[] cornerPoints, List<Map<String, Integer>> points) {
@@ -205,7 +211,9 @@ private void addPoints(Point[] cornerPoints, List<Map<String, Integer>> points)
205211

206212
private Map<String, Integer> getBoundingPoints(@Nullable Rect rect) {
207213
Map<String, Integer> frame = new HashMap<>();
208-
if (rect == null) return frame;
214+
if (rect == null) {
215+
return frame;
216+
}
209217
frame.put("left", rect.left);
210218
frame.put("right", rect.right);
211219
frame.put("top", rect.top);
@@ -216,7 +224,9 @@ private Map<String, Integer> getBoundingPoints(@Nullable Rect rect) {
216224
private void closeDetector(MethodCall call) {
217225
String id = call.argument("id");
218226
com.google.mlkit.vision.barcode.BarcodeScanner barcodeScanner = instances.get(id);
219-
if (barcodeScanner == null) return;
227+
if (barcodeScanner == null) {
228+
return;
229+
}
220230
barcodeScanner.close();
221231
instances.remove(id);
222232
}

‎packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,30 @@
22

33
import android.content.Context;
44
import android.graphics.ImageFormat;
5+
import android.graphics.SurfaceTexture;
6+
import android.media.Image;
7+
import android.media.ImageWriter;
58
import android.net.Uri;
69
import android.util.Log;
10+
import android.view.Surface;
711

812
import com.google.mlkit.vision.common.InputImage;
913

1014
import java.io.File;
1115
import java.io.IOException;
16+
import java.lang.AutoCloseable;
17+
import java.nio.ByteBuffer;
1218
import java.util.Map;
1319
import java.util.Objects;
1420

1521
import io.flutter.plugin.common.MethodChannel;
1622

17-
public class InputImageConverter {
23+
public class InputImageConverter implements AutoCloseable {
24+
25+
ImageWriter writer;
1826

1927
//Returns an [InputImage] from the image data received
20-
public static InputImage getInputImageFromData(Map<String, Object> imageData,
28+
public InputImage getInputImageFromData(Map<String, Object> imageData,
2129
Context context,
2230
MethodChannel.Result result) {
2331
//Differentiates whether the image data is a path for a image file, contains image data in form of bytes, or a bitmap
@@ -116,9 +124,36 @@ public static InputImage getInputImageFromData(Map<String, Object> imageData,
116124
rotationDegrees,
117125
imageFormat);
118126
}
127+
if (imageFormat == ImageFormat.YUV_420_888) {
128+
// This image format is only supported in InputImage.fromMediaImage, which requires to transform the data to the right java type.
129+
// TODO: Consider reusing the same Surface across multiple calls to save on allocations.
130+
writer = new ImageWriter.Builder(new Surface(new SurfaceTexture(true)))
131+
.setWidthAndHeight(width, height)
132+
.setImageFormat(imageFormat)
133+
.build();
134+
Image image = writer.dequeueInputImage();
135+
if (image == null) {
136+
result.error("InputImageConverterError", "failed to allocate space for input image", null);
137+
return null;
138+
}
139+
// Deconstruct individual planes again from flattened array.
140+
Image.Plane[] planes = image.getPlanes();
141+
// Y plane
142+
ByteBuffer yBuffer = planes[0].getBuffer();
143+
yBuffer.put(data, 0, width * height);
144+
145+
// U plane
146+
ByteBuffer uBuffer = planes[1].getBuffer();
147+
int uOffset = width * height;
148+
uBuffer.put(data, uOffset, (width * height) / 4);
149+
150+
// V plane
151+
ByteBuffer vBuffer = planes[2].getBuffer();
152+
int vOffset = uOffset + (width * height) / 4;
153+
vBuffer.put(data, vOffset, (width * height) / 4);
154+
return InputImage.fromMediaImage(image, rotationDegrees);
155+
}
119156
result.error("InputImageConverterError", "ImageFormat is not supported.", null);
120-
// TODO: Use InputImage.fromMediaImage, which supports more types, e.g. IMAGE_FORMAT_YUV_420_888.
121-
// See https://developers.google.com/android/reference/com/google/mlkit/vision/common/InputImage#fromMediaImage(android.media.Image,%20int)
122157
return null;
123158
} catch (Exception e) {
124159
Log.e("ImageError", "Getting Image failed");
@@ -133,4 +168,11 @@ public static InputImage getInputImageFromData(Map<String, Object> imageData,
133168
}
134169
}
135170

171+
@Override
172+
public void close() {
173+
if (writer != null) {
174+
writer.close();
175+
}
176+
}
177+
136178
}

‎packages/google_mlkit_face_detection/android/src/main/java/com/google_mlkit_face_detection/FaceDetector.java

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import io.flutter.plugin.common.MethodChannel;
2424

2525
class FaceDetector implements MethodChannel.MethodCallHandler {
26+
2627
private static final String START = "vision#startFaceDetector";
2728
private static final String CLOSE = "vision#closeFaceDetector";
2829

@@ -52,9 +53,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result
5253

5354
private void handleDetection(MethodCall call, final MethodChannel.Result result) {
5455
Map<String, Object> imageData = (Map<String, Object>) call.argument("imageData");
55-
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
56-
if (inputImage == null)
56+
InputImageConverter converter = new InputImageConverter();
57+
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
58+
if (inputImage == null) {
5759
return;
60+
}
5861

5962
String id = call.argument("id");
6063
com.google.mlkit.vision.face.FaceDetector detector = instances.get(id);
@@ -115,7 +118,10 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
115118
result.success(faces);
116119
})
117120
.addOnFailureListener(
118-
e -> result.error("FaceDetectorError", e.toString(), null));
121+
e -> result.error("FaceDetectorError", e.toString(), null))
122+
// Closing is necessary for both success and failure.
123+
.addOnCompleteListener(r -> converter.close());
124+
119125
}
120126

121127
private FaceDetectorOptions parseOptions(Map<String, Object> options) {
@@ -206,7 +212,7 @@ private Map<String, List<double[]>> getContourData(Face face) {
206212
private double[] landmarkPosition(Face face, int landmarkInt) {
207213
FaceLandmark landmark = face.getLandmark(landmarkInt);
208214
if (landmark != null) {
209-
return new double[] { landmark.getPosition().x, landmark.getPosition().y };
215+
return new double[]{landmark.getPosition().x, landmark.getPosition().y};
210216
}
211217
return null;
212218
}
@@ -217,7 +223,7 @@ private List<double[]> contourPosition(Face face, int contourInt) {
217223
List<PointF> contourPoints = contour.getPoints();
218224
List<double[]> result = new ArrayList<>();
219225
for (int i = 0; i < contourPoints.size(); i++) {
220-
result.add(new double[] { contourPoints.get(i).x, contourPoints.get(i).y });
226+
result.add(new double[]{contourPoints.get(i).x, contourPoints.get(i).y});
221227
}
222228
return result;
223229
}
@@ -227,8 +233,9 @@ private List<double[]> contourPosition(Face face, int contourInt) {
227233
private void closeDetector(MethodCall call) {
228234
String id = call.argument("id");
229235
com.google.mlkit.vision.face.FaceDetector detector = instances.get(id);
230-
if (detector == null)
236+
if (detector == null) {
231237
return;
238+
}
232239
detector.close();
233240
instances.remove(id);
234241
}

‎packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/FaceMeshDetector.java

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import io.flutter.plugin.common.MethodChannel;
2323

2424
class FaceMeshDetector implements MethodChannel.MethodCallHandler {
25+
2526
private static final String START = "vision#startFaceMeshDetector";
2627
private static final String CLOSE = "vision#closeFaceMeshDetector";
2728

@@ -51,8 +52,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result
5152

5253
private void handleDetection(MethodCall call, final MethodChannel.Result result) {
5354
Map<String, Object> imageData = (Map<String, Object>) call.argument("imageData");
54-
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
55-
if (inputImage == null) return;
55+
InputImageConverter converter = new InputImageConverter();
56+
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
57+
if (inputImage == null) {
58+
return;
59+
}
5660

5761
String id = call.argument("id");
5862
com.google.mlkit.vision.facemesh.FaceMeshDetector detector = instances.get(id);
@@ -104,18 +108,18 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
104108
meshData.put("triangles", triangles);
105109

106110
int[] types = {
107-
FaceMesh.FACE_OVAL,
108-
FaceMesh.LEFT_EYEBROW_TOP,
109-
FaceMesh.LEFT_EYEBROW_BOTTOM,
110-
FaceMesh.RIGHT_EYEBROW_TOP,
111-
FaceMesh.RIGHT_EYEBROW_BOTTOM,
112-
FaceMesh.LEFT_EYE,
113-
FaceMesh.RIGHT_EYE,
114-
FaceMesh.UPPER_LIP_TOP,
115-
FaceMesh.UPPER_LIP_BOTTOM,
116-
FaceMesh.LOWER_LIP_TOP,
117-
FaceMesh.LOWER_LIP_BOTTOM,
118-
FaceMesh.NOSE_BRIDGE
111+
FaceMesh.FACE_OVAL,
112+
FaceMesh.LEFT_EYEBROW_TOP,
113+
FaceMesh.LEFT_EYEBROW_BOTTOM,
114+
FaceMesh.RIGHT_EYEBROW_TOP,
115+
FaceMesh.RIGHT_EYEBROW_BOTTOM,
116+
FaceMesh.LEFT_EYE,
117+
FaceMesh.RIGHT_EYE,
118+
FaceMesh.UPPER_LIP_TOP,
119+
FaceMesh.UPPER_LIP_BOTTOM,
120+
FaceMesh.LOWER_LIP_TOP,
121+
FaceMesh.LOWER_LIP_BOTTOM,
122+
FaceMesh.NOSE_BRIDGE
119123
};
120124
Map<Integer, List<Map<String, Object>>> contours = new HashMap<>();
121125
for (int type : types) {
@@ -129,7 +133,10 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
129133
result.success(faceMeshes);
130134
})
131135
.addOnFailureListener(
132-
e -> result.error("FaceMeshDetectorError", e.toString(), null));
136+
e -> result.error("FaceMeshDetectorError", e.toString(), null))
137+
// Closing is necessary for both success and failure.
138+
.addOnCompleteListener(r -> converter.close());
139+
133140
}
134141

135142
private List<Map<String, Object>> pointsToList(List<FaceMeshPoint> points) {
@@ -152,7 +159,9 @@ private Map<String, Object> pointToMap(FaceMeshPoint point) {
152159
private void closeDetector(MethodCall call) {
153160
String id = call.argument("id");
154161
com.google.mlkit.vision.facemesh.FaceMeshDetector detector = instances.get(id);
155-
if (detector == null) return;
162+
if (detector == null) {
163+
return;
164+
}
156165
detector.close();
157166
instances.remove(id);
158167
}

‎packages/google_mlkit_image_labeling/android/src/main/java/com/google_mlkit_image_labeling/ImageLabelDetector.java

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import io.flutter.plugin.common.MethodChannel;
2626

2727
public class ImageLabelDetector implements MethodChannel.MethodCallHandler {
28+
2829
private static final String START = "vision#startImageLabelDetector";
2930
private static final String CLOSE = "vision#closeImageLabelDetector";
3031
private static final String MANAGE = "vision#manageFirebaseModels";
@@ -59,8 +60,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result
5960

6061
private void handleDetection(MethodCall call, final MethodChannel.Result result) {
6162
Map<String, Object> imageData = call.argument("imageData");
62-
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
63-
if (inputImage == null) return;
63+
InputImageConverter converter = new InputImageConverter();
64+
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
65+
if (inputImage == null) {
66+
return;
67+
}
6468

6569
String id = call.argument("id");
6670
ImageLabeler imageLabeler = instances.get(id);
@@ -106,7 +110,9 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
106110

107111
result.success(labels);
108112
})
109-
.addOnFailureListener(e -> result.error("ImageLabelDetectorError", e.toString(), null));
113+
.addOnFailureListener(e -> result.error("ImageLabelDetectorError", e.toString(), e))
114+
// Closing is necessary for both success and failure.
115+
.addOnCompleteListener(r -> converter.close());
110116
}
111117

112118
//Labeler options that are provided to default image labeler(uses inbuilt model).
@@ -152,7 +158,9 @@ private CustomImageLabelerOptions getRemoteOptions(Map<String, Object> labelerOp
152158
private void closeDetector(MethodCall call) {
153159
String id = call.argument("id");
154160
ImageLabeler imageLabeler = instances.get(id);
155-
if (imageLabeler == null) return;
161+
if (imageLabeler == null) {
162+
return;
163+
}
156164
imageLabeler.close();
157165
instances.remove(id);
158166
}

0 commit comments

Comments
 (0)