Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Move detection of cutouts in Android engine to onApplyWindowInsets #55992

Merged
merged 23 commits into from
Nov 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -196,13 +196,7 @@ public void onFlutterUiNoLongerDisplayed() {
}
};

private final Consumer<WindowLayoutInfo> windowInfoListener =
new Consumer<WindowLayoutInfo>() {
@Override
public void accept(WindowLayoutInfo layoutInfo) {
setWindowInfoListenerDisplayFeatures(layoutInfo);
}
};
private Consumer<WindowLayoutInfo> windowInfoListener;

/**
* Constructs a {@code FlutterView} programmatically, without any XML attributes.
Expand Down Expand Up @@ -514,6 +508,10 @@ protected void onAttachedToWindow() {
this.windowInfoRepo = createWindowInfoRepo();
Activity activity = ViewUtils.getActivity(getContext());
if (windowInfoRepo != null && activity != null) {
// Creating windowInfoListener on-demand instead of at initialization is necessary in order to
// prevent it from capturing the wrong instance of `this` when spying for testing.
// See https://github.com/mockito/mockito/issues/3479
windowInfoListener = this::setWindowInfoListenerDisplayFeatures;
windowInfoRepo.addWindowLayoutInfoListener(
activity, ContextCompat.getMainExecutor(getContext()), windowInfoListener);
}
Expand All @@ -526,9 +524,10 @@ protected void onAttachedToWindow() {
*/
@Override
protected void onDetachedFromWindow() {
if (windowInfoRepo != null) {
if (windowInfoRepo != null && windowInfoListener != null) {
windowInfoRepo.removeWindowLayoutInfoListener(windowInfoListener);
}
windowInfoListener = null;
this.windowInfoRepo = null;
super.onDetachedFromWindow();
}
Expand All @@ -539,12 +538,12 @@ protected void onDetachedFromWindow() {
*/
@TargetApi(API_LEVELS.API_28)
protected void setWindowInfoListenerDisplayFeatures(WindowLayoutInfo layoutInfo) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I dont understand why only the display cutout features need to be moved and not all the features in this method.

Unrelated to your change it is weird that this had a targetApi of 28 and still did an api check inside the method.

Copy link
Member

@gmackall gmackall Oct 21, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My understanding is that this listener is triggered (only) on changes to DisplayFeatures, which include hinges and display folds but (from @yaakovschectman and my manual testing) not camera cutouts (or other top of device sensors).

Those are instead reported as WindowInsets with a nonzero element reported by WindowInsets.getDisplayCutout() (and underlying changes to them instead trigger onApplyWindowInsets).

Flutter was conflating both of these two distinct groups into a List<FlutterRenderer.DisplayFeature>, but the list would only get updated in the case of changes to the former group, because underlying updates to the latter didn't trigger the listener (and we didn't handle them appropriately in the listener that was triggered).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK that makes sense thanks for the explanation.

List<DisplayFeature> displayFeatures = layoutInfo.getDisplayFeatures();
List<FlutterRenderer.DisplayFeature> result = new ArrayList<>();
List<DisplayFeature> newDisplayFeatures = layoutInfo.getDisplayFeatures();
List<FlutterRenderer.DisplayFeature> flutterDisplayFeatures = new ArrayList<>();

// Data from WindowInfoTracker display features. Fold and hinge areas are
// populated here.
for (DisplayFeature displayFeature : displayFeatures) {
for (DisplayFeature displayFeature : newDisplayFeatures) {
Log.v(
TAG,
"WindowInfoTracker Display Feature reported with bounds = "
Expand All @@ -567,31 +566,17 @@ protected void setWindowInfoListenerDisplayFeatures(WindowLayoutInfo layoutInfo)
} else {
state = DisplayFeatureState.UNKNOWN;
}
result.add(new FlutterRenderer.DisplayFeature(displayFeature.getBounds(), type, state));
flutterDisplayFeatures.add(
new FlutterRenderer.DisplayFeature(displayFeature.getBounds(), type, state));
} else {
result.add(
flutterDisplayFeatures.add(
new FlutterRenderer.DisplayFeature(
displayFeature.getBounds(),
DisplayFeatureType.UNKNOWN,
DisplayFeatureState.UNKNOWN));
}
}

// Data from the DisplayCutout bounds. Cutouts for cameras and other sensors are
// populated here. DisplayCutout was introduced in API 28.
if (Build.VERSION.SDK_INT >= API_LEVELS.API_28) {
WindowInsets insets = getRootWindowInsets();
if (insets != null) {
DisplayCutout cutout = insets.getDisplayCutout();
if (cutout != null) {
for (Rect bounds : cutout.getBoundingRects()) {
Log.v(TAG, "DisplayCutout area reported with bounds = " + bounds.toString());
result.add(new FlutterRenderer.DisplayFeature(bounds, DisplayFeatureType.CUTOUT));
}
}
}
}
viewportMetrics.displayFeatures = result;
viewportMetrics.setDisplayFeatures(flutterDisplayFeatures);
sendViewportMetricsToFlutter();
}

Expand Down Expand Up @@ -784,6 +769,22 @@ navigationBarVisible && guessBottomKeyboardInset(insets) == 0
viewportMetrics.viewInsetLeft = 0;
}

// Data from the DisplayCutout bounds. Cutouts for cameras and other sensors are
// populated here. DisplayCutout was introduced in API 28.
List<FlutterRenderer.DisplayFeature> displayCutouts = new ArrayList<>();
if (Build.VERSION.SDK_INT >= API_LEVELS.API_28) {
DisplayCutout cutout = insets.getDisplayCutout();
if (cutout != null) {
for (Rect bounds : cutout.getBoundingRects()) {
Log.v(TAG, "DisplayCutout area reported with bounds = " + bounds.toString());
displayCutouts.add(
new FlutterRenderer.DisplayFeature(
bounds, DisplayFeatureType.CUTOUT, DisplayFeatureState.UNKNOWN));
}
}
}
viewportMetrics.setDisplayCutouts(displayCutouts);

// The caption bar inset is a new addition, and the APIs called to query it utilize a list of
// bounding Rects instead of an Insets object, which is a newer API method, as compared to the
// existing Insets-based method calls above.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1157,6 +1157,13 @@ public void stopRenderingToSurface() {
}
}

private void translateFeatureBounds(int[] displayFeatureBounds, int offset, Rect bounds) {
displayFeatureBounds[offset] = bounds.left;
displayFeatureBounds[offset + 1] = bounds.top;
displayFeatureBounds[offset + 2] = bounds.right;
displayFeatureBounds[offset + 3] = bounds.bottom;
}

/**
* Notifies Flutter that the viewport metrics, e.g. window height and width, have changed.
*
Expand Down Expand Up @@ -1207,20 +1214,31 @@ public void setViewportMetrics(@NonNull ViewportMetrics viewportMetrics) {
+ viewportMetrics.systemGestureInsetRight
+ "\n"
+ "Display Features: "
+ viewportMetrics.displayFeatures.size());

int[] displayFeaturesBounds = new int[viewportMetrics.displayFeatures.size() * 4];
int[] displayFeaturesType = new int[viewportMetrics.displayFeatures.size()];
int[] displayFeaturesState = new int[viewportMetrics.displayFeatures.size()];
+ viewportMetrics.displayFeatures.size()
+ "\n"
+ "Display Cutouts: "
+ viewportMetrics.displayCutouts.size());

int totalFeaturesAndCutouts =
viewportMetrics.displayFeatures.size() + viewportMetrics.displayCutouts.size();
int[] displayFeaturesBounds = new int[totalFeaturesAndCutouts * 4];
int[] displayFeaturesType = new int[totalFeaturesAndCutouts];
int[] displayFeaturesState = new int[totalFeaturesAndCutouts];
for (int i = 0; i < viewportMetrics.displayFeatures.size(); i++) {
DisplayFeature displayFeature = viewportMetrics.displayFeatures.get(i);
displayFeaturesBounds[4 * i] = displayFeature.bounds.left;
displayFeaturesBounds[4 * i + 1] = displayFeature.bounds.top;
displayFeaturesBounds[4 * i + 2] = displayFeature.bounds.right;
displayFeaturesBounds[4 * i + 3] = displayFeature.bounds.bottom;
translateFeatureBounds(displayFeaturesBounds, 4 * i, displayFeature.bounds);
displayFeaturesType[i] = displayFeature.type.encodedValue;
displayFeaturesState[i] = displayFeature.state.encodedValue;
}
int cutoutOffset = viewportMetrics.displayFeatures.size() * 4;
for (int i = 0; i < viewportMetrics.displayCutouts.size(); i++) {
DisplayFeature displayCutout = viewportMetrics.displayCutouts.get(i);
translateFeatureBounds(displayFeaturesBounds, cutoutOffset + 4 * i, displayCutout.bounds);
displayFeaturesType[viewportMetrics.displayFeatures.size() + i] =
displayCutout.type.encodedValue;
displayFeaturesState[viewportMetrics.displayFeatures.size() + i] =
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems weird to intentionally be setting unknown here. It seems to be what were were doing before but should we have added a state enum value for these?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The current doc-comment for UNKNOWN states The display feature is a cutout or this state is new and not yet known to Flutter. The framework also currently expects cutouts to have a state of unknown.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems odd but out of scope for this pr.

displayCutout.state.encodedValue;
}

flutterJNI.setViewportMetrics(
viewportMetrics.devicePixelRatio,
Expand Down Expand Up @@ -1335,7 +1353,29 @@ boolean validate() {
return width > 0 && height > 0 && devicePixelRatio > 0;
}

public List<DisplayFeature> displayFeatures = new ArrayList<>();
// Features
private final List<DisplayFeature> displayFeatures = new ArrayList<>();

// Specifically display cutouts.
private final List<DisplayFeature> displayCutouts = new ArrayList<>();

public List<DisplayFeature> getDisplayFeatures() {
return displayFeatures;
}

public List<DisplayFeature> getDisplayCutouts() {
return displayCutouts;
}

public void setDisplayFeatures(List<DisplayFeature> newFeatures) {
displayFeatures.clear();
displayFeatures.addAll(newFeatures);
}

public void setDisplayCutouts(List<DisplayFeature> newCutouts) {
displayCutouts.clear();
displayCutouts.addAll(newCutouts);
}
}

/**
Expand All @@ -1358,12 +1398,6 @@ public DisplayFeature(Rect bounds, DisplayFeatureType type, DisplayFeatureState
this.type = type;
this.state = state;
}

public DisplayFeature(Rect bounds, DisplayFeatureType type) {
this.bounds = bounds;
this.type = type;
this.state = DisplayFeatureState.UNKNOWN;
}
}

/**
Expand Down
Loading