Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

External Texture support for macOS Metal #24523

Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ci/licenses_golden/licenses_flutter
Original file line number Diff line number Diff line change
@@ -1109,6 +1109,8 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngin
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureGL.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureGL.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterFrameBufferProvider.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterFrameBufferProvider.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h
@@ -1118,6 +1120,7 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterIOSur
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterIOSurfaceHolder.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterIntermediateKeyResponder.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterIntermediateKeyResponder.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterMacOSExternalTexture.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterMetalRenderer.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterMetalRenderer.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterMetalRendererTest.mm
@@ -1138,6 +1141,8 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextI
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputModel.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextureRegistrar.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextureRegistrar.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterView.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterView.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm
7 changes: 7 additions & 0 deletions shell/platform/darwin/macos/BUILD.gn
Original file line number Diff line number Diff line change
@@ -63,6 +63,8 @@ source_set("flutter_framework_source") {
"framework/Source/FlutterEngine_Internal.h",
"framework/Source/FlutterExternalTextureGL.h",
"framework/Source/FlutterExternalTextureGL.mm",
"framework/Source/FlutterExternalTextureMetal.h",
"framework/Source/FlutterExternalTextureMetal.mm",
"framework/Source/FlutterFrameBufferProvider.h",
"framework/Source/FlutterFrameBufferProvider.mm",
"framework/Source/FlutterGLCompositor.h",
@@ -71,6 +73,8 @@ source_set("flutter_framework_source") {
"framework/Source/FlutterIOSurfaceHolder.mm",
"framework/Source/FlutterIntermediateKeyResponder.h",
"framework/Source/FlutterIntermediateKeyResponder.mm",
"framework/Source/FlutterMacOSExternalTexture.h",
"framework/Source/FlutterMacOSExternalTexture.h",
"framework/Source/FlutterMetalRenderer.h",
"framework/Source/FlutterMetalRenderer.mm",
"framework/Source/FlutterMouseCursorPlugin.h",
@@ -88,6 +92,8 @@ source_set("flutter_framework_source") {
"framework/Source/FlutterTextInputModel.mm",
"framework/Source/FlutterTextInputPlugin.h",
"framework/Source/FlutterTextInputPlugin.mm",
"framework/Source/FlutterTextureRegistrar.h",
"framework/Source/FlutterTextureRegistrar.mm",
"framework/Source/FlutterView.h",
"framework/Source/FlutterView.mm",
"framework/Source/FlutterViewController.mm",
@@ -104,6 +110,7 @@ source_set("flutter_framework_source") {
"//flutter/fml",
"//flutter/shell/platform/common:common_cpp_switches",
"//flutter/shell/platform/darwin/common:framework_shared",
"//flutter/shell/platform/darwin/graphics:graphics",
"//flutter/shell/platform/embedder:embedder_as_internal_library",
"//third_party/skia",
]
Original file line number Diff line number Diff line change
@@ -4,17 +4,16 @@

#import <Foundation/Foundation.h>

#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
#import "flutter/shell/platform/embedder/embedder.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterMacOSExternalTexture.h"

/**
* Used to bridge FlutterTexture object and handle the texture copy request the
* Flutter engine.
*/
@interface FlutterExternalTextureGL : NSObject
@interface FlutterExternalTextureGL : NSObject <FlutterMacOSExternalTexture>

/**
* Initializes a texture adapter with |texture| return a instance.
* Initializes a texture adapter with |texture|.
*/
- (nonnull instancetype)initWithFlutterTexture:(nonnull id<FlutterTexture>)texture;

@@ -26,9 +25,4 @@
*/
- (BOOL)populateTexture:(nonnull FlutterOpenGLTexture*)openGLTexture;

/**
* Returns the ID for the FlutterTexture instance.
*/
- (int64_t)textureID;

@end
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#import <Foundation/Foundation.h>

#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterMacOSExternalTexture.h"

/**
* Used to bridge FlutterTexture object and handle the texture copy request the
* Flutter engine.
*/
@interface FlutterExternalTextureMetal : NSObject <FlutterMacOSExternalTexture>

/**
* Initializes a texture adapter with |texture|.
*/
- (nonnull instancetype)initWithFlutterTexture:(nonnull id<FlutterTexture>)texture
darwinMetalContext:(nonnull FlutterDarwinContextMetal*)context;

/**
* Accepts texture buffer copy request from the Flutter engine.
* When the user side marks the textureID as available, the Flutter engine will
* callback to this method and ask for populate the |metalTexture| object,
* such as the texture type and the format of the pixel buffer and the texture object.
*/
- (BOOL)populateTexture:(nonnull FlutterMetalExternalTexture*)metalTexture;

@end
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.h"

@implementation FlutterExternalTextureMetal {
FlutterDarwinContextMetal* _darwinMetalContext;

int64_t _textureID;

id<FlutterTexture> _texture;

id<MTLTexture> _mtlTexture;
}

- (instancetype)initWithFlutterTexture:(id<FlutterTexture>)texture
darwinMetalContext:(FlutterDarwinContextMetal*)context {
self = [super init];
if (self) {
_texture = texture;
_textureID = reinterpret_cast<int64_t>(_texture);
_darwinMetalContext = context;
}
return self;
}

- (int64_t)textureID {
return _textureID;
}

- (BOOL)populateTexture:(FlutterMetalExternalTexture*)textureOut {
// Copy the pixel buffer from the FlutterTexture instance implemented on the user side.
CVPixelBufferRef pixelBuffer = [_texture copyPixelBuffer];

if (!pixelBuffer) {
return NO;
}

SkISize textureSize =
SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer));

CVMetalTextureRef cvMetalTexture = nullptr;
CVReturn cvReturn =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/_darwinMetalContext.textureCache,
/*sourceImage=*/pixelBuffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatBGRA8Unorm,
/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
/*planeIndex=*/0u,
/*texture=*/&cvMetalTexture);

if (cvReturn != kCVReturnSuccess) {
NSLog(@"Could not create Metal texture from pixel buffer: CVReturn %d", cvReturn);
return NO;
}

_mtlTexture = CVMetalTextureGetTexture(cvMetalTexture);
CVBufferRelease(cvMetalTexture);

std::vector<FlutterMetalTextureHandle> textures = {
(__bridge FlutterMetalTextureHandle)_mtlTexture};

textureOut->num_textures = 1;
textureOut->height = textureSize.height();
textureOut->width = textureSize.width();
textureOut->pixel_format = FlutterMetalExternalTexturePixelFormat::kRGBA;
textureOut->textures = textures.data();

return YES;
}

@end
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#import <Foundation/Foundation.h>

#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
#import "flutter/shell/platform/embedder/embedder.h"

/*
* Embedding side texture wrappers for GL and Metal external textures.
*/
@protocol FlutterMacOSExternalTexture

/**
* Returns the ID for the FlutterTexture instance.
*/
- (int64_t)textureID;

@end
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@
* FlutterEngine creation and then attached to the FlutterView once the FlutterViewController is
* initialized.
*/
@interface FlutterMetalRenderer : NSObject <FlutterRenderer>
@interface FlutterMetalRenderer : FlutterTextureRegistrar <FlutterRenderer>

/**
* Interface to the system GPU. Used to issue all the rendering commands.
@@ -34,4 +34,10 @@
*/
- (BOOL)present:(int64_t)textureID;

/**
* Populates the texture registry with the provided metalTexture.
*/
- (BOOL)populateTextureWithIdentifier:(int64_t)textureID
metalTexture:(nonnull FlutterMetalExternalTexture*)metalTexture;

@end
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterMetalRenderer.h"

#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterView.h"
#include "flutter/shell/platform/embedder/embedder.h"

@@ -23,16 +24,27 @@ static bool OnPresentDrawable(FlutterEngine* engine, const FlutterMetalTexture*
return [metalRenderer present:texture->texture_id];
}

static bool OnAcquireExternalTexture(FlutterEngine* engine,
int64_t textureIdentifier,
size_t width,
size_t height,
FlutterMetalExternalTexture* metalTexture) {
FlutterMetalRenderer* metalRenderer = reinterpret_cast<FlutterMetalRenderer*>(engine.renderer);
return [metalRenderer populateTextureWithIdentifier:textureIdentifier metalTexture:metalTexture];
}

#pragma mark - FlutterMetalRenderer implementation

@implementation FlutterMetalRenderer {
__weak FlutterEngine* _engine;

FlutterView* _flutterView;

FlutterDarwinContextMetal* _darwinMetalContext;
}

- (instancetype)initWithFlutterEngine:(nonnull FlutterEngine*)flutterEngine {
self = [super init];
self = [super initWithDelegate:self engine:flutterEngine];
if (self) {
_engine = flutterEngine;

@@ -47,6 +59,9 @@ - (instancetype)initWithFlutterEngine:(nonnull FlutterEngine*)flutterEngine {
NSLog(@"Could not create Metal command queue.");
return nil;
}

_darwinMetalContext = [[FlutterDarwinContextMetal alloc] initWithMTLDevice:_device
commandQueue:_commandQueue];
}
return self;
}
@@ -65,6 +80,8 @@ - (FlutterRendererConfig)createRendererConfig {
reinterpret_cast<FlutterMetalTextureCallback>(OnGetNextDrawable),
.metal.present_drawable_callback =
reinterpret_cast<FlutterMetalPresentCallback>(OnPresentDrawable),
.metal.external_texture_frame_callback =
reinterpret_cast<FlutterMetalTextureFrameCallback>(OnAcquireExternalTexture),
};
return config;
}
@@ -92,20 +109,17 @@ - (BOOL)present:(int64_t)textureID {

#pragma mark - FlutterTextureRegistrar methods.

- (int64_t)registerTexture:(id<FlutterTexture>)texture {
NSAssert(NO, @"External textures aren't yet supported when using Metal on macOS."
" See: https://github.com/flutter/flutter/issues/73826");
return 0;
}

- (void)textureFrameAvailable:(int64_t)textureID {
NSAssert(NO, @"External textures aren't yet supported when using Metal on macOS."
" See: https://github.com/flutter/flutter/issues/73826");
- (BOOL)populateTextureWithIdentifier:(int64_t)textureID
metalTexture:(FlutterMetalExternalTexture*)textureOut {
id<FlutterMacOSExternalTexture> texture = [self getTextureWithID:textureID];
FlutterExternalTextureMetal* metalTexture =
reinterpret_cast<FlutterExternalTextureMetal*>(texture);
return [metalTexture populateTexture:textureOut];
}

- (void)unregisterTexture:(int64_t)textureID {
NSAssert(NO, @"External textures aren't yet supported when using Metal on macOS."
" See: https://github.com/flutter/flutter/issues/73826");
- (id<FlutterMacOSExternalTexture>)onRegisterTexture:(id<FlutterTexture>)texture {
return [[FlutterExternalTextureMetal alloc] initWithFlutterTexture:texture
darwinMetalContext:_darwinMetalContext];
}

@end
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
* texture management. This is initialized during FlutterEngine creation and then attached to the
* FlutterView once the FlutterViewController is initializer.
*/
@interface FlutterOpenGLRenderer : NSObject <FlutterRenderer>
@interface FlutterOpenGLRenderer : FlutterTextureRegistrar <FlutterRenderer>

/**
* The resource context used by the engine for texture uploads. FlutterViews associated with this
Original file line number Diff line number Diff line change
@@ -57,17 +57,13 @@ @implementation FlutterOpenGLRenderer {
// The context provided to the Flutter engine for resource loading.
NSOpenGLContext* _resourceContext;

// A mapping of textureID to internal FlutterExternalTextureGL adapter.
NSMutableDictionary<NSNumber*, FlutterExternalTextureGL*>* _textures;

__weak FlutterEngine* _flutterEngine;
}

- (instancetype)initWithFlutterEngine:(FlutterEngine*)flutterEngine {
self = [super init];
self = [super initWithDelegate:self engine:flutterEngine];
if (self) {
_flutterEngine = flutterEngine;
_textures = [[NSMutableDictionary alloc] init];
}
return self;
}
@@ -136,37 +132,13 @@ - (BOOL)makeResourceCurrent {

- (BOOL)populateTextureWithIdentifier:(int64_t)textureID
openGLTexture:(FlutterOpenGLTexture*)openGLTexture {
return [_textures[@(textureID)] populateTexture:openGLTexture];
}

- (int64_t)registerTexture:(id<FlutterTexture>)texture {
FlutterExternalTextureGL* externalTexture =
[[FlutterExternalTextureGL alloc] initWithFlutterTexture:texture];
int64_t textureID = [externalTexture textureID];
BOOL success = [_flutterEngine registerTextureWithID:textureID];
if (success) {
_textures[@(textureID)] = externalTexture;
return textureID;
} else {
NSLog(@"Unable to register the texture with id: %lld.", textureID);
return 0;
}
}

- (void)textureFrameAvailable:(int64_t)textureID {
BOOL success = [_flutterEngine markTextureFrameAvailable:textureID];
if (!success) {
NSLog(@"Unable to mark texture with id %lld as available.", textureID);
}
id<FlutterMacOSExternalTexture> texture = [self getTextureWithID:textureID];
FlutterExternalTextureGL* glTexture = reinterpret_cast<FlutterExternalTextureGL*>(texture);
return [glTexture populateTexture:openGLTexture];
}

- (void)unregisterTexture:(int64_t)textureID {
bool success = [_flutterEngine unregisterTextureWithID:textureID];
if (success) {
[_textures removeObjectForKey:@(textureID)];
} else {
NSLog(@"Unable to unregister texture with id: %lld.", textureID);
}
- (id<FlutterMacOSExternalTexture>)onRegisterTexture:(id<FlutterTexture>)texture {
return [[FlutterExternalTextureGL alloc] initWithFlutterTexture:texture];
}

- (FlutterRendererConfig)createRendererConfig {
Original file line number Diff line number Diff line change
@@ -5,13 +5,14 @@
#import <Cocoa/Cocoa.h>

#import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterEngine.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextureRegistrar.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterView.h"
#import "flutter/shell/platform/embedder/embedder.h"

/**
* Rendering backend agnostic FlutterRendererConfig provider to be used by the embedder API.
*/
@protocol FlutterRenderer <FlutterTextureRegistry>
@protocol FlutterRenderer <FlutterTextureRegistry, FlutterTextureRegistrarDelegate>

/**
* Intializes the renderer with the given FlutterEngine.
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#import <Cocoa/Cocoa.h>

#import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterEngine.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterMacOSExternalTexture.h"

/*
* Delegate methods for FlutterTextureRegistrar.
*/
@protocol FlutterTextureRegistrarDelegate

/*
* Called by the FlutterTextureRegistrar when a texture is registered.
*/
- (nonnull id<FlutterMacOSExternalTexture>)onRegisterTexture:(nonnull id<FlutterTexture>)texture;

@end

/*
* Holds the external textures and implements the FlutterTextureRegistry.
*/
@interface FlutterTextureRegistrar : NSObject <FlutterTextureRegistry>

/*
* Use `initWithDelegate:engine:` instead.
*/
- (nullable instancetype)init NS_UNAVAILABLE;

/*
* Use `initWithDelegate:engine:` instead.
*/
+ (nullable instancetype)new NS_UNAVAILABLE;

/*
* Initialzes the texture registrar.
*/
- (nullable instancetype)initWithDelegate:(nonnull id<FlutterTextureRegistrarDelegate>)delegate
engine:(nonnull FlutterEngine*)engine NS_DESIGNATED_INITIALIZER;

/*
* Returns the registered texture with the provided `textureID`.
*/
- (nullable id<FlutterMacOSExternalTexture>)getTextureWithID:(int64_t)textureID;

@end
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterTextureRegistrar.h"

#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h"

@implementation FlutterTextureRegistrar {
__weak id<FlutterTextureRegistrarDelegate> _delegate;

__weak FlutterEngine* _flutterEngine;

// A mapping of textureID to internal FlutterExternalTextureGL adapter.
NSMutableDictionary<NSNumber*, id<FlutterMacOSExternalTexture>>* _textures;
}

- (instancetype)init {
@throw([NSException exceptionWithName:@"FlutterTextureRegistrar must initWithDelegate"
reason:nil
userInfo:nil]);
}

- (instancetype)initWithDelegate:(id<FlutterTextureRegistrarDelegate>)delegate
engine:(FlutterEngine*)engine {
if (self = [super init]) {
_delegate = delegate;
_flutterEngine = engine;
_textures = [[NSMutableDictionary alloc] init];
}
return self;
}

- (int64_t)registerTexture:(id<FlutterTexture>)texture {
id<FlutterMacOSExternalTexture> externalTexture = [_delegate onRegisterTexture:texture];
int64_t textureID = [externalTexture textureID];
BOOL success = [_flutterEngine registerTextureWithID:textureID];
if (success) {
_textures[@(textureID)] = externalTexture;
return textureID;
} else {
NSLog(@"Unable to register the texture with id: %lld.", textureID);
return 0;
}
}

- (void)textureFrameAvailable:(int64_t)textureID {
BOOL success = [_flutterEngine markTextureFrameAvailable:textureID];
if (!success) {
NSLog(@"Unable to mark texture with id %lld as available.", textureID);
}
}

- (void)unregisterTexture:(int64_t)textureID {
bool success = [_flutterEngine unregisterTextureWithID:textureID];
if (success) {
[_textures removeObjectForKey:@(textureID)];
} else {
NSLog(@"Unable to unregister texture with id: %lld.", textureID);
}
}

- (id<FlutterMacOSExternalTexture>)getTextureWithID:(int64_t)textureID {
return _textures[@(textureID)];
}

@end
11 changes: 6 additions & 5 deletions tools/gn
Original file line number Diff line number Diff line change
@@ -257,16 +257,17 @@ def to_gn_args(args):
# attributes in release modes till the toolchain is updated.
gn_args['skia_enable_api_available_macro'] = args.runtime_mode != "release"

if sys.platform == 'darwin' and args.macos_enable_metal:
if sys.platform == 'darwin' and args.target_os not in ['android', 'fuchsia']:
# OpenGL is deprecated on macOS > 10.11.
# This is not neccessarily needed but enabling this until we have a way to
# build a macOS metal only shell and a gl only shell.
gn_args['allow_deprecated_api_calls'] = True
gn_args['skia_use_metal'] = True
gn_args['shell_enable_metal'] = True
# Skia has Metal support on macOS version >= 10.14.
MACOS_SKIA_METAL_SUPPORTED_MIN_VERSION = '10.14'
gn_args['mac_sdk_min'] = MACOS_SKIA_METAL_SUPPORTED_MIN_VERSION
if args.macos_enable_metal:
gn_args['shell_enable_metal'] = True
# Skia has Metal support on macOS version >= 10.14.
MACOS_SKIA_METAL_SUPPORTED_MIN_VERSION = '10.14'
gn_args['mac_sdk_min'] = MACOS_SKIA_METAL_SUPPORTED_MIN_VERSION

if args.enable_vulkan:
# Enable vulkan in the Flutter shell.