Custom GL Texture with gl_external_texture_frame_callback

[berkutta]

Hi

Not sure if GitHub issues are the right place for this question.

I'm trying to implement a custom Texture into flutter-pi.

First, I have tried the described setup from this issue entry #312 (comment) to integrate Gstreamer with a custom Texture. Unfortunately, I encountered issues with switching between camera and video playback. I guess this is some kind of issue in the Gstreamer abstraction while enabling and disabling streams.

Thus, my next step was trying to render a texture directly without the Gstreamer abstraction and texture registry.

My approach was to disable the texture registry code in on_gl_external_texture_frame_callback and add my custom code to return a "fixed" texture.

My expection with this code is to get a texture with random noise/pixels. But I just get a black texture/widget with a few red dots in the top left corner.

Is my approach wrong, or should something like this work? I know that this is no final solution, as I also need a way to tell the flutter engine when a new frame is available. But as per my understanding this should work to get at least a first initial frame.

The printf statements are fired a few times on startup. So Flutter definitely has polled the texture.

I have also tried to experiment with GL_TEXTURE_EXTERNAL_OES and GL_TEXTURE_2D.

When binding the texture with GL_TEXTURE_2D I get the following error from the engine:
[ERROR:flutter/shell/platform/embedder/embedder_external_texture_gl.cc(98)] Could not create external texture->

My code looks like this:

static bool on_gl_external_texture_frame_callback(
    void* userdata,
    int64_t texture_id,
    size_t width,
    size_t height,
    FlutterOpenGLTexture *texture_out
) {
    struct flutterpi *flutterpi;

    printf("on_gl_external_texture_frame_callback - id: %lld \n", texture_id);
    printf("on_gl_external_texture_frame_callback - width: %d, height: %d \n", width, height);
    fflush(stdout);

    DEBUG_ASSERT_NOT_NULL(userdata);

#if 1
    GLuint texture;
    GLubyte* data = (GLubyte*)malloc(width * height * 4 * sizeof(GLubyte));
    // Fill data with texture pixel data
    for (int i = 0; i < width * height * 4; i += 1)
    {
        data[i] = rand() % 256;
    }

    glGenTextures(1, &texture);
    glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture);

    glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);

    // Setup texture coordinate wrapping.
    glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

    glTexImage2D(GL_TEXTURE_EXTERNAL_OES, 0, GL_RGBA8_OES, width, height, 0, GL_RGBA8_OES, GL_UNSIGNED_BYTE, data);


    // Enable texture unit 0.
    glActiveTexture(GL_TEXTURE0);
    glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture);

    free(data);

    //texture_out->target = GL_TEXTURE_EXTERNAL_OES;
    texture_out->target = GL_TEXTURE_2D;
    texture_out->name = texture;
    texture_out->format = GL_RGBA8_OES;
    texture_out->width = width;
    texture_out->height = height;

    return true;

#endif

#if 0
    flutterpi = userdata;

    return texture_registry_gl_external_texture_frame_callback(
        flutterpi->texture_registry,
        texture_id,
        width,
        height,
        texture_out
    );
#endif
}

[ardera]

If I remember correctly, the width and height parameters are used for the texture coordinates. For a normal 2D texture, the upper right corner is gonna be (1, 1) (instead of maybe (800, 480)). If you specified GL_REPEAT, the red dots would probably fill the screen. I'd recommend to leave width and height at zero, then flutter should do the right thing automatically

[berkutta]

Thanks, I got it working in the meantime. The width and height parameters need to be zero indeed.

The code for in the on_gl_external_texture_frame_callback callback if someone wants to do something similar. Important is GL_TEXTURE_2D and the format GL_RGBA8. I don't yet understand how GL_TEXTURE_EXTERNAL_OES textures work..:

    #define GL_RGBA8 0x8058

    printf("on_gl_external_texture_frame_callback - id: %lld \n", texture_id);
    printf("on_gl_external_texture_frame_callback - width: %d, height: %d \n", width, height);
    fflush(stdout);

    GLuint texture;
    GLubyte *data = (GLubyte *)malloc(width * height * 4 * sizeof(GLubyte));

    // Fill data with texture pixel data
    for (int i = 0; i < width * height * 4; i += 4)
    {
        data[i] = rand() % 256;
        data[i+1] = rand() % 256;
        data[i+2] = rand() % 256;
        data[i+3] = 0xFF;
    }

    glGenTextures(1, &texture);
    glBindTexture(GL_TEXTURE_2D, texture);

    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA,
                 GL_UNSIGNED_BYTE, data);

    texture_out->target = GL_TEXTURE_2D;
    texture_out->name = texture;
    texture_out->format = GL_RGBA8;
    //texture_out->destruction_callback = nullptr;
    //texture_out->user_data = nullptr;
    texture_out->width = 0;
    texture_out->height = 0;

Also, it is important to register the Texture to the Flutter Engine:

flutterpi->flutter.libflutter_engine.FlutterEngineRegisterExternalTexture(flutterpi->flutter.engine, 1);

To push a new texture to the Flutter Engine (This results in the callback being called):

flutterpi->flutter.libflutter_engine.FlutterEngineMarkExternalTextureFrameAvailable(flutterpi->flutter.engine, 1);

[ardera]

You can also use:

struct texture *texture = texture_new(texture_registry);

texture_push_frame(
  texture,
  &(const struct gl_texture_frame) {
    .target = GL_TEXTURE_2D,
    ...
  },
  on_texture_destroy,
  texture_userdata
);

Which doesn't call FlutterEngineMarkExternalTextureFrameAvailable if not necessary (otherwise, the frame pipeline will blow up, leading to longer and longer input delay), is mt-safe and generally nicer to use :)

[SwordTechCorp]

@ardera
Hi! Thank you for building this wonderful project!
I was trying to implement the plugin for the texture example from flutter/examples. I think I'm encountering the same problem. Even if I set the height and width to 0 and used texture_push_frame to notify the engine, the widget remains black. I tried changing the formats, commenting out the parameters, following berkutta 's setup, but the widget is always black. Do you have any hints on the issue?

static struct texture* color_texture = NULL;

static uint8_t* pixels = NULL;

static void color_frame_destroy(const struct texture_frame *frame, void *userdata){
    (void) frame;
    (void)(userdata);
    printf("frame_destroy\n");
}

static int on_create(struct platch_obj *object, FlutterPlatformMessageResponseHandle *response_handle) {
    struct std_value *args;

    args = &object->std_arg;
    if (color_texture != NULL){
        return platch_respond_error_std(response_handle, "gl_error","Texture already created.",NULL);
    }
    if (args == NULL ) {
        return platch_respond_illegal_arg_std(response_handle, "Expected `arg` to be a int array with two elements.");
    }

    if (!STDVALUE_IS_LIST(*args)|| args->size != 2){
        return platch_respond_illegal_arg_std(response_handle, "Expected `arg` to be a int array with two elements.");
    }
    if (!STDVALUE_IS_INT(args->list[0]) || !STDVALUE_IS_INT(args->list[1])) {
        return platch_respond_illegal_arg_std(response_handle, "Expected `arg` to be a int array with two elements.");
    }
    GLsizei width = (GLsizei)(STDVALUE_AS_INT(args->list[0]));
    GLsizei height = (GLsizei)(STDVALUE_AS_INT(args->list[1]));

    color_texture = flutterpi_create_texture(flutterpi);
    
    if (color_texture == NULL) {
        return platch_respond_error_std(response_handle, "gl_error","Failed to create texture.",NULL);
    }

    GLuint gl_texture_id;
    pixels = malloc(width * height * 4);
    for (int32_t i = 0; i < width * height; i++) {
        pixels[i*4] = 255;
        pixels[i*4+1] = 0;
        pixels[i*4+2] = 0;
        pixels[i*4+3] = 255;
    }
    glGenTextures(1, &gl_texture_id);
    glBindTexture(GL_TEXTURE_2D, gl_texture_id);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8_OES, width, height, 0, GL_RGBA8_OES, GL_UNSIGNED_BYTE, pixels);
    
    struct texture_frame frame = {
        .gl = {
            .target = GL_TEXTURE_2D,
            .name = gl_texture_id, 
            .format = GL_RGBA8_OES,
            .width = 0,
            .height = 0,
        },
        .destroy = color_frame_destroy, 
        .userdata = pixels,
    };

    // Push the frame to the texture.
    texture_push_frame(color_texture, &frame);

    struct std_value result = {
        .type = kStdInt64,
        .int64_value = texture_get_id(color_texture),
    };

    int ok = platch_respond_success_std(
        response_handle,
        &result
    );

    return ok;
}