Skip to content

Commit

Permalink
[Impeller] Support for rendering Android Platform Views into a Hardwa…
Browse files Browse the repository at this point in the history
…reBuffer backed texture. (#44087)

- Introduce TextureRegistry.ImageTexture and related machinery.
- Introduce ImageReaderPlatformViewRenderTarget.
- Introduce HardwareBufferExternalTextureGL and related machinery.

NOTE: ImageReaderPlatformViewRenderTarget requires Android 26.

NOTE: This CL does not enable using ImageReaderPlatformViewRenderTarget
yet.

Related flutter/flutter#130892
  • Loading branch information
johnmccutchan authored Aug 1, 2023
1 parent afe200d commit 1397257
Show file tree
Hide file tree
Showing 36 changed files with 1,154 additions and 51 deletions.
17 changes: 17 additions & 0 deletions ci/licenses_golden/licenses_flutter
Original file line number Diff line number Diff line change
Expand Up @@ -1686,8 +1686,11 @@ ORIGIN: ../../../flutter/impeller/toolkit/egl/display.cc + ../../../flutter/LICE
ORIGIN: ../../../flutter/impeller/toolkit/egl/display.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/toolkit/egl/egl.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/toolkit/egl/egl.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/toolkit/egl/image.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/toolkit/egl/surface.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/toolkit/egl/surface.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/toolkit/gles/gles.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/toolkit/gles/texture.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_render_context_skia.cc + ../../../flutter/LICENSE
Expand Down Expand Up @@ -2322,6 +2325,8 @@ ORIGIN: ../../../flutter/shell/platform/android/external_view_embedder/surface_p
ORIGIN: ../../../flutter/shell/platform/android/external_view_embedder/surface_pool.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/flutter_main.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/flutter_main.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/Log.java + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/app/FlutterActivity.java + ../../../flutter/LICENSE
Expand Down Expand Up @@ -2463,6 +2468,8 @@ ORIGIN: ../../../flutter/shell/platform/android/jni/jni_mock.h + ../../../flutte
ORIGIN: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/library_loader.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/ndk_helpers.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/ndk_helpers.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/platform_message_handler_android.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/platform_message_response_android.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/platform_message_response_android.h + ../../../flutter/LICENSE
Expand Down Expand Up @@ -4382,8 +4389,13 @@ FILE: ../../../flutter/impeller/toolkit/egl/display.cc
FILE: ../../../flutter/impeller/toolkit/egl/display.h
FILE: ../../../flutter/impeller/toolkit/egl/egl.cc
FILE: ../../../flutter/impeller/toolkit/egl/egl.h
FILE: ../../../flutter/impeller/toolkit/egl/image.cc
FILE: ../../../flutter/impeller/toolkit/egl/image.h
FILE: ../../../flutter/impeller/toolkit/egl/surface.cc
FILE: ../../../flutter/impeller/toolkit/egl/surface.h
FILE: ../../../flutter/impeller/toolkit/gles/gles.h
FILE: ../../../flutter/impeller/toolkit/gles/texture.cc
FILE: ../../../flutter/impeller/toolkit/gles/texture.h
FILE: ../../../flutter/impeller/tools/malioc.json
FILE: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.cc
FILE: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.h
Expand Down Expand Up @@ -5020,6 +5032,8 @@ FILE: ../../../flutter/shell/platform/android/external_view_embedder/surface_poo
FILE: ../../../flutter/shell/platform/android/external_view_embedder/surface_pool.h
FILE: ../../../flutter/shell/platform/android/flutter_main.cc
FILE: ../../../flutter/shell/platform/android/flutter_main.h
FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.cc
FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.h
FILE: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java
FILE: ../../../flutter/shell/platform/android/io/flutter/Log.java
FILE: ../../../flutter/shell/platform/android/io/flutter/app/FlutterActivity.java
Expand Down Expand Up @@ -5137,6 +5151,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/editing/TextInpu
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/localization/LocalizationPlugin.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/mouse/MouseCursorPlugin.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/AccessibilityEventsDelegate.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformOverlayView.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformPlugin.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformView.java
Expand Down Expand Up @@ -5169,6 +5184,8 @@ FILE: ../../../flutter/shell/platform/android/jni/jni_mock.h
FILE: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.cc
FILE: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.h
FILE: ../../../flutter/shell/platform/android/library_loader.cc
FILE: ../../../flutter/shell/platform/android/ndk_helpers.cc
FILE: ../../../flutter/shell/platform/android/ndk_helpers.h
FILE: ../../../flutter/shell/platform/android/platform_message_handler_android.cc
FILE: ../../../flutter/shell/platform/android/platform_message_handler_android.h
FILE: ../../../flutter/shell/platform/android/platform_message_response_android.cc
Expand Down
2 changes: 2 additions & 0 deletions impeller/toolkit/egl/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ impeller_component("egl") {
"display.h",
"egl.cc",
"egl.h",
"image.cc",
"image.h",
"surface.cc",
"surface.h",
]
Expand Down
2 changes: 2 additions & 0 deletions impeller/toolkit/egl/egl.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
#pragma once

#include <EGL/egl.h>
#define EGL_EGLEXT_PROTOTYPES
#include <EGL/eglext.h>

#include <functional>

Expand Down
3 changes: 3 additions & 0 deletions impeller/toolkit/egl/image.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#include "flutter/impeller/toolkit/egl/image.h"

namespace impeller {}
74 changes: 74 additions & 0 deletions impeller/toolkit/egl/image.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#pragma once

#include "flutter/fml/unique_object.h"
#include "flutter/impeller/toolkit/egl/egl.h"

namespace impeller {

// Simple holder of an EGLImage and the owning EGLDisplay.
struct EGLImageWithDisplay {
EGLImage image = EGL_NO_IMAGE;
EGLDisplay display = EGL_NO_DISPLAY;

constexpr bool operator==(const EGLImageWithDisplay& other) const {
return image == other.image && display == other.display;
}

constexpr bool operator!=(const EGLImageWithDisplay& other) const {
return !(*this == other);
}
};

struct EGLImageWithDisplayTraits {
static EGLImageWithDisplay InvalidValue() {
return {EGL_NO_IMAGE, EGL_NO_DISPLAY};
}

static bool IsValid(const EGLImageWithDisplay& value) {
return value != InvalidValue();
}

static void Free(EGLImageWithDisplay image) {
eglDestroyImage(image.display, image.image);
}
};

using UniqueEGLImage =
fml::UniqueObject<EGLImageWithDisplay, EGLImageWithDisplayTraits>;

// Simple holder of an EGLImageKHR and the owning EGLDisplay.
struct EGLImageKHRWithDisplay {
EGLImageKHR image = EGL_NO_IMAGE_KHR;
EGLDisplay display = EGL_NO_DISPLAY;

constexpr bool operator==(const EGLImageKHRWithDisplay& other) const {
return image == other.image && display == other.display;
}

constexpr bool operator!=(const EGLImageKHRWithDisplay& other) const {
return !(*this == other);
}
};

struct EGLImageKHRWithDisplayTraits {
static EGLImageKHRWithDisplay InvalidValue() {
return {EGL_NO_IMAGE_KHR, EGL_NO_DISPLAY};
}

static bool IsValid(const EGLImageKHRWithDisplay& value) {
return value != InvalidValue();
}

static void Free(EGLImageKHRWithDisplay image) {
eglDestroyImageKHR(image.display, image.image);
}
};

using UniqueEGLImageKHR =
fml::UniqueObject<EGLImageKHRWithDisplay, EGLImageKHRWithDisplayTraits>;

} // namespace impeller
20 changes: 20 additions & 0 deletions impeller/toolkit/gles/BUILD.gn
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import("../../tools/impeller.gni")

impeller_component("gles") {
sources = [
"gles.h",
"texture.cc",
"texture.h",
]

deps = [
"../../base",
"//flutter/fml",
]

libs = []
}
9 changes: 9 additions & 0 deletions impeller/toolkit/gles/gles.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#pragma once

#include "GLES3/gl3.h"
#define GL_GLEXT_PROTOTYPES
#include "GLES2/gl2ext.h"
3 changes: 3 additions & 0 deletions impeller/toolkit/gles/texture.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#include "flutter/impeller/toolkit/gles/texture.h"

namespace impeller {}
39 changes: 39 additions & 0 deletions impeller/toolkit/gles/texture.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#pragma once

#include "flutter/fml/unique_object.h"
#include "flutter/impeller/toolkit/gles/gles.h"

namespace impeller {

// Simple holder of an GLTexture and the owning EGLDisplay.
struct GLTexture {
GLuint texture_name;

constexpr bool operator==(const GLTexture& other) const {
return texture_name == other.texture_name;
}

constexpr bool operator!=(const GLTexture& other) const {
return !(*this == other);
}
};

struct GLTextureTraits {
static GLTexture InvalidValue() { return {0}; }

static bool IsValid(const GLTexture& value) {
return value != InvalidValue();
}

static void Free(GLTexture image) {
glDeleteTextures(1, &image.texture_name);
}
};

using UniqueGLTexture = fml::UniqueObject<GLTexture, GLTextureTraits>;

} // namespace impeller
6 changes: 6 additions & 0 deletions shell/platform/android/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,11 @@ source_set("flutter_shell_native_src") {
"apk_asset_provider.h",
"flutter_main.cc",
"flutter_main.h",
"hardware_buffer_external_texture_gl.cc",
"hardware_buffer_external_texture_gl.h",
"library_loader.cc",
"ndk_helpers.cc",
"ndk_helpers.h",
"platform_message_handler_android.cc",
"platform_message_handler_android.h",
"platform_message_response_android.cc",
Expand All @@ -122,6 +126,7 @@ source_set("flutter_shell_native_src") {
"//flutter/fml",
"//flutter/impeller",
"//flutter/impeller/toolkit/egl",
"//flutter/impeller/toolkit/gles",
"//flutter/lib/ui",
"//flutter/runtime",
"//flutter/runtime:libdart",
Expand Down Expand Up @@ -287,6 +292,7 @@ android_java_sources = [
"io/flutter/plugin/localization/LocalizationPlugin.java",
"io/flutter/plugin/mouse/MouseCursorPlugin.java",
"io/flutter/plugin/platform/AccessibilityEventsDelegate.java",
"io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java",
"io/flutter/plugin/platform/PlatformOverlayView.java",
"io/flutter/plugin/platform/PlatformPlugin.java",
"io/flutter/plugin/platform/PlatformView.java",
Expand Down
8 changes: 8 additions & 0 deletions shell/platform/android/android_context_gl_skia.cc
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,14 @@ bool AndroidContextGLSkia::ClearCurrent() const {
return true;
}

EGLContext AndroidContextGLSkia::GetEGLContext() const {
return context_;
}

EGLDisplay AndroidContextGLSkia::GetEGLDisplay() const {
return environment_->Display();
}

EGLContext AndroidContextGLSkia::CreateNewContext() const {
bool success;
EGLContext context;
Expand Down
14 changes: 14 additions & 0 deletions shell/platform/android/android_context_gl_skia.h
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,20 @@ class AndroidContextGLSkia : public AndroidContext {
///
bool ClearCurrent() const;

//----------------------------------------------------------------------------
/// @brief Returns the EGLContext.
///
/// @return EGLContext.
///
EGLContext GetEGLContext() const;

//----------------------------------------------------------------------------
/// @brief Returns the EGLDisplay.
///
/// @return EGLDisplay.
///
EGLDisplay GetEGLDisplay() const;

//----------------------------------------------------------------------------
/// @brief Create a new EGLContext using the same EGLConfig.
///
Expand Down
14 changes: 7 additions & 7 deletions shell/platform/android/android_external_texture_gl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@ AndroidExternalTextureGL::AndroidExternalTextureGL(
transform(SkMatrix::I()) {}

AndroidExternalTextureGL::~AndroidExternalTextureGL() {
if (state_ == AttachmentState::attached) {
if (state_ == AttachmentState::kAttached) {
glDeleteTextures(1, &texture_name_);
}
}

void AndroidExternalTextureGL::OnGrContextCreated() {
state_ = AttachmentState::uninitialized;
state_ = AttachmentState::kUninitialized;
}

void AndroidExternalTextureGL::MarkNewFrameAvailable() {
Expand All @@ -46,13 +46,13 @@ void AndroidExternalTextureGL::Paint(PaintContext& context,
const SkRect& bounds,
bool freeze,
const DlImageSampling sampling) {
if (state_ == AttachmentState::detached) {
if (state_ == AttachmentState::kDetached) {
return;
}
if (state_ == AttachmentState::uninitialized) {
if (state_ == AttachmentState::kUninitialized) {
glGenTextures(1, &texture_name_);
Attach(static_cast<jint>(texture_name_));
state_ = AttachmentState::attached;
state_ = AttachmentState::kAttached;
}
if (!freeze && new_frame_ready_) {
Update();
Expand Down Expand Up @@ -108,11 +108,11 @@ void AndroidExternalTextureGL::UpdateTransform() {
}

void AndroidExternalTextureGL::OnGrContextDestroyed() {
if (state_ == AttachmentState::attached) {
if (state_ == AttachmentState::kAttached) {
Detach();
glDeleteTextures(1, &texture_name_);
}
state_ = AttachmentState::detached;
state_ = AttachmentState::kDetached;
}

void AndroidExternalTextureGL::Attach(jint textureName) {
Expand Down
4 changes: 2 additions & 2 deletions shell/platform/android/android_external_texture_gl.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,13 @@ class AndroidExternalTextureGL : public flutter::Texture {

void UpdateTransform();

enum class AttachmentState { uninitialized, attached, detached };
enum class AttachmentState { kUninitialized, kAttached, kDetached };

std::shared_ptr<PlatformViewAndroidJNI> jni_facade_;

fml::jni::ScopedJavaGlobalRef<jobject> surface_texture_;

AttachmentState state_ = AttachmentState::uninitialized;
AttachmentState state_ = AttachmentState::kUninitialized;

bool new_frame_ready_ = false;

Expand Down
6 changes: 6 additions & 0 deletions shell/platform/android/android_shell_holder_unittests.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include "flutter/shell/platform/android/android_shell_holder.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "shell/platform/android/jni/platform_view_android_jni.h"

namespace flutter {
namespace testing {
Expand Down Expand Up @@ -30,6 +31,11 @@ class MockPlatformViewAndroidJNI : public PlatformViewAndroidJNI {
void(JavaLocalRef surface_texture, SkMatrix& transform));
MOCK_METHOD1(SurfaceTextureDetachFromGLContext,
void(JavaLocalRef surface_texture));
MOCK_METHOD1(ImageTextureEntryAcquireLatestImage,
JavaLocalRef(JavaLocalRef image_texture_entry));
MOCK_METHOD1(ImageGetHardwareBuffer, JavaLocalRef(JavaLocalRef image));
MOCK_METHOD1(ImageClose, void(JavaLocalRef image));
MOCK_METHOD1(HardwareBufferClose, void(JavaLocalRef hardware_buffer));
MOCK_METHOD8(FlutterViewOnDisplayPlatformView,
void(int view_id,
int x,
Expand Down
Loading

0 comments on commit 1397257

Please sign in to comment.