diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 95fc04d7caffa..dc1f98f635e79 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1686,8 +1686,11 @@ ORIGIN: ../../../flutter/impeller/toolkit/egl/display.cc + ../../../flutter/LICE ORIGIN: ../../../flutter/impeller/toolkit/egl/display.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/toolkit/egl/egl.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/toolkit/egl/egl.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/toolkit/egl/image.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/toolkit/egl/surface.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/toolkit/egl/surface.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/toolkit/gles/gles.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/toolkit/gles/texture.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/typographer/backends/skia/text_render_context_skia.cc + ../../../flutter/LICENSE @@ -2322,6 +2325,8 @@ ORIGIN: ../../../flutter/shell/platform/android/external_view_embedder/surface_p ORIGIN: ../../../flutter/shell/platform/android/external_view_embedder/surface_pool.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/flutter_main.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/flutter_main.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/Log.java + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/app/FlutterActivity.java + ../../../flutter/LICENSE @@ -2463,6 +2468,8 @@ ORIGIN: ../../../flutter/shell/platform/android/jni/jni_mock.h + ../../../flutte ORIGIN: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/library_loader.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/ndk_helpers.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/ndk_helpers.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/platform_message_handler_android.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/platform_message_response_android.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/platform_message_response_android.h + ../../../flutter/LICENSE @@ -4382,8 +4389,13 @@ FILE: ../../../flutter/impeller/toolkit/egl/display.cc FILE: ../../../flutter/impeller/toolkit/egl/display.h FILE: ../../../flutter/impeller/toolkit/egl/egl.cc FILE: ../../../flutter/impeller/toolkit/egl/egl.h +FILE: ../../../flutter/impeller/toolkit/egl/image.cc +FILE: ../../../flutter/impeller/toolkit/egl/image.h FILE: ../../../flutter/impeller/toolkit/egl/surface.cc FILE: ../../../flutter/impeller/toolkit/egl/surface.h +FILE: ../../../flutter/impeller/toolkit/gles/gles.h +FILE: ../../../flutter/impeller/toolkit/gles/texture.cc +FILE: ../../../flutter/impeller/toolkit/gles/texture.h FILE: ../../../flutter/impeller/tools/malioc.json FILE: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.cc FILE: ../../../flutter/impeller/typographer/backends/skia/text_frame_skia.h @@ -5020,6 +5032,8 @@ FILE: ../../../flutter/shell/platform/android/external_view_embedder/surface_poo FILE: ../../../flutter/shell/platform/android/external_view_embedder/surface_pool.h FILE: ../../../flutter/shell/platform/android/flutter_main.cc FILE: ../../../flutter/shell/platform/android/flutter_main.h +FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.cc +FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.h FILE: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java FILE: ../../../flutter/shell/platform/android/io/flutter/Log.java FILE: ../../../flutter/shell/platform/android/io/flutter/app/FlutterActivity.java @@ -5137,6 +5151,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/editing/TextInpu FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/localization/LocalizationPlugin.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/mouse/MouseCursorPlugin.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/AccessibilityEventsDelegate.java +FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformOverlayView.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformPlugin.java FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformView.java @@ -5169,6 +5184,8 @@ FILE: ../../../flutter/shell/platform/android/jni/jni_mock.h FILE: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.cc FILE: ../../../flutter/shell/platform/android/jni/platform_view_android_jni.h FILE: ../../../flutter/shell/platform/android/library_loader.cc +FILE: ../../../flutter/shell/platform/android/ndk_helpers.cc +FILE: ../../../flutter/shell/platform/android/ndk_helpers.h FILE: ../../../flutter/shell/platform/android/platform_message_handler_android.cc FILE: ../../../flutter/shell/platform/android/platform_message_handler_android.h FILE: ../../../flutter/shell/platform/android/platform_message_response_android.cc diff --git a/impeller/toolkit/egl/BUILD.gn b/impeller/toolkit/egl/BUILD.gn index ec34bfead9829..82d10a14d195e 100644 --- a/impeller/toolkit/egl/BUILD.gn +++ b/impeller/toolkit/egl/BUILD.gn @@ -14,6 +14,8 @@ impeller_component("egl") { "display.h", "egl.cc", "egl.h", + "image.cc", + "image.h", "surface.cc", "surface.h", ] diff --git a/impeller/toolkit/egl/egl.h b/impeller/toolkit/egl/egl.h index 63c57075d63a9..af1bcca52ffb3 100644 --- a/impeller/toolkit/egl/egl.h +++ b/impeller/toolkit/egl/egl.h @@ -5,6 +5,8 @@ #pragma once #include +#define EGL_EGLEXT_PROTOTYPES +#include #include diff --git a/impeller/toolkit/egl/image.cc b/impeller/toolkit/egl/image.cc new file mode 100644 index 0000000000000..c1f17ebddc960 --- /dev/null +++ b/impeller/toolkit/egl/image.cc @@ -0,0 +1,3 @@ +#include "flutter/impeller/toolkit/egl/image.h" + +namespace impeller {} diff --git a/impeller/toolkit/egl/image.h b/impeller/toolkit/egl/image.h new file mode 100644 index 0000000000000..2fafe0f994b82 --- /dev/null +++ b/impeller/toolkit/egl/image.h @@ -0,0 +1,74 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "flutter/fml/unique_object.h" +#include "flutter/impeller/toolkit/egl/egl.h" + +namespace impeller { + +// Simple holder of an EGLImage and the owning EGLDisplay. +struct EGLImageWithDisplay { + EGLImage image = EGL_NO_IMAGE; + EGLDisplay display = EGL_NO_DISPLAY; + + constexpr bool operator==(const EGLImageWithDisplay& other) const { + return image == other.image && display == other.display; + } + + constexpr bool operator!=(const EGLImageWithDisplay& other) const { + return !(*this == other); + } +}; + +struct EGLImageWithDisplayTraits { + static EGLImageWithDisplay InvalidValue() { + return {EGL_NO_IMAGE, EGL_NO_DISPLAY}; + } + + static bool IsValid(const EGLImageWithDisplay& value) { + return value != InvalidValue(); + } + + static void Free(EGLImageWithDisplay image) { + eglDestroyImage(image.display, image.image); + } +}; + +using UniqueEGLImage = + fml::UniqueObject; + +// Simple holder of an EGLImageKHR and the owning EGLDisplay. +struct EGLImageKHRWithDisplay { + EGLImageKHR image = EGL_NO_IMAGE_KHR; + EGLDisplay display = EGL_NO_DISPLAY; + + constexpr bool operator==(const EGLImageKHRWithDisplay& other) const { + return image == other.image && display == other.display; + } + + constexpr bool operator!=(const EGLImageKHRWithDisplay& other) const { + return !(*this == other); + } +}; + +struct EGLImageKHRWithDisplayTraits { + static EGLImageKHRWithDisplay InvalidValue() { + return {EGL_NO_IMAGE_KHR, EGL_NO_DISPLAY}; + } + + static bool IsValid(const EGLImageKHRWithDisplay& value) { + return value != InvalidValue(); + } + + static void Free(EGLImageKHRWithDisplay image) { + eglDestroyImageKHR(image.display, image.image); + } +}; + +using UniqueEGLImageKHR = + fml::UniqueObject; + +} // namespace impeller diff --git a/impeller/toolkit/gles/BUILD.gn b/impeller/toolkit/gles/BUILD.gn new file mode 100644 index 0000000000000..06a16f95bf354 --- /dev/null +++ b/impeller/toolkit/gles/BUILD.gn @@ -0,0 +1,20 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("../../tools/impeller.gni") + +impeller_component("gles") { + sources = [ + "gles.h", + "texture.cc", + "texture.h", + ] + + deps = [ + "../../base", + "//flutter/fml", + ] + + libs = [] +} diff --git a/impeller/toolkit/gles/gles.h b/impeller/toolkit/gles/gles.h new file mode 100644 index 0000000000000..599e3c9433c97 --- /dev/null +++ b/impeller/toolkit/gles/gles.h @@ -0,0 +1,9 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "GLES3/gl3.h" +#define GL_GLEXT_PROTOTYPES +#include "GLES2/gl2ext.h" diff --git a/impeller/toolkit/gles/texture.cc b/impeller/toolkit/gles/texture.cc new file mode 100644 index 0000000000000..56f3d5b7b746f --- /dev/null +++ b/impeller/toolkit/gles/texture.cc @@ -0,0 +1,3 @@ +#include "flutter/impeller/toolkit/gles/texture.h" + +namespace impeller {} diff --git a/impeller/toolkit/gles/texture.h b/impeller/toolkit/gles/texture.h new file mode 100644 index 0000000000000..b90007b414237 --- /dev/null +++ b/impeller/toolkit/gles/texture.h @@ -0,0 +1,39 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "flutter/fml/unique_object.h" +#include "flutter/impeller/toolkit/gles/gles.h" + +namespace impeller { + +// Simple holder of an GLTexture and the owning EGLDisplay. +struct GLTexture { + GLuint texture_name; + + constexpr bool operator==(const GLTexture& other) const { + return texture_name == other.texture_name; + } + + constexpr bool operator!=(const GLTexture& other) const { + return !(*this == other); + } +}; + +struct GLTextureTraits { + static GLTexture InvalidValue() { return {0}; } + + static bool IsValid(const GLTexture& value) { + return value != InvalidValue(); + } + + static void Free(GLTexture image) { + glDeleteTextures(1, &image.texture_name); + } +}; + +using UniqueGLTexture = fml::UniqueObject; + +} // namespace impeller diff --git a/shell/platform/android/BUILD.gn b/shell/platform/android/BUILD.gn index 80fd97c3991f6..d2e201ea9f184 100644 --- a/shell/platform/android/BUILD.gn +++ b/shell/platform/android/BUILD.gn @@ -98,7 +98,11 @@ source_set("flutter_shell_native_src") { "apk_asset_provider.h", "flutter_main.cc", "flutter_main.h", + "hardware_buffer_external_texture_gl.cc", + "hardware_buffer_external_texture_gl.h", "library_loader.cc", + "ndk_helpers.cc", + "ndk_helpers.h", "platform_message_handler_android.cc", "platform_message_handler_android.h", "platform_message_response_android.cc", @@ -122,6 +126,7 @@ source_set("flutter_shell_native_src") { "//flutter/fml", "//flutter/impeller", "//flutter/impeller/toolkit/egl", + "//flutter/impeller/toolkit/gles", "//flutter/lib/ui", "//flutter/runtime", "//flutter/runtime:libdart", @@ -287,6 +292,7 @@ android_java_sources = [ "io/flutter/plugin/localization/LocalizationPlugin.java", "io/flutter/plugin/mouse/MouseCursorPlugin.java", "io/flutter/plugin/platform/AccessibilityEventsDelegate.java", + "io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java", "io/flutter/plugin/platform/PlatformOverlayView.java", "io/flutter/plugin/platform/PlatformPlugin.java", "io/flutter/plugin/platform/PlatformView.java", diff --git a/shell/platform/android/android_context_gl_skia.cc b/shell/platform/android/android_context_gl_skia.cc index adc29695d7347..8dcdeb5b62fab 100644 --- a/shell/platform/android/android_context_gl_skia.cc +++ b/shell/platform/android/android_context_gl_skia.cc @@ -205,6 +205,14 @@ bool AndroidContextGLSkia::ClearCurrent() const { return true; } +EGLContext AndroidContextGLSkia::GetEGLContext() const { + return context_; +} + +EGLDisplay AndroidContextGLSkia::GetEGLDisplay() const { + return environment_->Display(); +} + EGLContext AndroidContextGLSkia::CreateNewContext() const { bool success; EGLContext context; diff --git a/shell/platform/android/android_context_gl_skia.h b/shell/platform/android/android_context_gl_skia.h index 145be6144b218..1c9e092ff3a33 100644 --- a/shell/platform/android/android_context_gl_skia.h +++ b/shell/platform/android/android_context_gl_skia.h @@ -76,6 +76,20 @@ class AndroidContextGLSkia : public AndroidContext { /// bool ClearCurrent() const; + //---------------------------------------------------------------------------- + /// @brief Returns the EGLContext. + /// + /// @return EGLContext. + /// + EGLContext GetEGLContext() const; + + //---------------------------------------------------------------------------- + /// @brief Returns the EGLDisplay. + /// + /// @return EGLDisplay. + /// + EGLDisplay GetEGLDisplay() const; + //---------------------------------------------------------------------------- /// @brief Create a new EGLContext using the same EGLConfig. /// diff --git a/shell/platform/android/android_external_texture_gl.cc b/shell/platform/android/android_external_texture_gl.cc index 3e40b99c3d582..472c426747299 100644 --- a/shell/platform/android/android_external_texture_gl.cc +++ b/shell/platform/android/android_external_texture_gl.cc @@ -29,13 +29,13 @@ AndroidExternalTextureGL::AndroidExternalTextureGL( transform(SkMatrix::I()) {} AndroidExternalTextureGL::~AndroidExternalTextureGL() { - if (state_ == AttachmentState::attached) { + if (state_ == AttachmentState::kAttached) { glDeleteTextures(1, &texture_name_); } } void AndroidExternalTextureGL::OnGrContextCreated() { - state_ = AttachmentState::uninitialized; + state_ = AttachmentState::kUninitialized; } void AndroidExternalTextureGL::MarkNewFrameAvailable() { @@ -46,13 +46,13 @@ void AndroidExternalTextureGL::Paint(PaintContext& context, const SkRect& bounds, bool freeze, const DlImageSampling sampling) { - if (state_ == AttachmentState::detached) { + if (state_ == AttachmentState::kDetached) { return; } - if (state_ == AttachmentState::uninitialized) { + if (state_ == AttachmentState::kUninitialized) { glGenTextures(1, &texture_name_); Attach(static_cast(texture_name_)); - state_ = AttachmentState::attached; + state_ = AttachmentState::kAttached; } if (!freeze && new_frame_ready_) { Update(); @@ -108,11 +108,11 @@ void AndroidExternalTextureGL::UpdateTransform() { } void AndroidExternalTextureGL::OnGrContextDestroyed() { - if (state_ == AttachmentState::attached) { + if (state_ == AttachmentState::kAttached) { Detach(); glDeleteTextures(1, &texture_name_); } - state_ = AttachmentState::detached; + state_ = AttachmentState::kDetached; } void AndroidExternalTextureGL::Attach(jint textureName) { diff --git a/shell/platform/android/android_external_texture_gl.h b/shell/platform/android/android_external_texture_gl.h index d1c8f08be3d2b..ee2a88358ff5d 100644 --- a/shell/platform/android/android_external_texture_gl.h +++ b/shell/platform/android/android_external_texture_gl.h @@ -43,13 +43,13 @@ class AndroidExternalTextureGL : public flutter::Texture { void UpdateTransform(); - enum class AttachmentState { uninitialized, attached, detached }; + enum class AttachmentState { kUninitialized, kAttached, kDetached }; std::shared_ptr jni_facade_; fml::jni::ScopedJavaGlobalRef surface_texture_; - AttachmentState state_ = AttachmentState::uninitialized; + AttachmentState state_ = AttachmentState::kUninitialized; bool new_frame_ready_ = false; diff --git a/shell/platform/android/android_shell_holder_unittests.cc b/shell/platform/android/android_shell_holder_unittests.cc index fa138c920371b..831abc2cd6269 100644 --- a/shell/platform/android/android_shell_holder_unittests.cc +++ b/shell/platform/android/android_shell_holder_unittests.cc @@ -2,6 +2,7 @@ #include "flutter/shell/platform/android/android_shell_holder.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "shell/platform/android/jni/platform_view_android_jni.h" namespace flutter { namespace testing { @@ -30,6 +31,11 @@ class MockPlatformViewAndroidJNI : public PlatformViewAndroidJNI { void(JavaLocalRef surface_texture, SkMatrix& transform)); MOCK_METHOD1(SurfaceTextureDetachFromGLContext, void(JavaLocalRef surface_texture)); + MOCK_METHOD1(ImageTextureEntryAcquireLatestImage, + JavaLocalRef(JavaLocalRef image_texture_entry)); + MOCK_METHOD1(ImageGetHardwareBuffer, JavaLocalRef(JavaLocalRef image)); + MOCK_METHOD1(ImageClose, void(JavaLocalRef image)); + MOCK_METHOD1(HardwareBufferClose, void(JavaLocalRef hardware_buffer)); MOCK_METHOD8(FlutterViewOnDisplayPlatformView, void(int view_id, int x, diff --git a/shell/platform/android/hardware_buffer_external_texture_gl.cc b/shell/platform/android/hardware_buffer_external_texture_gl.cc new file mode 100644 index 0000000000000..1b513917c1643 --- /dev/null +++ b/shell/platform/android/hardware_buffer_external_texture_gl.cc @@ -0,0 +1,164 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/android/hardware_buffer_external_texture_gl.h" + +#include +#include +#include "impeller/toolkit/egl/image.h" +#include "impeller/toolkit/gles/texture.h" +#include "shell/platform/android/ndk_helpers.h" + +#include "flutter/display_list/effects/dl_color_source.h" +#include "third_party/skia/include/core/SkAlphaType.h" +#include "third_party/skia/include/core/SkColorSpace.h" +#include "third_party/skia/include/core/SkColorType.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/gpu/GrBackendSurface.h" +#include "third_party/skia/include/gpu/GrDirectContext.h" +#include "third_party/skia/include/gpu/ganesh/SkImageGanesh.h" + +namespace flutter { + +HardwareBufferExternalTextureGL::HardwareBufferExternalTextureGL( + const std::shared_ptr& context, + int64_t id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry, + const std::shared_ptr& jni_facade) + : Texture(id), + context_(context), + image_texture_entry_(image_texture_entry), + jni_facade_(jni_facade) {} + +HardwareBufferExternalTextureGL::~HardwareBufferExternalTextureGL() {} + +// Implementing flutter::Texture. +void HardwareBufferExternalTextureGL::Paint(PaintContext& context, + const SkRect& bounds, + bool freeze, + const DlImageSampling sampling) { + if (state_ == AttachmentState::kDetached) { + return; + } + if (state_ == AttachmentState::kUninitialized) { + GLuint texture_name; + glGenTextures(1, &texture_name); + texture_.reset(impeller::GLTexture{texture_name}); + state_ = AttachmentState::kAttached; + } + glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture_.get().texture_name); + if (!freeze && new_frame_ready_) { + new_frame_ready_ = false; + Update(); + } + GrGLTextureInfo textureInfo = {GL_TEXTURE_EXTERNAL_OES, + texture_.get().texture_name, GL_RGBA8_OES}; + GrBackendTexture backendTexture(1, 1, GrMipMapped::kNo, textureInfo); + sk_sp image = SkImages::BorrowTextureFrom( + context.gr_context, backendTexture, kTopLeft_GrSurfaceOrigin, + kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr); + if (image) { + DlAutoCanvasRestore autoRestore(context.canvas, true); + context.canvas->Translate(bounds.x(), bounds.y()); + context.canvas->Scale(bounds.width(), bounds.height()); + auto dl_image = DlImage::Make(image); + context.canvas->DrawImage(dl_image, {0, 0}, sampling, context.paint); + } else { + FML_LOG(ERROR) << "Skia could not borrow texture"; + } +} + +// Implementing flutter::Texture. +void HardwareBufferExternalTextureGL::MarkNewFrameAvailable() { + new_frame_ready_ = true; +} + +// Implementing flutter::Texture. +void HardwareBufferExternalTextureGL::OnTextureUnregistered() {} + +// Implementing flutter::ContextListener. +void HardwareBufferExternalTextureGL::OnGrContextCreated() { + state_ = AttachmentState::kUninitialized; +} + +AHardwareBuffer* HardwareBufferExternalTextureGL::GetLatestHardwareBuffer() { + JNIEnv* env = fml::jni::AttachCurrentThread(); + FML_CHECK(env != nullptr); + + // ImageTextureEntry.acquireLatestImage. + JavaLocalRef image_java = jni_facade_->ImageTextureEntryAcquireLatestImage( + JavaLocalRef(image_texture_entry_)); + if (image_java.obj() == nullptr) { + return nullptr; + } + + // Image.getHardwareBuffer. + JavaLocalRef hardware_buffer_java = + jni_facade_->ImageGetHardwareBuffer(image_java); + if (hardware_buffer_java.obj() == nullptr) { + jni_facade_->ImageClose(image_java); + return nullptr; + } + + // Convert into NDK HardwareBuffer. + AHardwareBuffer* latest_hardware_buffer = + NDKHelpers::AHardwareBuffer_fromHardwareBuffer( + env, hardware_buffer_java.obj()); + if (latest_hardware_buffer == nullptr) { + return nullptr; + } + + // Keep hardware buffer alive. + NDKHelpers::AHardwareBuffer_acquire(latest_hardware_buffer); + + // Now that we have referenced the native hardware buffer, close the Java + // Image and HardwareBuffer objects. + jni_facade_->HardwareBufferClose(hardware_buffer_java); + jni_facade_->ImageClose(image_java); + + return latest_hardware_buffer; +} + +void HardwareBufferExternalTextureGL::Update() { + EGLDisplay display = eglGetCurrentDisplay(); + FML_CHECK(display != EGL_NO_DISPLAY); + + image_.reset(); + + AHardwareBuffer* latest_hardware_buffer = GetLatestHardwareBuffer(); + if (latest_hardware_buffer == nullptr) { + FML_LOG(WARNING) << "GetLatestHardwareBuffer returned null."; + return; + } + + EGLClientBuffer client_buffer = + NDKHelpers::eglGetNativeClientBufferANDROID(latest_hardware_buffer); + if (client_buffer == nullptr) { + FML_LOG(WARNING) << "eglGetNativeClientBufferAndroid returned null."; + return; + } + FML_CHECK(client_buffer != nullptr); + image_.reset(impeller::EGLImageKHRWithDisplay{ + eglCreateImageKHR(display, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, + client_buffer, 0), + display}); + FML_CHECK(image_.get().image != EGL_NO_IMAGE_KHR); + glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, + (GLeglImageOES)image_.get().image); + + // Drop our temporary reference to the hardware buffer as the call to + // eglCreateImageKHR now has the reference. + NDKHelpers::AHardwareBuffer_release(latest_hardware_buffer); +} + +// Implementing flutter::ContextListener. +void HardwareBufferExternalTextureGL::OnGrContextDestroyed() { + if (state_ == AttachmentState::kAttached) { + image_.reset(); + texture_.reset(); + } + state_ = AttachmentState::kDetached; +} + +} // namespace flutter diff --git a/shell/platform/android/hardware_buffer_external_texture_gl.h b/shell/platform/android/hardware_buffer_external_texture_gl.h new file mode 100644 index 0000000000000..6890f5bcb3007 --- /dev/null +++ b/shell/platform/android/hardware_buffer_external_texture_gl.h @@ -0,0 +1,71 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ +#define FLUTTER_SHELL_PLATFORM_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ + +#include + +#include "flutter/impeller/toolkit/egl/egl.h" +#include "flutter/impeller/toolkit/egl/image.h" +#include "flutter/impeller/toolkit/gles/texture.h" + +#include +#include + +#include "flutter/common/graphics/texture.h" +#include "flutter/fml/logging.h" +#include "flutter/shell/platform/android/android_context_gl_skia.h" +#include "flutter/shell/platform/android/platform_view_android_jni_impl.h" + +namespace flutter { + +class HardwareBufferExternalTextureGL : public flutter::Texture { + public: + explicit HardwareBufferExternalTextureGL( + const std::shared_ptr& context, + int64_t id, + const fml::jni::ScopedJavaGlobalRef& + hardware_buffer_texture_entry, + const std::shared_ptr& jni_facade); + ~HardwareBufferExternalTextureGL() override; + + // |flutter::Texture|. + void Paint(PaintContext& context, + const SkRect& bounds, + bool freeze, + const DlImageSampling sampling) override; + + // |flutter::Texture|. + void MarkNewFrameAvailable() override; + + // |flutter::Texture| + void OnTextureUnregistered() override; + + // |flutter::ContextListener| + void OnGrContextCreated() override; + + // |flutter::ContextListener| + void OnGrContextDestroyed() override; + + private: + AHardwareBuffer* GetLatestHardwareBuffer(); + void Update(); + + const std::shared_ptr context_; + fml::jni::ScopedJavaGlobalRef image_texture_entry_; + std::shared_ptr jni_facade_; + + enum class AttachmentState { kUninitialized, kAttached, kDetached }; + AttachmentState state_ = AttachmentState::kUninitialized; + bool new_frame_ready_ = false; + impeller::UniqueEGLImageKHR image_; + impeller::UniqueGLTexture texture_; + + FML_DISALLOW_COPY_AND_ASSIGN(HardwareBufferExternalTextureGL); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_ANDROID_EXTERNAL_TEXTURE_GL_H_ diff --git a/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java b/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java index 4471d73f0f098..d579f1fceb108 100644 --- a/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java +++ b/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java @@ -33,6 +33,7 @@ import io.flutter.util.Preconditions; import io.flutter.view.AccessibilityBridge; import io.flutter.view.FlutterCallbackInformation; +import io.flutter.view.TextureRegistry; import java.io.IOException; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; @@ -901,6 +902,27 @@ private native void nativeRegisterTexture( long textureId, @NonNull WeakReference textureWrapper); + /** + * Registers a ImageTexture with the given id. + * + *

REQUIRED: Callers should eventually unregisterTexture with the same id. + */ + @UiThread + public void registerImageTexture( + long textureId, @NonNull TextureRegistry.ImageTextureEntry imageTextureEntry) { + ensureRunningOnMainThread(); + ensureAttachedToNative(); + nativeRegisterImageTexture( + nativeShellHolderId, + textureId, + new WeakReference(imageTextureEntry)); + } + + private native void nativeRegisterImageTexture( + long nativeShellHolderId, + long textureId, + @NonNull WeakReference imageTextureEntry); + /** * Call this method to inform Flutter that a texture previously registered with {@link * #registerTexture(long, SurfaceTextureWrapper)} has a new frame available. diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java index 1eac92546a659..423a355038c4a 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java @@ -4,9 +4,11 @@ package io.flutter.embedding.engine.renderer; +import android.annotation.TargetApi; import android.graphics.Bitmap; import android.graphics.Rect; import android.graphics.SurfaceTexture; +import android.media.Image; import android.os.Build; import android.os.Handler; import android.view.Surface; @@ -15,7 +17,9 @@ import androidx.annotation.VisibleForTesting; import io.flutter.Log; import io.flutter.embedding.engine.FlutterJNI; +import io.flutter.embedding.engine.renderer.FlutterRenderer.ImageTextureRegistryEntry; import io.flutter.view.TextureRegistry; +import io.flutter.view.TextureRegistry.ImageTextureEntry; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -158,6 +162,15 @@ public SurfaceTextureEntry registerSurfaceTexture(@NonNull SurfaceTexture surfac return entry; } + @Override + public ImageTextureEntry createImageTexture() { + final ImageTextureRegistryEntry entry = + new ImageTextureRegistryEntry(nextTextureId.getAndIncrement()); + Log.v(TAG, "New ImageTextureEntry ID: " + entry.id()); + registerImageTexture(entry.id(), entry); + return entry; + } + @Override public void onTrimMemory(int level) { final Iterator> iterator = onTrimMemoryListeners.iterator(); @@ -270,7 +283,7 @@ protected void finalize() throws Throwable { return; } - handler.post(new SurfaceTextureFinalizerRunnable(id, flutterJNI)); + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); } finally { super.finalize(); } @@ -287,11 +300,11 @@ public void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) { } } - static final class SurfaceTextureFinalizerRunnable implements Runnable { + static final class TextureFinalizerRunnable implements Runnable { private final long id; private final FlutterJNI flutterJNI; - SurfaceTextureFinalizerRunnable(long id, @NonNull FlutterJNI flutterJNI) { + TextureFinalizerRunnable(long id, @NonNull FlutterJNI flutterJNI) { this.id = id; this.flutterJNI = flutterJNI; } @@ -301,10 +314,73 @@ public void run() { if (!flutterJNI.isAttached()) { return; } - Log.v(TAG, "Releasing a SurfaceTexture (" + id + ")."); + Log.v(TAG, "Releasing a Texture (" + id + ")."); flutterJNI.unregisterTexture(id); } } + + final class ImageTextureRegistryEntry implements TextureRegistry.ImageTextureEntry { + private final long id; + private boolean released; + private Image image; + + ImageTextureRegistryEntry(long id) { + this.id = id; + } + + @Override + public long id() { + return id; + } + + @Override + public void release() { + if (released) { + return; + } + released = true; + unregisterTexture(id); + } + + @Override + @TargetApi(19) + public void pushImage(Image image) { + Image toClose; + synchronized (this) { + toClose = this.image; + this.image = image; + } + // Close the previously pushed buffer. + if (toClose != null) { + toClose.close(); + } + // Mark that we have a new frame available. + markTextureFrameAvailable(id); + } + + @Override + public Image acquireLatestImage() { + Image r; + synchronized (this) { + r = this.image; + this.image = null; + } + return r; + } + + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; + } + + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } + } + } // ------ END TextureRegistry IMPLEMENTATION ---- /** @@ -489,6 +565,11 @@ private void registerTexture(long textureId, @NonNull SurfaceTextureWrapper text flutterJNI.registerTexture(textureId, textureWrapper); } + private void registerImageTexture( + long textureId, @NonNull TextureRegistry.ImageTextureEntry textureEntry) { + flutterJNI.registerImageTexture(textureId, textureEntry); + } + // TODO(mattcarroll): describe the native behavior that this invokes private void markTextureFrameAvailable(long textureId) { flutterJNI.markTextureFrameAvailable(textureId); diff --git a/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java b/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java new file mode 100644 index 0000000000000..8a7c41b8b4c63 --- /dev/null +++ b/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java @@ -0,0 +1,140 @@ +package io.flutter.plugin.platform; + +import android.annotation.TargetApi; +import android.graphics.Canvas; +import android.graphics.ImageFormat; +import android.hardware.HardwareBuffer; +import android.media.Image; +import android.media.ImageReader; +import android.os.Build; +import android.os.Handler; +import android.view.Surface; +import io.flutter.view.TextureRegistry.ImageTextureEntry; + +@TargetApi(26) +public class ImageReaderPlatformViewRenderTarget implements PlatformViewRenderTarget { + private ImageTextureEntry textureEntry; + private boolean mustRecreateImageReader = false; + private ImageReader reader; + private int bufferWidth = 0; + private int bufferHeight = 0; + private static final String TAG = "ImageReaderPlatformViewRenderTarget"; + + private void closeReader() { + if (this.reader != null) { + this.reader.close(); + this.reader = null; + } + } + + private void recreateImageReaderIfNeeded() { + if (!mustRecreateImageReader) { + return; + } + mustRecreateImageReader = false; + closeReader(); + this.reader = createImageReader(); + } + + private final Handler onImageAvailableHandler = new Handler(); + private final ImageReader.OnImageAvailableListener onImageAvailableListener = + new ImageReader.OnImageAvailableListener() { + @Override + public void onImageAvailable(ImageReader reader) { + final Image image = reader.acquireLatestImage(); + if (image == null) { + return; + } + textureEntry.pushImage(image); + } + }; + + @TargetApi(33) + protected ImageReader createImageReader33() { + final ImageReader.Builder builder = new ImageReader.Builder(bufferWidth, bufferHeight); + // Allow for double buffering. + builder.setMaxImages(2); + // Assume that we will be producing 32-bit RGBA values. + builder.setDefaultHardwareBufferFormat(HardwareBuffer.RGBA_8888); + // Hint that consumed images will only be read by GPU. + builder.setUsage(HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + final ImageReader reader = builder.build(); + reader.setOnImageAvailableListener(this.onImageAvailableListener, onImageAvailableHandler); + return reader; + } + + @TargetApi(29) + protected ImageReader createImageReader29() { + return ImageReader.newInstance( + bufferWidth, bufferHeight, ImageFormat.UNKNOWN, 2, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + } + + @TargetApi(26) + protected ImageReader createImageReader26() { + return ImageReader.newInstance(bufferWidth, bufferHeight, ImageFormat.UNKNOWN, 2); + } + + protected ImageReader createImageReader() { + if (Build.VERSION.SDK_INT >= 33) { + return createImageReader33(); + } else if (Build.VERSION.SDK_INT >= 29) { + return createImageReader29(); + } else { + return createImageReader26(); + } + } + + public ImageReaderPlatformViewRenderTarget(ImageTextureEntry textureEntry) { + if (Build.VERSION.SDK_INT < 26) { + throw new UnsupportedOperationException( + "ImageReaderPlatformViewRenderTarget requires API version 26+"); + } + this.textureEntry = textureEntry; + } + + public void resize(int width, int height) { + if (this.reader != null && bufferWidth == width && bufferHeight == height) { + // No size change. + return; + } + closeReader(); + bufferWidth = width; + bufferHeight = height; + this.reader = createImageReader(); + } + + public int getWidth() { + return this.bufferWidth; + } + + public int getHeight() { + return this.bufferHeight; + } + + public Canvas lockHardwareCanvas() { + return getSurface().lockHardwareCanvas(); + } + + public void unlockCanvasAndPost(Canvas canvas) { + getSurface().unlockCanvasAndPost(canvas); + } + + public long getId() { + return this.textureEntry.id(); + } + + public void release() { + // textureEntry has a finalizer attached. + textureEntry = null; + closeReader(); + } + + public boolean isReleased() { + return this.textureEntry == null; + } + + public Surface getSurface() { + recreateImageReaderIfNeeded(); + return this.reader.getSurface(); + } +} diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java index c5d2282b9dbb2..aca60f4ca02e2 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java @@ -24,7 +24,6 @@ import io.flutter.Log; import io.flutter.embedding.android.AndroidTouchProcessor; import io.flutter.util.ViewUtils; -import io.flutter.view.TextureRegistry; /** * Wraps a platform view to intercept gestures and project this view onto a {@link @@ -55,12 +54,6 @@ public PlatformViewWrapper(@NonNull Context context) { setWillNotDraw(false); } - public PlatformViewWrapper( - @NonNull Context context, @NonNull TextureRegistry.SurfaceTextureEntry textureEntry) { - this(context); - this.renderTarget = new SurfaceTexturePlatformViewRenderTarget(textureEntry); - } - public PlatformViewWrapper( @NonNull Context context, @NonNull PlatformViewRenderTarget renderTarget) { this(context); diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java index b348c1d0b53c9..b0600b65bd22a 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java @@ -550,7 +550,7 @@ private long configureForVirtualDisplay( Log.i(TAG, "Hosting view in a virtual display for platform view: " + request.viewId); - final TextureRegistry.SurfaceTextureEntry textureEntry = textureRegistry.createSurfaceTexture(); + final PlatformViewRenderTarget renderTarget = makePlatformViewRenderTarget(textureRegistry); final int physicalWidth = toPhysicalPixels(request.logicalWidth); final int physicalHeight = toPhysicalPixels(request.logicalHeight); final VirtualDisplayController vdController = @@ -558,7 +558,7 @@ private long configureForVirtualDisplay( context, accessibilityEventsDelegate, platformView, - textureEntry, + renderTarget, physicalWidth, physicalHeight, request.viewId, @@ -584,7 +584,7 @@ private long configureForVirtualDisplay( final View embeddedView = platformView.getView(); contextToEmbeddedView.put(embeddedView.getContext(), embeddedView); - return textureEntry.id(); + return renderTarget.getId(); } // Configures the view for Texture Layer Hybrid Composition mode, returning the associated @@ -610,10 +610,9 @@ public long configureForTextureLayerComposition( viewWrapper = new PlatformViewWrapper(context); textureId = -1; } else { - final TextureRegistry.SurfaceTextureEntry textureEntry = - textureRegistry.createSurfaceTexture(); - viewWrapper = new PlatformViewWrapper(context, textureEntry); - textureId = textureEntry.id(); + final PlatformViewRenderTarget renderTarget = makePlatformViewRenderTarget(textureRegistry); + viewWrapper = new PlatformViewWrapper(context, renderTarget); + textureId = renderTarget.getId(); } viewWrapper.setTouchProcessor(androidTouchProcessor); viewWrapper.resizeRenderTarget(physicalWidth, physicalHeight); @@ -968,6 +967,17 @@ private void unlockInputConnection(@NonNull VirtualDisplayController controller) controller.onInputConnectionUnlocked(); } + private static PlatformViewRenderTarget makePlatformViewRenderTarget( + TextureRegistry textureRegistry) { + // TODO(johnmccutchan): Enable ImageReaderPlatformViewRenderTarget for public use. + if (false) { + final TextureRegistry.ImageTextureEntry textureEntry = textureRegistry.createImageTexture(); + return new ImageReaderPlatformViewRenderTarget(textureEntry); + } + final TextureRegistry.SurfaceTextureEntry textureEntry = textureRegistry.createSurfaceTexture(); + return new SurfaceTexturePlatformViewRenderTarget(textureEntry); + } + private static boolean validateDirection(int direction) { return direction == View.LAYOUT_DIRECTION_LTR || direction == View.LAYOUT_DIRECTION_RTL; } diff --git a/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java b/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java index 4f2fb12beac22..deff5834aadfe 100644 --- a/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java +++ b/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java @@ -164,7 +164,7 @@ public boolean isReleased() { } public void release() { - // Don't release the texture. + // Don't release the texture, let the GC finalize it. surfaceTexture = null; if (surface != null) { surface.release(); diff --git a/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java b/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java index 20886358086be..1b90ca5c3a368 100644 --- a/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java +++ b/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java @@ -17,7 +17,6 @@ import android.view.ViewTreeObserver; import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; -import io.flutter.view.TextureRegistry; @TargetApi(20) class VirtualDisplayController { @@ -27,7 +26,7 @@ public static VirtualDisplayController create( Context context, AccessibilityEventsDelegate accessibilityEventsDelegate, PlatformView view, - TextureRegistry.SurfaceTextureEntry textureEntry, + PlatformViewRenderTarget renderTarget, int width, int height, int viewId, @@ -40,9 +39,6 @@ public static VirtualDisplayController create( DisplayManager displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); final DisplayMetrics metrics = context.getResources().getDisplayMetrics(); - final PlatformViewRenderTarget renderTarget = - new SurfaceTexturePlatformViewRenderTarget(textureEntry); - // Virtual Display crashes for some PlatformViews if the width or height is bigger // than the physical screen size. We have tried to clamp or scale down the size to prevent // the crash, but both solutions lead to unwanted behavior because the @@ -73,7 +69,6 @@ public static VirtualDisplayController create( virtualDisplay, view, renderTarget, - textureEntry, focusChangeListener, viewId, createParams); @@ -86,7 +81,6 @@ public static VirtualDisplayController create( private final AccessibilityEventsDelegate accessibilityEventsDelegate; private final int densityDpi; private final int viewId; - private final TextureRegistry.SurfaceTextureEntry textureEntry; private final PlatformViewRenderTarget renderTarget; private final OnFocusChangeListener focusChangeListener; @@ -98,14 +92,12 @@ private VirtualDisplayController( VirtualDisplay virtualDisplay, PlatformView view, PlatformViewRenderTarget renderTarget, - TextureRegistry.SurfaceTextureEntry textureEntry, OnFocusChangeListener focusChangeListener, int viewId, Object createParams) { this.context = context; this.accessibilityEventsDelegate = accessibilityEventsDelegate; this.renderTarget = renderTarget; - this.textureEntry = textureEntry; this.focusChangeListener = focusChangeListener; this.viewId = viewId; this.virtualDisplay = virtualDisplay; @@ -204,7 +196,7 @@ public void dispose() { presentation.cancel(); presentation.detachState(); virtualDisplay.release(); - textureEntry.release(); + renderTarget.release(); } /** See {@link PlatformView#onFlutterViewAttached(View)} */ diff --git a/shell/platform/android/io/flutter/view/FlutterView.java b/shell/platform/android/io/flutter/view/FlutterView.java index eb498a2335e96..7d15aab7ad89d 100644 --- a/shell/platform/android/io/flutter/view/FlutterView.java +++ b/shell/platform/android/io/flutter/view/FlutterView.java @@ -898,6 +898,12 @@ public TextureRegistry.SurfaceTextureEntry createSurfaceTexture() { return registerSurfaceTexture(surfaceTexture); } + @Override + @NonNull + public ImageTextureEntry createImageTexture() { + throw new UnsupportedOperationException("Image textures are not supported in this mode."); + } + @Override @NonNull public TextureRegistry.SurfaceTextureEntry registerSurfaceTexture( diff --git a/shell/platform/android/io/flutter/view/TextureRegistry.java b/shell/platform/android/io/flutter/view/TextureRegistry.java index 18dcbe100774e..98ec56476668d 100644 --- a/shell/platform/android/io/flutter/view/TextureRegistry.java +++ b/shell/platform/android/io/flutter/view/TextureRegistry.java @@ -5,6 +5,7 @@ package io.flutter.view; import android.graphics.SurfaceTexture; +import android.media.Image; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -31,6 +32,14 @@ public interface TextureRegistry { @NonNull SurfaceTextureEntry registerSurfaceTexture(@NonNull SurfaceTexture surfaceTexture); + /** + * Creates and registers a texture managed by the Flutter engine. + * + * @return a ImageTextureEntry. + */ + @NonNull + ImageTextureEntry createImageTexture(); + /** * Callback invoked when memory is low. * @@ -38,18 +47,27 @@ public interface TextureRegistry { */ default void onTrimMemory(int level) {} - /** A registry entry for a managed SurfaceTexture. */ - interface SurfaceTextureEntry { - /** @return The managed SurfaceTexture. */ - @NonNull - SurfaceTexture surfaceTexture(); + /** An entry in the texture registry. */ + interface TextureEntry { + /** @return The identity of this texture. */ + long id(); - /** @return The identity of this SurfaceTexture. */ + /** Deregisters and releases all resources . */ + void release(); + } + + /** A registry entry for a managed SurfaceTexture. */ + interface SurfaceTextureEntry extends TextureEntry { + /** @return The identity of this texture. */ long id(); - /** Deregisters and releases this SurfaceTexture. */ + /** Deregisters and releases all resources . */ void release(); + /** @return The managed SurfaceTexture. */ + @NonNull + SurfaceTexture surfaceTexture(); + /** Set a listener that will be notified when the most recent image has been consumed. */ default void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) {} @@ -57,6 +75,33 @@ default void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listen default void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) {} } + interface ImageTextureEntry extends TextureEntry { + /** @return the identity of this ImageTextureEntry */ + long id(); + + /** Deregisters and releases all resources. */ + void release(); + + /** + * Next paint will update texture to use the contents of image. + * + *

NOTE: Caller should not call Image.close() on the pushed image. + * + *

NOTE: In the case that multiple calls to PushFrame occur before the next paint only the + * last frame pushed will be used (dropping the missed frames). + */ + void pushImage(Image image); + + /** + * Retrieve the last Image pushed. + * + *

NOTE: Caller must call Image.close() on returned image. + * + * @return Image or null. + */ + Image acquireLatestImage(); + } + /** Listener invoked when the most recent image has been consumed. */ interface OnFrameConsumedListener { /** diff --git a/shell/platform/android/jni/jni_mock.h b/shell/platform/android/jni/jni_mock.h index b2966e6e67156..8b84793844857 100644 --- a/shell/platform/android/jni/jni_mock.h +++ b/shell/platform/android/jni/jni_mock.h @@ -59,6 +59,23 @@ class JNIMock final : public PlatformViewAndroidJNI { (JavaLocalRef surface_texture, SkMatrix& transform), (override)); + MOCK_METHOD(JavaLocalRef, + ImageTextureEntryAcquireLatestImage, + (JavaLocalRef image_texture_entry), + (override)); + + MOCK_METHOD(JavaLocalRef, + ImageGetHardwareBuffer, + (JavaLocalRef image), + (override)); + + MOCK_METHOD(void, ImageClose, (JavaLocalRef image), (override)); + + MOCK_METHOD(void, + HardwareBufferClose, + (JavaLocalRef hardware_buffer), + (override)); + MOCK_METHOD(void, SurfaceTextureDetachFromGLContext, (JavaLocalRef surface_texture), diff --git a/shell/platform/android/jni/platform_view_android_jni.h b/shell/platform/android/jni/platform_view_android_jni.h index 35e949f0bc9dd..c876b093dc08e 100644 --- a/shell/platform/android/jni/platform_view_android_jni.h +++ b/shell/platform/android/jni/platform_view_android_jni.h @@ -109,6 +109,27 @@ class PlatformViewAndroidJNI { virtual void SurfaceTextureDetachFromGLContext( JavaLocalRef surface_texture) = 0; + //---------------------------------------------------------------------------- + /// @brief Acquire the latest image available. + /// + virtual JavaLocalRef ImageTextureEntryAcquireLatestImage( + JavaLocalRef image_texture_entry) = 0; + + //---------------------------------------------------------------------------- + /// @brief Grab the HardwareBuffer from image. + /// + virtual JavaLocalRef ImageGetHardwareBuffer(JavaLocalRef image) = 0; + + //---------------------------------------------------------------------------- + /// @brief Call close on image. + /// + virtual void ImageClose(JavaLocalRef image) = 0; + + //---------------------------------------------------------------------------- + /// @brief Call close on hardware_buffer. + /// + virtual void HardwareBufferClose(JavaLocalRef hardware_buffer) = 0; + //---------------------------------------------------------------------------- /// @brief Positions and sizes a platform view if using hybrid /// composition. diff --git a/shell/platform/android/library_loader.cc b/shell/platform/android/library_loader.cc index 644bd5de3b8ec..646f232e8bc4b 100644 --- a/shell/platform/android/library_loader.cc +++ b/shell/platform/android/library_loader.cc @@ -5,6 +5,7 @@ #include "flutter/fml/platform/android/jni_util.h" #include "flutter/shell/platform/android/android_image_generator.h" #include "flutter/shell/platform/android/flutter_main.h" +#include "flutter/shell/platform/android/ndk_helpers.h" #include "flutter/shell/platform/android/platform_view_android.h" #include "flutter/shell/platform/android/vsync_waiter_android.h" diff --git a/shell/platform/android/ndk_helpers.cc b/shell/platform/android/ndk_helpers.cc new file mode 100644 index 0000000000000..568be5469ab74 --- /dev/null +++ b/shell/platform/android/ndk_helpers.cc @@ -0,0 +1,101 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/android/ndk_helpers.h" + +#include "fml/native_library.h" +#include "shell/platform/android/ndk_helpers.h" + +#include "flutter/fml/logging.h" + +#include + +namespace flutter { + +namespace { + +typedef AHardwareBuffer* (*fp_AHardwareBuffer_fromHardwareBuffer)( + JNIEnv* env, + jobject hardwareBufferObj); +typedef void (*fp_AHardwareBuffer_acquire)(AHardwareBuffer* buffer); +typedef void (*fp_AHardwareBuffer_release)(AHardwareBuffer* buffer); +typedef EGLClientBuffer (*fp_eglGetNativeClientBufferANDROID)( + AHardwareBuffer* buffer); + +AHardwareBuffer* (*_AHardwareBuffer_fromHardwareBuffer)( + JNIEnv* env, + jobject hardwareBufferObj) = nullptr; +void (*_AHardwareBuffer_acquire)(AHardwareBuffer* buffer) = nullptr; +void (*_AHardwareBuffer_release)(AHardwareBuffer* buffer) = nullptr; +EGLClientBuffer (*_eglGetNativeClientBufferANDROID)(AHardwareBuffer* buffer) = + nullptr; + +std::once_flag init_once; + +void InitOnceCallback() { + static fml::RefPtr android = + fml::NativeLibrary::Create("libandroid.so"); + FML_CHECK(android.get() != nullptr); + static fml::RefPtr egl = + fml::NativeLibrary::Create("libEGL.so"); + FML_CHECK(egl.get() != nullptr); + _eglGetNativeClientBufferANDROID = + egl->ResolveFunction( + "eglGetNativeClientBufferANDROID") + .value_or(nullptr); + _AHardwareBuffer_fromHardwareBuffer = + android + ->ResolveFunction( + "AHardwareBuffer_fromHardwareBuffer") + .value_or(nullptr); + _AHardwareBuffer_acquire = android + ->ResolveFunction( + "AHardwareBuffer_acquire") + .value_or(nullptr); + _AHardwareBuffer_release = android + ->ResolveFunction( + "AHardwareBuffer_release") + .value_or(nullptr); +} + +} // namespace + +void NDKHelpers::Init() { + std::call_once(init_once, InitOnceCallback); +} + +bool NDKHelpers::HardwareBufferSupported() { + NDKHelpers::Init(); + const bool r = _AHardwareBuffer_fromHardwareBuffer != nullptr; + return r; +} + +AHardwareBuffer* NDKHelpers::AHardwareBuffer_fromHardwareBuffer( + JNIEnv* env, + jobject hardwareBufferObj) { + NDKHelpers::Init(); + FML_CHECK(_AHardwareBuffer_fromHardwareBuffer != nullptr); + return _AHardwareBuffer_fromHardwareBuffer(env, hardwareBufferObj); +} + +void NDKHelpers::AHardwareBuffer_acquire(AHardwareBuffer* buffer) { + NDKHelpers::Init(); + FML_CHECK(_AHardwareBuffer_acquire != nullptr); + _AHardwareBuffer_acquire(buffer); +} + +void NDKHelpers::AHardwareBuffer_release(AHardwareBuffer* buffer) { + NDKHelpers::Init(); + FML_CHECK(_AHardwareBuffer_release != nullptr); + _AHardwareBuffer_release(buffer); +} + +EGLClientBuffer NDKHelpers::eglGetNativeClientBufferANDROID( + AHardwareBuffer* buffer) { + NDKHelpers::Init(); + FML_CHECK(_eglGetNativeClientBufferANDROID != nullptr); + return _eglGetNativeClientBufferANDROID(buffer); +} + +} // namespace flutter diff --git a/shell/platform/android/ndk_helpers.h b/shell/platform/android/ndk_helpers.h new file mode 100644 index 0000000000000..3ec0500d99e9a --- /dev/null +++ b/shell/platform/android/ndk_helpers.h @@ -0,0 +1,37 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_ANDROID_NDK_HELPERS_H_ +#define FLUTTER_SHELL_PLATFORM_ANDROID_NDK_HELPERS_H_ + +#include "flutter/fml/native_library.h" +#include "flutter/fml/platform/android/jni_util.h" + +#include "flutter/impeller/toolkit/egl/egl.h" + +#include + +namespace flutter { + +// A collection of NDK functions that are available depending on the version of +// the Android SDK we are linked with at runtime. +class NDKHelpers { + public: + // API Version 26 + static bool HardwareBufferSupported(); + static AHardwareBuffer* AHardwareBuffer_fromHardwareBuffer( + JNIEnv* env, + jobject hardwareBufferObj); + static void AHardwareBuffer_acquire(AHardwareBuffer* buffer); + static void AHardwareBuffer_release(AHardwareBuffer* buffer); + static EGLClientBuffer eglGetNativeClientBufferANDROID( + AHardwareBuffer* buffer); + + private: + static void Init(); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_ANDROID_NDK_HELPERS_H_ diff --git a/shell/platform/android/platform_view_android.cc b/shell/platform/android/platform_view_android.cc index 2993c3d8804a3..ff941ef81dc3c 100644 --- a/shell/platform/android/platform_view_android.cc +++ b/shell/platform/android/platform_view_android.cc @@ -7,6 +7,7 @@ #include #include +#include "common/graphics/texture.h" #include "flutter/fml/synchronization/waitable_event.h" #include "flutter/shell/common/shell_io_manager.h" #include "flutter/shell/gpu/gpu_surface_gl_delegate.h" @@ -16,6 +17,7 @@ #include "flutter/shell/platform/android/android_surface_gl_impeller.h" #include "flutter/shell/platform/android/android_surface_gl_skia.h" #include "flutter/shell/platform/android/android_surface_software.h" +#include "flutter/shell/platform/android/hardware_buffer_external_texture_gl.h" #if IMPELLER_ENABLE_VULKAN // b/258506856 for why this is behind an if #include "flutter/shell/platform/android/android_surface_vulkan_impeller.h" #endif @@ -304,6 +306,18 @@ void PlatformViewAndroid::RegisterExternalTexture( } } +void PlatformViewAndroid::RegisterImageTexture( + int64_t texture_id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry) { + if (android_context_->RenderingApi() == AndroidRenderingAPI::kOpenGLES) { + RegisterTexture(std::make_shared( + std::static_pointer_cast(android_context_), + texture_id, image_texture_entry, jni_facade_)); + } else { + FML_LOG(INFO) << "Attempted to use a GL texture in a non GL context."; + } +} + // |PlatformView| std::unique_ptr PlatformViewAndroid::CreateVSyncWaiter() { return std::make_unique(task_runners_); diff --git a/shell/platform/android/platform_view_android.h b/shell/platform/android/platform_view_android.h index 069b2f920b3af..014e2c8c6fa2d 100644 --- a/shell/platform/android/platform_view_android.h +++ b/shell/platform/android/platform_view_android.h @@ -10,6 +10,7 @@ #include #include +#include #include "flutter/fml/memory/weak_ptr.h" #include "flutter/fml/platform/android/scoped_java_ref.h" #include "flutter/lib/ui/window/platform_message.h" @@ -91,6 +92,10 @@ class PlatformViewAndroid final : public PlatformView { int64_t texture_id, const fml::jni::ScopedJavaGlobalRef& surface_texture); + void RegisterImageTexture( + int64_t texture_id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry); + // |PlatformView| void LoadDartDeferredLibrary( intptr_t loading_unit_id, diff --git a/shell/platform/android/platform_view_android_jni_impl.cc b/shell/platform/android/platform_view_android_jni_impl.cc index 72abd446061b0..692326bdf6d71 100644 --- a/shell/platform/android/platform_view_android_jni_impl.cc +++ b/shell/platform/android/platform_view_android_jni_impl.cc @@ -4,6 +4,7 @@ #include "flutter/shell/platform/android/platform_view_android_jni_impl.h" +#include #include #include #include @@ -11,6 +12,8 @@ #include #include +#include "include/android/SkImageAndroid.h" +#include "shell/platform/android/ndk_helpers.h" #include "unicode/uchar.h" #include "flutter/assets/directory_asset_bundle.h" @@ -29,6 +32,7 @@ #include "flutter/shell/platform/android/android_shell_holder.h" #include "flutter/shell/platform/android/apk_asset_provider.h" #include "flutter/shell/platform/android/flutter_main.h" +#include "flutter/shell/platform/android/hardware_buffer_external_texture_gl.h" #include "flutter/shell/platform/android/jni/platform_view_android_jni.h" #include "flutter/shell/platform/android/platform_view_android.h" @@ -49,6 +53,13 @@ static fml::jni::ScopedJavaGlobalRef* g_java_weak_reference_class = static fml::jni::ScopedJavaGlobalRef* g_texture_wrapper_class = nullptr; +static fml::jni::ScopedJavaGlobalRef* g_image_texture_entry_class = + nullptr; + +static fml::jni::ScopedJavaGlobalRef* g_image_class = nullptr; + +static fml::jni::ScopedJavaGlobalRef* g_hardware_buffer_class = nullptr; + static fml::jni::ScopedJavaGlobalRef* g_java_long_class = nullptr; static fml::jni::ScopedJavaGlobalRef* g_bitmap_class = nullptr; @@ -106,6 +117,14 @@ static jmethodID g_get_transform_matrix_method = nullptr; static jmethodID g_detach_from_gl_context_method = nullptr; +static jmethodID g_acquire_latest_image_method = nullptr; + +static jmethodID g_image_get_hardware_buffer_method = nullptr; + +static jmethodID g_image_close_method = nullptr; + +static jmethodID g_hardware_buffer_close_method = nullptr; + static jmethodID g_compute_platform_resolved_locale_method = nullptr; static jmethodID g_request_dart_deferred_library_method = nullptr; @@ -475,12 +494,15 @@ static void RegisterTexture(JNIEnv* env, ); } -static void MarkTextureFrameAvailable(JNIEnv* env, - jobject jcaller, - jlong shell_holder, - jlong texture_id) { - ANDROID_SHELL_HOLDER->GetPlatformView()->MarkTextureFrameAvailable( - static_cast(texture_id)); +static void RegisterImageTexture(JNIEnv* env, + jobject jcaller, + jlong shell_holder, + jlong texture_id, + jobject image_texture_entry) { + ANDROID_SHELL_HOLDER->GetPlatformView()->RegisterImageTexture( + static_cast(texture_id), // + fml::jni::ScopedJavaGlobalRef(env, image_texture_entry) // + ); } static void UnregisterTexture(JNIEnv* env, @@ -491,6 +513,14 @@ static void UnregisterTexture(JNIEnv* env, static_cast(texture_id)); } +static void MarkTextureFrameAvailable(JNIEnv* env, + jobject jcaller, + jlong shell_holder, + jlong texture_id) { + ANDROID_SHELL_HOLDER->GetPlatformView()->MarkTextureFrameAvailable( + static_cast(texture_id)); +} + static void InvokePlatformMessageResponseCallback(JNIEnv* env, jobject jcaller, jlong shell_holder, @@ -742,6 +772,12 @@ bool RegisterApi(JNIEnv* env) { "WeakReference;)V", .fnPtr = reinterpret_cast(&RegisterTexture), }, + { + .name = "nativeRegisterImageTexture", + .signature = "(JJLjava/lang/ref/" + "WeakReference;)V", + .fnPtr = reinterpret_cast(&RegisterImageTexture), + }, { .name = "nativeMarkTextureFrameAvailable", .signature = "(JJ)V", @@ -1126,6 +1162,59 @@ bool PlatformViewAndroid::Register(JNIEnv* env) { FML_LOG(ERROR) << "Could not locate detachFromGlContext method"; return false; } + g_image_texture_entry_class = new fml::jni::ScopedJavaGlobalRef( + env, env->FindClass("io/flutter/view/TextureRegistry$ImageTextureEntry")); + if (g_image_texture_entry_class->is_null()) { + FML_LOG(ERROR) << "Could not locate ImageTextureEntry class"; + return false; + } + + g_acquire_latest_image_method = + env->GetMethodID(g_image_texture_entry_class->obj(), "acquireLatestImage", + "()Landroid/media/Image;"); + if (g_acquire_latest_image_method == nullptr) { + FML_LOG(ERROR) << "Could not locate acquireLatestImage on " + "ImageTextureEntry class"; + return false; + } + + g_image_class = new fml::jni::ScopedJavaGlobalRef( + env, env->FindClass("android/media/Image")); + if (g_image_texture_entry_class->is_null()) { + FML_LOG(ERROR) << "Could not locate Image class"; + return false; + } + + g_image_get_hardware_buffer_method = + env->GetMethodID(g_image_class->obj(), "getHardwareBuffer", + "()Landroid/hardware/HardwareBuffer;"); + + if (g_image_get_hardware_buffer_method == nullptr) { + FML_LOG(ERROR) << "Could not locate getHardwareBuffer on " + "Image class"; + return false; + } + + g_image_close_method = env->GetMethodID(g_image_class->obj(), "close", "()V"); + + if (g_image_close_method == nullptr) { + FML_LOG(ERROR) << "Could not locate close on Image class"; + return false; + } + + g_hardware_buffer_class = new fml::jni::ScopedJavaGlobalRef( + env, env->FindClass("android/hardware/HardwareBuffer")); + if (g_hardware_buffer_class->is_null()) { + FML_LOG(ERROR) << "Could not locate android.hardware.HardwareBuffer class"; + return false; + } + + g_hardware_buffer_close_method = + env->GetMethodID(g_hardware_buffer_class->obj(), "close", "()V"); + if (g_hardware_buffer_close_method == nullptr) { + FML_LOG(ERROR) << "Could not locate close on HardwareBuffer class"; + return false; + } g_compute_platform_resolved_locale_method = env->GetMethodID( g_flutter_jni_class->obj(), "computePlatformResolvedLocale", @@ -1420,6 +1509,66 @@ void PlatformViewAndroidJNIImpl::SurfaceTextureDetachFromGLContext( FML_CHECK(fml::jni::CheckException(env)); } +JavaLocalRef PlatformViewAndroidJNIImpl::ImageTextureEntryAcquireLatestImage( + JavaLocalRef image_texture_entry) { + JNIEnv* env = fml::jni::AttachCurrentThread(); + + if (image_texture_entry.is_null()) { + // Return null. + return JavaLocalRef(); + } + + // Convert the weak reference to ImageTextureEntry into a strong local + // reference. + fml::jni::ScopedJavaLocalRef image_texture_entry_local_ref( + env, env->CallObjectMethod(image_texture_entry.obj(), + g_java_weak_reference_get_method)); + + if (image_texture_entry_local_ref.is_null()) { + // Return null. + return JavaLocalRef(); + } + + JavaLocalRef r = JavaLocalRef( + env, env->CallObjectMethod(image_texture_entry_local_ref.obj(), + g_acquire_latest_image_method)); + FML_CHECK(fml::jni::CheckException(env)); + return r; +} + +JavaLocalRef PlatformViewAndroidJNIImpl::ImageGetHardwareBuffer( + JavaLocalRef image) { + JNIEnv* env = fml::jni::AttachCurrentThread(); + if (image.is_null()) { + // Return null. + return JavaLocalRef(); + } + JavaLocalRef r = JavaLocalRef( + env, + env->CallObjectMethod(image.obj(), g_image_get_hardware_buffer_method)); + FML_CHECK(fml::jni::CheckException(env)); + return r; +} + +void PlatformViewAndroidJNIImpl::ImageClose(JavaLocalRef image) { + JNIEnv* env = fml::jni::AttachCurrentThread(); + if (image.is_null()) { + return; + } + env->CallVoidMethod(image.obj(), g_image_close_method); + FML_CHECK(fml::jni::CheckException(env)); +} + +void PlatformViewAndroidJNIImpl::HardwareBufferClose( + JavaLocalRef hardware_buffer) { + JNIEnv* env = fml::jni::AttachCurrentThread(); + if (hardware_buffer.is_null()) { + return; + } + env->CallVoidMethod(hardware_buffer.obj(), g_hardware_buffer_close_method); + FML_CHECK(fml::jni::CheckException(env)); +} + void PlatformViewAndroidJNIImpl::FlutterViewOnDisplayPlatformView( int view_id, int x, diff --git a/shell/platform/android/platform_view_android_jni_impl.h b/shell/platform/android/platform_view_android_jni_impl.h index 6b1e283e812d0..33de0ae255bd0 100644 --- a/shell/platform/android/platform_view_android_jni_impl.h +++ b/shell/platform/android/platform_view_android_jni_impl.h @@ -52,6 +52,15 @@ class PlatformViewAndroidJNIImpl final : public PlatformViewAndroidJNI { void SurfaceTextureDetachFromGLContext(JavaLocalRef surface_texture) override; + JavaLocalRef ImageTextureEntryAcquireLatestImage( + JavaLocalRef image_texture_entry) override; + + JavaLocalRef ImageGetHardwareBuffer(JavaLocalRef image) override; + + void ImageClose(JavaLocalRef image) override; + + void HardwareBufferClose(JavaLocalRef hardware_buffer) override; + void FlutterViewOnDisplayPlatformView(int view_id, int x, int y, diff --git a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java index 0d2b573fe6522..9dfd58ddeaa4f 100644 --- a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java +++ b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java @@ -14,6 +14,7 @@ import android.content.MutableContextWrapper; import android.content.res.AssetManager; import android.graphics.SurfaceTexture; +import android.media.Image; import android.util.SparseArray; import android.view.MotionEvent; import android.view.Surface; @@ -1566,6 +1567,27 @@ public long id() { public void release() {} }; } + + @Override + public ImageTextureEntry createImageTexture() { + return new ImageTextureEntry() { + @Override + public long id() { + return 0; + } + + @Override + public void release() {} + + @Override + public void pushImage(Image image) {} + + @Override + public Image acquireLatestImage() { + return null; + } + }; + } }; platformViewsController.attach(context, registry, executor);