Skip to content

Commit

Permalink
Add a PlatformViewRenderTarget abstraction (#43813)
Browse files Browse the repository at this point in the history 
- Introduce PlatformViewRenderTarget interface.
- Refactor VirtualDisplayController and PlatformViewWrapper to extract
SurfaceTexturePlatformViewRenderTarget into a separate class.

In a future CL I will add an ImageReaderPlatformViewRenderTarget which
will enable Platform Views on Impeller/Vulkan.

Tracking issue: flutter/flutter#130892
  • Loading branch information
@johnmccutchan @harryterkelsen
johnmccutchan authored and harryterkelsen committed Jul 20, 2023
1 parent 7997970 commit af2df6f
Show file tree
Hide file tree
Showing 9 changed files with 459 additions and 337 deletions.
3 changes: 3 additions & 0 deletions ci/licenses_golden/licenses_flutter
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2435,6 +2435,7 @@ ORIGIN: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/Platf
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewFactory.java + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewRegistry.java + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewRegistryImpl.java + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewRenderTarget.java + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewsAccessibilityDelegate.java + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java + ../../../flutter/LICENSE
Expand Down Expand Up @@ -5136,10 +5137,12 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/Platfor
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewFactory.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewRegistry.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewRegistryImpl.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewRenderTarget.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewsAccessibilityDelegate.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/SingleViewPresentation.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java
FILE: ../../../flutter/shell/platform/android/io/flutter/util/HandlerCompat.java
FILE: ../../../flutter/shell/platform/android/io/flutter/util/PathUtils.java
Expand Down
2 changes: 2 additions & 0 deletions shell/platform/android/BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,10 +297,12 @@ android_java_sources = [
"io/flutter/plugin/platform/PlatformViewFactory.java",
"io/flutter/plugin/platform/PlatformViewRegistry.java",
"io/flutter/plugin/platform/PlatformViewRegistryImpl.java",
"io/flutter/plugin/platform/PlatformViewRenderTarget.java",
"io/flutter/plugin/platform/PlatformViewWrapper.java",
"io/flutter/plugin/platform/PlatformViewsAccessibilityDelegate.java",
"io/flutter/plugin/platform/PlatformViewsController.java",
"io/flutter/plugin/platform/SingleViewPresentation.java",
"io/flutter/plugin/platform/SurfaceTexturePlatformViewRenderTarget.java",
"io/flutter/plugin/platform/VirtualDisplayController.java",
"io/flutter/util/HandlerCompat.java",
"io/flutter/util/PathUtils.java",
Expand Down
45 changes: 45 additions & 0 deletions shell/platform/android/io/flutter/plugin/platform/PlatformViewRenderTarget.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package io.flutter.plugin.platform;

import android.graphics.Canvas;
import android.view.Surface;

/**
* A PlatformViewRenderTarget interface allows an Android Platform View to be rendered into an
* offscreen buffer (usually a texture is involved) that the engine can compose into the
* FlutterView.
*/
public interface PlatformViewRenderTarget {
// Called when the render target should be resized.
public void resize(int width, int height);

// Returns the currently specified width.
public int getWidth();

// Returns the currently specified height.
public int getHeight();

// Forwards call to Surface returned by getSurface.
// NOTE: If this returns null the RenderTarget is "full" and has no room for a
// new frame.
Canvas lockHardwareCanvas();

// Forwards call to Surface returned by getSurface.
// NOTE: Must be called if lockHardwareCanvas returns a non-null Canvas.
void unlockCanvasAndPost(Canvas canvas);

// The id of this render target.
public long getId();

// Releases backing resources.
public void release();

// Returns true in the case that backing resource have been released.
public boolean isReleased();

// Returns the Surface to be rendered on to.
public Surface getSurface();
}
225 changes: 48 additions & 177 deletions shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@

package io.flutter.plugin.platform;

import static android.content.ComponentCallbacks2.TRIM_MEMORY_COMPLETE;

import android.annotation.SuppressLint;
import android.annotation.TargetApi;
import android.content.Context;
Expand All @@ -14,10 +12,7 @@
import android.graphics.Matrix;
import android.graphics.PorterDuff;
import android.graphics.Rect;
import android.graphics.SurfaceTexture;
import android.os.Build;
import android.view.MotionEvent;
import android.view.Surface;
import android.view.View;
import android.view.ViewParent;
import android.view.ViewTreeObserver;
Expand All @@ -30,85 +25,30 @@
import io.flutter.embedding.android.AndroidTouchProcessor;
import io.flutter.util.ViewUtils;
import io.flutter.view.TextureRegistry;
import java.util.concurrent.atomic.AtomicLong;

/**
* Wraps a platform view to intercept gestures and project this view onto a {@link SurfaceTexture}.
* Wraps a platform view to intercept gestures and project this view onto a {@link
* PlatformViewRenderTarget}.
*
* <p>An Android platform view is composed by the engine using a {@code TextureLayer}. The view is
* embeded to the Android view hierarchy like a normal view, but it's projected onto a {@link
* SurfaceTexture}, so it can be efficiently composed by the engine.
* PlatformViewRenderTarget}, so it can be efficiently composed by the engine.
*
* <p>Since the view is in the Android view hierarchy, keyboard and accessibility interactions
* behave normally.
*/
@TargetApi(23)
class PlatformViewWrapper extends FrameLayout {
public class PlatformViewWrapper extends FrameLayout {
private static final String TAG = "PlatformViewWrapper";

private int prevLeft;
private int prevTop;
private int left;
private int top;
private int bufferWidth;
private int bufferHeight;
private SurfaceTexture tx;
private Surface surface;
private AndroidTouchProcessor touchProcessor;
private PlatformViewRenderTarget renderTarget;

@Nullable @VisibleForTesting ViewTreeObserver.OnGlobalFocusChangeListener activeFocusListener;
private final AtomicLong pendingFramesCount = new AtomicLong(0L);

private final TextureRegistry.OnFrameConsumedListener frameConsumedListener =
new TextureRegistry.OnFrameConsumedListener() {
@Override
public void onFrameConsumed() {
if (Build.VERSION.SDK_INT == 29) {
pendingFramesCount.decrementAndGet();
}
}
};

private boolean shouldRecreateSurfaceForLowMemory = false;
private final TextureRegistry.OnTrimMemoryListener trimMemoryListener =
new TextureRegistry.OnTrimMemoryListener() {
@Override
public void onTrimMemory(int level) {
// When a memory pressure warning is received and the level equal {@code
// ComponentCallbacks2.TRIM_MEMORY_COMPLETE}, the Android system releases the underlying
// surface. If we continue to use the surface (e.g., call lockHardwareCanvas), a crash
// occurs, and we found that this crash appeared on Android10 and above.
// See https://github.com/flutter/flutter/issues/103870 for more details.
//
// Here our workaround is to recreate the surface before using it.
if (level == TRIM_MEMORY_COMPLETE && Build.VERSION.SDK_INT >= 29) {
shouldRecreateSurfaceForLowMemory = true;
}
}
};

private void onFrameProduced() {
if (Build.VERSION.SDK_INT == 29) {
pendingFramesCount.incrementAndGet();
}
}

private void recreateSurfaceIfNeeded() {
if (shouldRecreateSurfaceForLowMemory) {
if (surface != null) {
surface.release();
}
surface = createSurface(tx);
shouldRecreateSurfaceForLowMemory = false;
}
}

private boolean shouldDrawToSurfaceNow() {
if (Build.VERSION.SDK_INT == 29) {
return pendingFramesCount.get() <= 0L;
}
return true;
}
private ViewTreeObserver.OnGlobalFocusChangeListener activeFocusListener;

public PlatformViewWrapper(@NonNull Context context) {
super(context);
Expand All @@ -118,9 +58,13 @@ public PlatformViewWrapper(@NonNull Context context) {
public PlatformViewWrapper(
@NonNull Context context, @NonNull TextureRegistry.SurfaceTextureEntry textureEntry) {
this(context);
textureEntry.setOnFrameConsumedListener(frameConsumedListener);
textureEntry.setOnTrimMemoryListener(trimMemoryListener);
setTexture(textureEntry.surfaceTexture());
this.renderTarget = new SurfaceTexturePlatformViewRenderTarget(textureEntry);
}

public PlatformViewWrapper(
@NonNull Context context, @NonNull PlatformViewRenderTarget renderTarget) {
this(context);
this.renderTarget = renderTarget;
}

/**
Expand All @@ -132,62 +76,6 @@ public void setTouchProcessor(@Nullable AndroidTouchProcessor newTouchProcessor)
touchProcessor = newTouchProcessor;
}

/**
* Sets the texture where the view is projected onto.
*
* <p>{@link PlatformViewWrapper} doesn't take ownership of the {@link SurfaceTexture}. As a
* result, the caller is responsible for releasing the texture.
*
* <p>{@link io.flutter.view.TextureRegistry} is responsible for creating and registering textures
* in the engine. Therefore, the engine is responsible for also releasing the texture.
*
* @param newTx The texture where the view is projected onto.
*/
@SuppressLint("NewApi")
public void setTexture(@Nullable SurfaceTexture newTx) {
if (Build.VERSION.SDK_INT < 23) {
Log.e(
TAG,
"Platform views cannot be displayed below API level 23. "
+ "You can prevent this issue by setting `minSdkVersion: 23` in build.gradle.");
return;
}

tx = newTx;

if (bufferWidth > 0 && bufferHeight > 0) {
tx.setDefaultBufferSize(bufferWidth, bufferHeight);
}

if (surface != null) {
surface.release();
}
surface = createSurface(newTx);

// Fill the entire canvas with a transparent color.
// As a result, the background color of the platform view container is displayed
// to the user until the platform view draws its first frame.
final Canvas canvas = surface.lockHardwareCanvas();
try {
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
onFrameProduced();
} finally {
surface.unlockCanvasAndPost(canvas);
}
}

@NonNull
@VisibleForTesting
protected Surface createSurface(@NonNull SurfaceTexture tx) {
return new Surface(tx);
}

/** Returns the texture where the view is projected. */
@Nullable
public SurfaceTexture getTexture() {
return tx;
}

/**
* Sets the layout parameters for this view.
*
Expand All @@ -200,37 +88,31 @@ public void setLayoutParams(@NonNull FrameLayout.LayoutParams params) {
top = params.topMargin;
}

/**
* Sets the size of the image buffer.
*
* @param width The width of the screen buffer.
* @param height The height of the screen buffer.
*/
public void setBufferSize(int width, int height) {
bufferWidth = width;
bufferHeight = height;
if (tx != null) {
tx.setDefaultBufferSize(width, height);
public void resizeRenderTarget(int width, int height) {
if (renderTarget != null) {
renderTarget.resize(width, height);
}
}

/** Returns the image buffer width. */
public int getBufferWidth() {
return bufferWidth;
public int getRenderTargetWidth() {
if (renderTarget != null) {
return renderTarget.getWidth();
}
return 0;
}

/** Returns the image buffer height. */
public int getBufferHeight() {
return bufferHeight;
public int getRenderTargetHeight() {
if (renderTarget != null) {
return renderTarget.getHeight();
}
return 0;
}

/** Releases the surface. */
/** Releases resources. */
public void release() {
// Don't release the texture.
tx = null;
if (surface != null) {
surface.release();
surface = null;
if (renderTarget != null) {
renderTarget.release();
renderTarget = null;
}
}

Expand Down Expand Up @@ -271,42 +153,26 @@ public ViewParent invalidateChildInParent(int[] location, Rect dirty) {
@Override
@SuppressLint("NewApi")
public void draw(Canvas canvas) {
if (surface == null) {
if (renderTarget == null) {
super.draw(canvas);
Log.e(TAG, "Platform view cannot be composed without a surface.");
Log.e(TAG, "Platform view cannot be composed without a RenderTarget.");
return;
}
if (!surface.isValid()) {
Log.e(TAG, "Invalid surface. The platform view cannot be displayed.");
return;
}
if (tx == null || tx.isReleased()) {
Log.e(TAG, "Invalid texture. The platform view cannot be displayed.");
final Canvas targetCanvas = renderTarget.lockHardwareCanvas();
if (targetCanvas == null) {
// Cannot render right now.
invalidate();
return;
}
// We've observed on Android Q that we have to wait for the consumer of {@link SurfaceTexture}
// to consume the last image before continuing to draw, otherwise subsequent calls to
// {@code dequeueBuffer} to request a free buffer from the {@link BufferQueue} will fail.
// See https://github.com/flutter/flutter/issues/98722
if (!shouldDrawToSurfaceNow()) {
// If there are still frames that are not consumed, we will draw them next time.
invalidate();
} else {
// We try to recreate the surface before using it to avoid the crash:
// https://github.com/flutter/flutter/issues/103870
recreateSurfaceIfNeeded();

try {
// Fill the render target with transparent pixels. This is needed for platform views that
// expect a transparent background.
targetCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
// Override the canvas that this subtree of views will use to draw.
final Canvas surfaceCanvas = surface.lockHardwareCanvas();
try {
// Clear the current pixels in the canvas.
// This helps when a WebView renders an HTML document with transparent background.
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
super.draw(surfaceCanvas);
onFrameProduced();
} finally {
surface.unlockCanvasAndPost(surfaceCanvas);
}
super.draw(targetCanvas);
} finally {
renderTarget.unlockCanvasAndPost(targetCanvas);
}
}

Expand Down Expand Up @@ -338,6 +204,11 @@ public boolean onTouchEvent(@NonNull MotionEvent event) {
return touchProcessor.onTouchEvent(event, screenMatrix);
}

@VisibleForTesting
public ViewTreeObserver.OnGlobalFocusChangeListener getActiveFocusListener() {
return this.activeFocusListener;
}

public void setOnDescendantFocusChangeListener(@NonNull OnFocusChangeListener userFocusListener) {
unsetOnDescendantFocusChangeListener();
final ViewTreeObserver observer = getViewTreeObserver();
Expand Down
Loading

0 comments on commit af2df6f

Please sign in to comment.