From 355aa03d3b57f60a5b1f3939a2d60335c974771c Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 15 Aug 2019 12:14:59 +1200 Subject: [PATCH 1/2] Add support for image keyboards to TextInput on Android Adds the `onImageInput` prop to TextInput --- Libraries/Components/TextInput/TextInput.js | 19 ++++++ .../DeprecatedTextInputPropTypes.js | 5 ++ .../views/textinput/ImageInputWatcher.java | 12 ++++ .../react/views/textinput/ReactEditText.java | 67 +++++++++++++++++-- .../textinput/ReactTextInputImageEvent.java | 56 ++++++++++++++++ .../textinput/ReactTextInputManager.java | 39 +++++++++++ 6 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 ReactAndroid/src/main/java/com/facebook/react/views/textinput/ImageInputWatcher.java create mode 100644 ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputImageEvent.java diff --git a/Libraries/Components/TextInput/TextInput.js b/Libraries/Components/TextInput/TextInput.js index 02e1db674aff5e..6ff9a4b4cdacc7 100644 --- a/Libraries/Components/TextInput/TextInput.js +++ b/Libraries/Components/TextInput/TextInput.js @@ -119,6 +119,14 @@ export type EditingEvent = SyntheticEvent< |}>, >; +export type ImageInputEvent = SyntheticEvent< + $Readonly<{| + uri: string, + linkUri: string, + mime: string, + |}>, +>; + type DataDetectorTypesType = | 'phoneNumber' | 'link' @@ -584,6 +592,12 @@ export type Props = $ReadOnly<{| */ onScroll?: ?(e: ScrollEvent) => mixed, + /** + * Invoked on image input from IME with `{ nativeEvent: { uri, linkUri, data, mime } }`. + * @platform android + */ + onImageInput?: ?(e: ImageInputEvent) => mixed, + /** * The string that will be rendered before text input has been entered. */ @@ -1019,6 +1033,10 @@ function InternalTextInput(props: Props): React.Node { props.onScroll && props.onScroll(event); }; + const _onImageInput = (event: ImageInputEvent) => { + this.props.onImageInput && this.props.onImageInput(event); + } + let textInput = null; let additionalTouchableProps: {| rejectResponderTermination?: $PropertyType< @@ -1084,6 +1102,7 @@ function InternalTextInput(props: Props): React.Node { onBlur={_onBlur} onChange={_onChange} onFocus={_onFocus} + onImageInput={_onImageInput} onScroll={_onScroll} onSelectionChange={_onSelectionChange} selection={selection} diff --git a/Libraries/DeprecatedPropTypes/DeprecatedTextInputPropTypes.js b/Libraries/DeprecatedPropTypes/DeprecatedTextInputPropTypes.js index 275d607b94821d..e522776880ce47 100644 --- a/Libraries/DeprecatedPropTypes/DeprecatedTextInputPropTypes.js +++ b/Libraries/DeprecatedPropTypes/DeprecatedTextInputPropTypes.js @@ -371,6 +371,11 @@ module.exports = { /** * The string that will be rendered before text input has been entered. */ + /** + * Invoked on image input from IME with `{ nativeEvent: { uri, linkUri, mime } }`. + * @platform android + */ + onImageInput: PropTypes.func, placeholder: PropTypes.string, /** * The text color of the placeholder string. diff --git a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ImageInputWatcher.java b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ImageInputWatcher.java new file mode 100644 index 00000000000000..631e3eaeb9a2a0 --- /dev/null +++ b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ImageInputWatcher.java @@ -0,0 +1,12 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.react.views.textinput; + +public interface ImageInputWatcher { + public void onImageInput(String uri, String linkUri, String mime); +} diff --git a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactEditText.java b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactEditText.java index 67b4c1aa702330..d732b9c5dc1c46 100644 --- a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactEditText.java +++ b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactEditText.java @@ -10,9 +10,11 @@ import static com.facebook.react.uimanager.UIManagerHelper.getReactContext; import android.content.Context; +import android.content.ClipDescription; import android.graphics.Rect; import android.graphics.Typeface; import android.graphics.drawable.Drawable; +import android.net.Uri; import android.os.Build; import android.os.Bundle; import android.text.Editable; @@ -36,6 +38,9 @@ import androidx.appcompat.widget.AppCompatEditText; import androidx.core.view.AccessibilityDelegateCompat; import androidx.core.view.ViewCompat; +import androidx.core.view.inputmethod.EditorInfoCompat; +import androidx.core.view.inputmethod.InputContentInfoCompat; +import androidx.core.view.inputmethod.InputConnectionCompat; import com.facebook.infer.annotation.Assertions; import com.facebook.react.bridge.JavaOnlyMap; import com.facebook.react.bridge.ReactContext; @@ -219,13 +224,67 @@ protected void onScrollChanged(int horiz, int vert, int oldHoriz, int oldVert) { } } - @Override + @Override public InputConnection onCreateInputConnection(EditorInfo outAttrs) { + ReactContext reactContext = getReactContext(this); - InputConnection inputConnection = super.onCreateInputConnection(outAttrs); + InputConnection ic = super.onCreateInputConnection(outAttrs); + + EditorInfoCompat.setContentMimeTypes(outAttrs, new String [] {"image/png", "image/gif", "image/jpg", "image/jpeg"}); + + final InputConnectionCompat.OnCommitContentListener callback = + new InputConnectionCompat.OnCommitContentListener() { + @Override + public boolean onCommitContent(InputContentInfoCompat inputContentInfo, int flags, Bundle opts) { + // read and display inputContentInfo asynchronously + if (BuildCompat.isAtLeastNMR1() && (flags & + InputConnectionCompat.INPUT_CONTENT_GRANT_READ_URI_PERMISSION) != 0) { + try { + inputContentInfo.requestPermission(); + } + catch (Exception e) { + return false; + } + } + + String uri = null; + String linkUri = null; + String mime = null; + + // Get the uri to the local content + Uri contentUri = inputContentInfo.getContentUri(); + if (contentUri != null) { + uri = contentUri.toString(); + } + + // Get the optional uri to web link + Uri link = inputContentInfo.getLinkUri(); + if (link != null) { + linkUri = link.toString(); + } + + // Get the mime type of the image + ClipDescription description = inputContentInfo.getDescription(); + if (description != null && description.getMimeTypeCount() > 0) { + mime = description.getMimeType(0); + } + + if (mImageInputWatcher != null) { + mImageInputWatcher.onImageInput(uri, linkUri, mime); + } + + // TODO find better place to call this + // inputContentInfo.releasePermission(); + + return true; + } + }; + + InputConnection inputConnection = InputConnectionCompat.createWrapper(ic, outAttrs, callback); + +>>>>>>> Add support for image keyboards to TextInput on Android if (inputConnection != null && mOnKeyPress) { - inputConnection = - new ReactEditTextInputConnectionWrapper(inputConnection, reactContext, this); + inputConnection = new ReactEditTextInputConnectionWrapper(inputConnection, reactContext, this); } if (isMultiline() && getBlurOnSubmit()) { diff --git a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputImageEvent.java b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputImageEvent.java new file mode 100644 index 00000000000000..38e3ae35d872da --- /dev/null +++ b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputImageEvent.java @@ -0,0 +1,56 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.react.views.textinput; + +import com.facebook.react.bridge.Arguments; +import com.facebook.react.bridge.WritableMap; +import com.facebook.react.uimanager.events.Event; +import com.facebook.react.uimanager.events.RCTEventEmitter; + +/** + * Event emitted by EditText native view when text changes. + * VisibleForTesting from {@link TextInputEventsTestCase}. + */ +public class ReactTextInputImageEvent extends Event { + + public static final String EVENT_NAME = "topImage"; + + private String mUri; + private String mLinkUri; + private String mMime; + + public ReactTextInputImageEvent( + int viewId, + String uri, + String linkUri, + String mime) { + super(viewId); + mUri = uri; + mLinkUri = linkUri; + mMime = mime; + } + + @Override + public String getEventName() { + return EVENT_NAME; + } + + @Override + public void dispatch(RCTEventEmitter rctEventEmitter) { + rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData()); + } + + private WritableMap serializeEventData() { + WritableMap eventData = Arguments.createMap(); + eventData.putString("uri", mUri); + eventData.putString("linkUri", mLinkUri); + eventData.putString("mime", mMime); + eventData.putInt("target", getViewTag()); + return eventData; + } +} diff --git a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputManager.java b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputManager.java index fedddd1f32500d..4ce15351414e13 100644 --- a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputManager.java +++ b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputManager.java @@ -166,6 +166,11 @@ public Map getExportedCustomBubblingEventTypeConstants() { MapBuilder.of( "phasedRegistrationNames", MapBuilder.of("bubbled", "onKeyPress", "captured", "onKeyPressCapture"))) + .put( + "topImageInput", + MapBuilder.of( + "phasedRegistrationNames", + MapBuilder.of("bubbled", "onImageInput", "captured", "onImageInputCapture"))) .build(); } @@ -380,6 +385,15 @@ public void setOnKeyPress(final ReactEditText view, boolean onKeyPress) { view.setOnKeyPress(onKeyPress); } + @ReactProp(name = "onImageInput", defaultBoolean = false) + public void setOnImageInput(final ReactEditText view, boolean onImageInput) { + if (onImageInput) { + view.setImageInputWatcher(new ReactImageInputWatcher(view)); + } else { + view.setImageInputWatcher(null); + } + } + // Sets the letter spacing as an absolute point size. // This extra handling, on top of what ReactBaseTextShadowNode already does, is required for the // correct display of spacing in placeholder (hint) text. @@ -1168,6 +1182,31 @@ public void onScrollChanged(int horiz, int vert, int oldHoriz, int oldVert) { } } + private class ReactImageInputWatcher implements ImageInputWatcher { + + private ReactEditText mReactEditText; + private EventDispatcher mEventDispatcher; + + public ReactImageInputWatcher(ReactEditText editText) { + mReactEditText = editText; + ReactContext reactContext = (ReactContext) editText.getContext(); + mEventDispatcher = reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher(); + } + + @Override + public void onImageInput(String uri, String linkUri, String mime) { + if (uri != null) { + ReactTextInputImageEvent event = new ReactTextInputImageEvent( + mReactEditText.getId(), + uri, + linkUri, + mime); + + mEventDispatcher.dispatchEvent(event); + } + } + } + @Override public @Nullable Map getExportedViewConstants() { return MapBuilder.of( From b15ad2c95895d1f9ec50106842ddf04f302ae39d Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Fri, 16 Aug 2019 17:17:23 +1200 Subject: [PATCH 2/2] Provide base64 encoded image in event --- Libraries/Components/TextInput/TextInput.js | 7 +- .../views/textinput/ImageInputWatcher.java | 2 +- .../react/views/textinput/ReactEditText.java | 64 ++++++++++++++++--- .../textinput/ReactTextInputImageEvent.java | 6 +- .../textinput/ReactTextInputManager.java | 9 +-- 5 files changed, 69 insertions(+), 19 deletions(-) diff --git a/Libraries/Components/TextInput/TextInput.js b/Libraries/Components/TextInput/TextInput.js index 6ff9a4b4cdacc7..edf458a002981d 100644 --- a/Libraries/Components/TextInput/TextInput.js +++ b/Libraries/Components/TextInput/TextInput.js @@ -120,9 +120,10 @@ export type EditingEvent = SyntheticEvent< >; export type ImageInputEvent = SyntheticEvent< - $Readonly<{| + $ReadOnly<{| uri: string, linkUri: string, + data: string, mime: string, |}>, >; @@ -1034,8 +1035,8 @@ function InternalTextInput(props: Props): React.Node { }; const _onImageInput = (event: ImageInputEvent) => { - this.props.onImageInput && this.props.onImageInput(event); - } + props.onImageInput && props.onImageInput(event); + }; let textInput = null; let additionalTouchableProps: {| diff --git a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ImageInputWatcher.java b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ImageInputWatcher.java index 631e3eaeb9a2a0..41b5ef523972e3 100644 --- a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ImageInputWatcher.java +++ b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ImageInputWatcher.java @@ -8,5 +8,5 @@ package com.facebook.react.views.textinput; public interface ImageInputWatcher { - public void onImageInput(String uri, String linkUri, String mime); + public void onImageInput(String uri, String linkUri, String data, String mime); } diff --git a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactEditText.java b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactEditText.java index d732b9c5dc1c46..74f21e0903c36f 100644 --- a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactEditText.java +++ b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactEditText.java @@ -34,8 +34,11 @@ import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputMethodManager; +import android.util.Base64; +import android.util.Base64OutputStream; import androidx.annotation.Nullable; import androidx.appcompat.widget.AppCompatEditText; +import androidx.core.os.BuildCompat; import androidx.core.view.AccessibilityDelegateCompat; import androidx.core.view.ViewCompat; import androidx.core.view.inputmethod.EditorInfoCompat; @@ -52,6 +55,11 @@ import com.facebook.react.views.text.TextAttributes; import com.facebook.react.views.text.TextInlineImageSpan; import com.facebook.react.views.view.ReactViewBackgroundManager; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.IOException; import java.util.ArrayList; /** @@ -95,6 +103,7 @@ public class ReactEditText extends AppCompatEditText { private @Nullable SelectionWatcher mSelectionWatcher; private @Nullable ContentSizeWatcher mContentSizeWatcher; private @Nullable ScrollWatcher mScrollWatcher; + private @Nullable ImageInputWatcher mImageInputWatcher; private final InternalKeyListener mKeyListener; private boolean mDetectScrollMovement = false; private boolean mOnKeyPress = false; @@ -136,6 +145,7 @@ public ReactEditText(Context context) { mStagedInputType = getInputType(); mKeyListener = new InternalKeyListener(); mScrollWatcher = null; + mImageInputWatcher = null; mTextAttributes = new TextAttributes(); applyTextAttributes(); @@ -227,10 +237,14 @@ protected void onScrollChanged(int horiz, int vert, int oldHoriz, int oldVert) { @Override public InputConnection onCreateInputConnection(EditorInfo outAttrs) { - ReactContext reactContext = getReactContext(this); + final ReactContext reactContext = getReactContext(this); InputConnection ic = super.onCreateInputConnection(outAttrs); - EditorInfoCompat.setContentMimeTypes(outAttrs, new String [] {"image/png", "image/gif", "image/jpg", "image/jpeg"}); + EditorInfoCompat.setContentMimeTypes(outAttrs, new String [] {"image/png", + "image/gif", + "image/jpg", + "image/jpeg", + "image/webp"}); final InputConnectionCompat.OnCommitContentListener callback = new InputConnectionCompat.OnCommitContentListener() { @@ -247,14 +261,28 @@ public boolean onCommitContent(InputContentInfoCompat inputContentInfo, int flag } } + // Avoid loading the image into memory if its not going to be used + if (mImageInputWatcher == null) { + return false; + } + String uri = null; String linkUri = null; String mime = null; + String data = null; - // Get the uri to the local content Uri contentUri = inputContentInfo.getContentUri(); if (contentUri != null) { uri = contentUri.toString(); + + // Load the data, we have to do this now otherwise we cannot release permissions + try { + data = loadFile(reactContext, contentUri); + } + catch(IOException e) { + inputContentInfo.releasePermission(); + return false; + } } // Get the optional uri to web link @@ -263,18 +291,17 @@ public boolean onCommitContent(InputContentInfoCompat inputContentInfo, int flag linkUri = link.toString(); } - // Get the mime type of the image ClipDescription description = inputContentInfo.getDescription(); if (description != null && description.getMimeTypeCount() > 0) { mime = description.getMimeType(0); } if (mImageInputWatcher != null) { - mImageInputWatcher.onImageInput(uri, linkUri, mime); + mImageInputWatcher.onImageInput(uri, linkUri, data, mime); } - // TODO find better place to call this - // inputContentInfo.releasePermission(); + // Releasing the permission means that the content at the uri is probably no longer accessible + inputContentInfo.releasePermission(); return true; } @@ -282,7 +309,6 @@ public boolean onCommitContent(InputContentInfoCompat inputContentInfo, int flag InputConnection inputConnection = InputConnectionCompat.createWrapper(ic, outAttrs, callback); ->>>>>>> Add support for image keyboards to TextInput on Android if (inputConnection != null && mOnKeyPress) { inputConnection = new ReactEditTextInputConnectionWrapper(inputConnection, reactContext, this); } @@ -355,6 +381,10 @@ public void setScrollWatcher(ScrollWatcher scrollWatcher) { mScrollWatcher = scrollWatcher; } + public void setImageInputWatcher(ImageInputWatcher imageInputWatcher) { + mImageInputWatcher = imageInputWatcher; + } + @Override public void setSelection(int start, int end) { // Skip setting the selection if the text wasn't set because of an out of date value. @@ -864,6 +894,24 @@ protected void applyTextAttributes() { } } + private static String loadFile(Context context, Uri contentUri) throws IOException { + InputStream inputStream = context.getContentResolver().openInputStream(contentUri); + byte[] buffer = new byte[8192]; + int bytesRead; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Base64OutputStream output64 = new Base64OutputStream(output, Base64.DEFAULT); + try { + while ((bytesRead = inputStream.read(buffer)) != -1) { + output64.write(buffer, 0, bytesRead); + } + } catch (IOException e) { + e.printStackTrace(); + } + output64.close(); + + return output.toString(); + } + /** * This class will redirect *TextChanged calls to the listeners only in the case where the text is * changed by the user, and not explicitly set by JS. diff --git a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputImageEvent.java b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputImageEvent.java index 38e3ae35d872da..afe2095ca6077d 100644 --- a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputImageEvent.java +++ b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputImageEvent.java @@ -18,20 +18,23 @@ */ public class ReactTextInputImageEvent extends Event { - public static final String EVENT_NAME = "topImage"; + public static final String EVENT_NAME = "topImageInput"; private String mUri; private String mLinkUri; + private String mData; private String mMime; public ReactTextInputImageEvent( int viewId, String uri, String linkUri, + String data, String mime) { super(viewId); mUri = uri; mLinkUri = linkUri; + mData = data; mMime = mime; } @@ -49,6 +52,7 @@ private WritableMap serializeEventData() { WritableMap eventData = Arguments.createMap(); eventData.putString("uri", mUri); eventData.putString("linkUri", mLinkUri); + eventData.putString("data", mData); eventData.putString("mime", mMime); eventData.putInt("target", getViewTag()); return eventData; diff --git a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputManager.java b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputManager.java index 4ce15351414e13..042ffd43e0d74b 100644 --- a/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputManager.java +++ b/ReactAndroid/src/main/java/com/facebook/react/views/textinput/ReactTextInputManager.java @@ -166,11 +166,6 @@ public Map getExportedCustomBubblingEventTypeConstants() { MapBuilder.of( "phasedRegistrationNames", MapBuilder.of("bubbled", "onKeyPress", "captured", "onKeyPressCapture"))) - .put( - "topImageInput", - MapBuilder.of( - "phasedRegistrationNames", - MapBuilder.of("bubbled", "onImageInput", "captured", "onImageInputCapture"))) .build(); } @@ -181,6 +176,7 @@ public Map getExportedCustomDirectEventTypeConstants() { .put( ScrollEventType.getJSEventName(ScrollEventType.SCROLL), MapBuilder.of("registrationName", "onScroll")) + .put("topImageInput", MapBuilder.of("registrationName", "onImageInput")) .build(); } @@ -1194,12 +1190,13 @@ public ReactImageInputWatcher(ReactEditText editText) { } @Override - public void onImageInput(String uri, String linkUri, String mime) { + public void onImageInput(String uri, String linkUri, String data, String mime) { if (uri != null) { ReactTextInputImageEvent event = new ReactTextInputImageEvent( mReactEditText.getId(), uri, linkUri, + data, mime); mEventDispatcher.dispatchEvent(event);