[WIP]Support inject unicoe text

.gitignore

@@ -2,3 +2,6 @@ build/
 /dist/
 .idea/
 .gradle/
+x
+*.iml
+local.properties

app/src/control_msg.c

@@ -80,6 +80,9 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
         case CONTROL_MSG_TYPE_GET_CLIPBOARD:
             // no additional data
             return 1;
+        case CONTROL_MSG_TYPE_SET_INJECT_TEXT_MODE:
+            buf[1] = msg->set_inject_text_mode.mode;
+            return 2;
         default:
             LOGW("Unknown message type: %u", (unsigned) msg->type);
             return 0;

app/src/control_msg.h

@@ -28,6 +28,7 @@ enum control_msg_type {
     CONTROL_MSG_TYPE_GET_CLIPBOARD,
     CONTROL_MSG_TYPE_SET_CLIPBOARD,
     CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE,
+    CONTROL_MSG_TYPE_SET_INJECT_TEXT_MODE,
 };
 
 enum screen_power_mode {
@@ -36,6 +37,11 @@ enum screen_power_mode {
     SCREEN_POWER_MODE_NORMAL = 2,
 };
 
+enum inject_text_mode {
+    USE_INPUT_MANAGER = 0,
+    USE_SCRCPY_IME = 1,
+};
+
 struct control_msg {
     enum control_msg_type type;
     union {
@@ -65,6 +71,9 @@ struct control_msg {
         struct {
             enum screen_power_mode mode;
         } set_screen_power_mode;
+        struct {
+            enum inject_text_mode mode;
+        } set_inject_text_mode;
     };
 };
 

app/src/input_manager.c

@@ -173,6 +173,18 @@ set_screen_power_mode(struct controller *controller,
     }
 }
 
+static void
+set_inject_text_mode(struct controller *controller,
+                      enum inject_text_mode mode) {
+    struct control_msg msg;
+    msg.type = CONTROL_MSG_TYPE_SET_INJECT_TEXT_MODE;
+    msg.set_inject_text_mode.mode = mode;
+
+    if (!controller_push_msg(controller, &msg)) {
+        LOGW("Could not request 'set screen power mode'");
+    }
+}
+
 static void
 switch_fps_counter_state(struct fps_counter *fps_counter) {
     // the started state can only be written from the current thread, so there
@@ -388,6 +400,15 @@ input_manager_process_key(struct input_manager *im,
                     }
                 }
                 return;
+            case SDLK_e:
+                if (control && cmd && !repeat && down) {
+                    if (shift) {
+                        set_inject_text_mode(controller, USE_INPUT_MANAGER);
+                    } else {
+                        set_inject_text_mode(controller, USE_SCRCPY_IME);
+                    }
+                }
+                return;
         }
 
         return;

app/src/scrcpy.c

@@ -146,12 +146,16 @@ handle_event(SDL_Event *event, bool control) {
         case SDL_WINDOWEVENT:
             screen_handle_window_event(&screen, &event->window);
             break;
+        case SDL_TEXTEDITING:
+            break;
         case SDL_TEXTINPUT:
             if (!control) {
                 break;
             }
             input_manager_process_text_input(&input_manager, &event->text);
             break;
+        case SDL_KEYMAPCHANGED:
+            break;
         case SDL_KEYDOWN:
         case SDL_KEYUP:
             // some key events do not interact with the device, so process the
@@ -195,6 +199,8 @@ handle_event(SDL_Event *event, bool control) {
             file_handler_request(&file_handler, action, event->drop.file);
             break;
         }
+        default:
+            LOGD("Unknow event type:%d", event->type);
     }
     return EVENT_RESULT_CONTINUE;
 }

ime/build.gradle

@@ -0,0 +1,28 @@
+apply plugin: 'com.android.application'
+
+android {
+    compileSdkVersion 29
+    defaultConfig {
+        applicationId "com.genymobile.scrcpy.ime"
+        minSdkVersion 21
+        targetSdkVersion 29
+        versionCode 1
+        versionName "1.0"
+
+        testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+
+}
+
+dependencies {
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    testImplementation 'junit:junit:4.12'
+}

ime/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

ime/src/main/AndroidManifest.xml

@@ -0,0 +1,10 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.genymobile.scrcpy.ime">
+    <application android:label="@string/app_name">
+        <service android:label="@string/app_name" android:name="com.genymobile.scrcpy.ime.ScrcpyIME" android:permission="android.permission.BIND_INPUT_METHOD">
+            <intent-filter>
+                <action android:name="android.view.InputMethod"/>
+            </intent-filter>
+            <meta-data android:name="android.view.im" android:resource="@xml/method" />
+        </service>
+    </application>
+</manifest>

ime/src/main/java/com/genymobile/scrcpy/ime/ScrcpyIME.java

@@ -0,0 +1,81 @@
+package com.genymobile.scrcpy.ime;
+
+import android.content.BroadcastReceiver;
+import android.content.Context;
+import android.content.Intent;
+import android.content.IntentFilter;
+import android.inputmethodservice.InputMethodService;
+import android.util.Log;
+import android.view.KeyEvent;
+import android.view.inputmethod.InputConnection;
+
+public class ScrcpyIME extends InputMethodService {
+    private static String TAG = "ScrcpyIME";
+    private BroadcastReceiver receiver;
+    private static final String COMMIT_TEXT_ACTION = "com.genymobile.scrcpy.ime.COMMIT_TEXT_ACTION";
+    private static final String STATE_CHANGE_ACTION = "com.genymobile.scrcpy.ime.STATE_CHANGE_ACTION";
+
+    @Override
+    public void onCreate() {
+        super.onCreate();
+        this.receiver = new BroadcastReceiver() {
+            @Override
+            public void onReceive(Context context, Intent intent) {
+                InputConnection inputConnection = getCurrentInputConnection();
+                if(inputConnection == null){
+                    return;
+                }
+                String text = null;
+                KeyEvent keyEvent = null;
+                if((text = intent.getStringExtra("text")) != null && text.length() > 0) {
+                   inputConnection.commitText(text, 0);
+                }else if((keyEvent = intent.getParcelableExtra("keyEvent")) != null) {
+                    inputConnection.sendKeyEvent(keyEvent);
+                }
+            }
+        };
+        IntentFilter localIntentFilter = new IntentFilter(COMMIT_TEXT_ACTION);
+        registerReceiver(this.receiver, localIntentFilter);
+    }
+
+    @Override
+    public void onDestroy() {
+        unregisterReceiver(this.receiver);
+        Log.i(TAG, "disabling self due to destroy");
+        super.onDestroy();
+    }
+
+    @Override
+    public void onBindInput() {
+        super.onBindInput();
+        Log.i(TAG, "BindInput");
+        sendStateBroadcast("BindInput");
+    }
+
+    @Override
+    public void onUnbindInput() {
+        super.onUnbindInput();
+        Log.i(TAG, "UnbindInput");
+        sendStateBroadcast("UnbindInput");
+    }
+
+    @Override
+    public void onWindowShown() {
+        super.onWindowShown();
+        Log.i(TAG, "WindowShown");
+        sendStateBroadcast("WindowShown");
+    }
+
+    @Override
+    public void onWindowHidden() {
+        super.onWindowHidden();
+        Log.i(TAG, "WindowHidden");
+        sendStateBroadcast("WindowHidden");
+    }
+
+    private void sendStateBroadcast(String state) {
+        Intent intent = new Intent(STATE_CHANGE_ACTION);
+        intent.putExtra("state", state);
+        sendBroadcast(intent);
+    }
+}

ime/src/main/res/values/strings.xml

@@ -0,0 +1,3 @@
+<resources>
+    <string name="app_name">Scrcpy</string>
+</resources>

ime/src/main/res/xml/method.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<input-method xmlns:android="http://schemas.android.com/apk/res/android">
+    <subtype android:label="@string/app_name" android:imeSubtypeLocale="en_US" android:imeSubtypeMode="keyboard" />
+</input-method>

server/meson.build

@@ -3,7 +3,7 @@
 prebuilt_server = get_option('prebuilt_server')
 if prebuilt_server == ''
     custom_target('scrcpy-server',
-                  build_always: true,  # gradle is responsible for tracking source changes
+                  build_by_default: true,  # gradle is responsible for tracking source changes
                   output: 'scrcpy-server',
                   command: [find_program('./scripts/build-wrapper.sh'), meson.current_source_dir(), '@OUTPUT@', get_option('buildtype')],
                   console: true,

server/src/main/aidl/android/content/IIntentReceiver.aidl

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2006 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.content;
+import android.content.Intent;
+import android.os.Bundle;
+/**
+ * System private API for dispatching intent broadcasts.  This is given to the
+ * activity manager as part of registering for an intent broadcasts, and is
+ * called when it receives intents.
+ *
+ * {@hide}
+ */
+oneway interface IIntentReceiver {
+    void performReceive(in Intent intent, int resultCode, String data,
+            in Bundle extras, boolean ordered, boolean sticky, int sendingUser);
+}

server/src/main/aidl/com/android/internal/view/IInputMethodClient.aidl

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.internal.view;
+
+/**
+ * Interface a client of the IInputMethodManager implements, to identify
+ * itself and receive information about changes to the global manager state.
+ */
+interface IInputMethodClient {
+}

server/src/main/java/com/genymobile/scrcpy/ControlMessage.java

@@ -15,6 +15,7 @@ public final class ControlMessage {
     public static final int TYPE_GET_CLIPBOARD = 7;
     public static final int TYPE_SET_CLIPBOARD = 8;
     public static final int TYPE_SET_SCREEN_POWER_MODE = 9;
+    public static final int TYPE_SET_INJECT_TEXT_MODE = 10;
 
     private int type;
     private String text;
@@ -85,6 +86,13 @@ public static ControlMessage createSetScreenPowerMode(int mode) {
         return msg;
     }
 
+    public static ControlMessage createSetInjectTextMode(int mode) {
+        ControlMessage msg = new ControlMessage();
+        msg.type = TYPE_SET_INJECT_TEXT_MODE;
+        msg.action = mode;
+        return msg;
+    }
+
     public static ControlMessage createEmpty(int type) {
         ControlMessage msg = new ControlMessage();
         msg.type = type;

server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java

@@ -13,6 +13,7 @@ public class ControlMessageReader {
     private static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 21;
     private static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
     private static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
+    private static final int SET_INJECT_TEXT_MODE_PAYLOAD_LENGTH = 1;
 
     public static final int TEXT_MAX_LENGTH = 300;
     public static final int CLIPBOARD_TEXT_MAX_LENGTH = 4093;
@@ -78,6 +79,9 @@ public ControlMessage next() {
             case ControlMessage.TYPE_GET_CLIPBOARD:
                 msg = ControlMessage.createEmpty(type);
                 break;
+            case ControlMessage.TYPE_SET_INJECT_TEXT_MODE:
+                msg = parseSetInjectTextMode();
+                break;
             default:
                 Ln.w("Unknown event type: " + type);
                 msg = null;
@@ -163,6 +167,14 @@ private ControlMessage parseSetScreenPowerMode() {
         return ControlMessage.createSetScreenPowerMode(mode);
     }
 
+    private ControlMessage parseSetInjectTextMode() {
+        if (buffer.remaining() < SET_INJECT_TEXT_MODE_PAYLOAD_LENGTH) {
+            return null;
+        }
+        int mode = buffer.get();
+        return ControlMessage.createSetInjectTextMode(mode);
+    }
+
     private static Position readPosition(ByteBuffer buffer) {
         int x = buffer.getInt();
         int y = buffer.getInt();