diff --git a/BUILD.md b/BUILD.md index 87078b715b..864de36e8c 100644 --- a/BUILD.md +++ b/BUILD.md @@ -88,11 +88,12 @@ Install the required packages from your package manager. ```bash # runtime dependencies -sudo apt install ffmpeg libsdl2-2.0-0 adb +sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0 # client build dependencies sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \ - libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev + libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev\ + libusb-dev # server build dependencies sudo apt install openjdk-11-jdk @@ -114,7 +115,7 @@ pip3 install meson sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm # client build dependencies -sudo dnf install SDL2-devel ffms2-devel meson gcc make +sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make # server build dependencies sudo dnf install java-devel @@ -159,7 +160,8 @@ install the required packages: ```bash # runtime dependencies pacman -S mingw-w64-x86_64-SDL2 \ - mingw-w64-x86_64-ffmpeg + mingw-w64-x86_64-ffmpeg \ + mingw-w64-x86_64-libusb # client build dependencies pacman -S mingw-w64-x86_64-make \ @@ -173,7 +175,8 @@ For a 32 bits version, replace `x86_64` by `i686`: ```bash # runtime dependencies pacman -S mingw-w64-i686-SDL2 \ - mingw-w64-i686-ffmpeg + mingw-w64-i686-ffmpeg \ + mingw-w64-i686-libusb # client build dependencies pacman -S mingw-w64-i686-make \ @@ -197,7 +200,7 @@ Install the packages with [Homebrew]: ```bash # runtime dependencies -brew install sdl2 ffmpeg +brew install sdl2 ffmpeg libusb # client build dependencies brew install pkg-config meson diff --git a/README.md b/README.md index 7b1d2e782a..1f1e4f4973 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,29 @@ scrcpy --crop 1224:1440:0:0 # 1224x1440 at offset (0,0) If `--max-size` is also specified, resizing is applied after cropping. +#### USB HID over AOAv2 + +Scrcpy can simulate a USB physical keyboard on Android to provide better input +experience, you need to connect your device via USB, not wireless. + +However, due to some limitation of libusb and WinUSB driver, you cannot use HID +over AOAv2 on Windows. + +Currently a USB serial number is needed to use HID over AOAv2. + +```bash +scrcpy --serial XXXXXXXXXXXXXXXX # don't use HID +scrcpy --serial XXXXXXXXXXXXXXXX --input-mode inject # don't use HID +scrcpy --serial XXXXXXXXXXXXXXXX --input-mode hid # try HID and exit if failed +``` + +Serial number can be found by `adb get-serialno`. + +If you are a non-QWERTY keyboard user and using HID mode, please remember to set +correct physical keyboard layout manually in Android settings, because scrcpy +just forwards scancodes to Android device and Android system is responsible for +converting scancodes to correct keycode on Android device (your system does this +on your PC). #### Lock video orientation diff --git a/app/meson.build b/app/meson.build index f5345803cc..9a7f3854dd 100644 --- a/app/meson.build +++ b/app/meson.build @@ -1,6 +1,7 @@ src = [ 'src/main.c', 'src/adb.c', + 'src/aoa_hid.c', 'src/cli.c', 'src/clock.c', 'src/compat.c', @@ -12,6 +13,7 @@ src = [ 'src/file_handler.c', 'src/fps_counter.c', 'src/frame_buffer.c', + 'src/hid_keyboard.c', 'src/input_manager.c', 'src/opengl.c', 'src/receiver.c', @@ -54,6 +56,7 @@ if not get_option('crossbuild_windows') dependency('libavformat'), dependency('libavcodec'), dependency('libavutil'), + dependency('libusb-1.0'), dependency('sdl2'), ] @@ -90,8 +93,20 @@ else include_directories: include_directories(ffmpeg_include_dir) ) + prebuilt_libusb = meson.get_cross_property('prebuilt_libusb') + libusb_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_libusb + '/dll' + libusb_include_dir = '../prebuilt-deps/' + prebuilt_libusb + '/include' + + libusb = declare_dependency( + dependencies: [ + cc.find_library('libusb-1.0', dirs: libusb_bin_dir), + ], + include_directories: include_directories(libusb_include_dir) + ) + dependencies = [ ffmpeg, + libusb, sdl2, cc.find_library('mingw32') ] diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 1b69a0650f..8d68cdff2c 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -82,6 +82,16 @@ Start in fullscreen. .B \-h, \-\-help Print this help. +.TP +.B \-i, \-\-input\-mode mode +Select input mode for keyboard events. + +Possible values are "hid" and "inject". + +"hid" uses Android's USB HID over AOAv2 feature to simulate physical keyboard's events, which provides better experience for IME users if supported by you device. + +"inject" is the legacy scrcpy way by injecting keycode events on Android, works on most devices and is the default. + .TP .B \-\-legacy\-paste Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v). diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.c new file mode 100644 index 0000000000..8012ecf1fc --- /dev/null +++ b/app/src/aoa_hid.c @@ -0,0 +1,321 @@ +#include "util/log.h" + +#include +#include + +#include "aoa_hid.h" + +// See . +#define ACCESSORY_REGISTER_HID 54 +#define ACCESSORY_SET_HID_REPORT_DESC 56 +#define ACCESSORY_SEND_HID_EVENT 57 +#define ACCESSORY_UNREGISTER_HID 55 + +#define DEFAULT_TIMEOUT 1000 + +// 128 seems to be enough for serial. +#define SERIAL_BUFFER_SIZE 128 + +void hid_event_log(const struct hid_event *event) { + // HID Event: [00] FF FF FF FF... + unsigned int buffer_size = event->size * 3 + 1; + char *buffer = malloc(sizeof(*buffer) * buffer_size); + if (!buffer) { + return; + } + buffer[0] = '\0'; + for (unsigned int i = 0; i < event->size; ++i) { + snprintf(buffer + i * 3, buffer_size - i * 3, " %02x", + event->buffer[i]); + } + LOGV("HID Event: [%d]%s", event->from_accessory_id, buffer); + free(buffer); + return; +} + +void hid_event_destroy(struct hid_event *event) { + free(event->buffer); +} + +inline static void log_libusb_error(enum libusb_error errcode) { + LOGW("libusb error: %s", libusb_strerror(errcode)); +} + +inline static int +get_usb_serial(libusb_device *device, char *buffer, int size) { + libusb_device_handle *handle; + int result = libusb_open(device, &handle); + if (result < 0) { + log_libusb_error((enum libusb_error)result); + return result; + } + + struct libusb_device_descriptor desc; + libusb_get_device_descriptor(device, &desc); + if (!desc.iSerialNumber) { + libusb_close(handle); + LOGW("USB device %04x:%04x has no serial number", + desc.idVendor, desc.idProduct); + return 1; + } + + result = libusb_get_string_descriptor_ascii(handle, desc.iSerialNumber, + (unsigned char *)buffer, size); + if (result < 0) { + log_libusb_error((enum libusb_error)result); + libusb_close(handle); + return result; + } + + libusb_close(handle); + buffer[SERIAL_BUFFER_SIZE - 1] = '\0'; + return 0; +} + +inline static libusb_device *aoa_find_usb_device(const char *serial) { + if (!serial) { + return NULL; + } + + libusb_device **list; + libusb_device *result = NULL; + ssize_t count = libusb_get_device_list(NULL, &list); + if (count < 0) { + log_libusb_error((enum libusb_error)count); + return NULL; + } + + char buffer[SERIAL_BUFFER_SIZE]; + for (ssize_t i = 0; i < count; ++i) { + libusb_device *device = list[i]; + int error = get_usb_serial(device, buffer, SERIAL_BUFFER_SIZE); + if (!error && !strcmp(buffer, serial)) { + result = libusb_ref_device(device); + break; + } + } + libusb_free_device_list(list, 1); + return result; +} + +inline static int +aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) { + int result = libusb_open(device, handle); + if (result < 0) { + log_libusb_error((enum libusb_error)result); + return result; + } + return 0; +} + +bool aoa_init(struct aoa *aoa, const struct scrcpy_options *options) { + aoa->usb_context = NULL; + aoa->usb_device = NULL; + aoa->usb_handle = NULL; + aoa->next_accessories_id = 0; + + cbuf_init(&aoa->queue); + + if (!sc_mutex_init(&aoa->mutex)) { + return false; + } + + if (!sc_cond_init(&aoa->event_cond)) { + sc_mutex_destroy(&aoa->mutex); + return false; + } + + libusb_init(&aoa->usb_context); + + aoa->usb_device = aoa_find_usb_device(options->serial); + if (!aoa->usb_device) { + LOGW("USB device of serial %s not found", options->serial); + sc_mutex_destroy(&aoa->mutex); + sc_cond_destroy(&aoa->event_cond); + return false; + } + + if (aoa_open_usb_handle(aoa->usb_device, &aoa->usb_handle) < 0) { + LOGW("Open USB handle failed"); + sc_cond_destroy(&aoa->event_cond); + sc_mutex_destroy(&aoa->mutex); + libusb_unref_device(aoa->usb_device); + return false; + } + + aoa->stopped = false; + + return true; +} + +uint16_t aoa_get_new_accessory_id(struct aoa *aoa) { + return aoa->next_accessories_id++; +} + +int +aoa_register_hid(struct aoa *aoa, const uint16_t accessory_id, + uint16_t report_desc_size) { + const uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; + const uint8_t request = ACCESSORY_REGISTER_HID; + // See . + // value (arg0): accessory assigned ID for the HID device + // index (arg1): total length of the HID report descriptor + const uint16_t value = accessory_id; + const uint16_t index = report_desc_size; + unsigned char *buffer = NULL; + const uint16_t length = 0; + int result = libusb_control_transfer(aoa->usb_handle, request_type, request, + value, index, buffer, length, DEFAULT_TIMEOUT); + if (result < 0) { + log_libusb_error((enum libusb_error)result); + return result; + } + return 0; +} + +int +aoa_set_hid_report_desc(struct aoa *aoa, + const struct report_desc *report_desc) { + const uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; + const uint8_t request = ACCESSORY_SET_HID_REPORT_DESC; + /** + * If the HID descriptor is longer than the endpoint zero max packet size, + * the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC + * commands. The data for the descriptor must be sent sequentially + * if multiple packets are needed. + * See . + * + * libusb handles packet abstraction internally, so we don't need to care + * about bMaxPacketSize0 here. + * See + */ + // value (arg0): accessory assigned ID for the HID device + // index (arg1): offset of data (buffer) in descriptor + const uint16_t value = report_desc->from_accessory_id; + const uint16_t index = 0; + // libusb_control_transfer expects non-const but should not modify it. + unsigned char *buffer = (unsigned char *)report_desc->buffer; + const uint16_t length = report_desc->size; + int result = libusb_control_transfer(aoa->usb_handle, request_type, request, + value, index, buffer, length, DEFAULT_TIMEOUT); + if (result < 0) { + log_libusb_error((enum libusb_error)result); + return result; + } + return 0; +} + +int +aoa_send_hid_event(struct aoa *aoa, const struct hid_event *event) { + const uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; + const uint8_t request = ACCESSORY_SEND_HID_EVENT; + // See . + // value (arg0): accessory assigned ID for the HID device + // index (arg1): 0 (unused) + const uint16_t value = event->from_accessory_id; + const uint16_t index = 0; + // libusb_control_transfer expects non-const but should not modify it. + unsigned char *buffer = (unsigned char *)event->buffer; + const uint16_t length = event->size; + int result = libusb_control_transfer(aoa->usb_handle, request_type, request, + value, index, buffer, length, DEFAULT_TIMEOUT); + if (result < 0) { + log_libusb_error((enum libusb_error)result); + return result; + } + return 0; +} + +int aoa_unregister_hid(struct aoa *aoa, const uint16_t accessory_id) { + const uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; + const uint8_t request = ACCESSORY_UNREGISTER_HID; + // See . + // value (arg0): accessory assigned ID for the HID device + // index (arg1): 0 + const uint16_t value = accessory_id; + const uint16_t index = 0; + unsigned char *buffer = NULL; + const uint16_t length = 0; + int result = libusb_control_transfer(aoa->usb_handle, request_type, request, + value, index, buffer, length, DEFAULT_TIMEOUT); + if (result < 0) { + log_libusb_error((enum libusb_error)result); + return result; + } + return 0; +} + +bool aoa_push_hid_event(struct aoa *aoa, const struct hid_event *event) { + hid_event_log(event); + sc_mutex_lock(&aoa->mutex); + bool was_empty = cbuf_is_empty(&aoa->queue); + bool res = cbuf_push(&aoa->queue, *event); + if (was_empty) { + sc_cond_signal(&aoa->event_cond); + } + sc_mutex_unlock(&aoa->mutex); + return res; +} + +inline static bool +process_hid_event(struct aoa *aoa, const struct hid_event *event) { + return aoa_send_hid_event(aoa, event) == 0; +} + +inline static int run_aoa_thread(void *data) { + struct aoa *aoa = data; + while (true) { + sc_mutex_lock(&aoa->mutex); + while (!aoa->stopped && cbuf_is_empty(&aoa->queue)) { + sc_cond_wait(&aoa->event_cond, &aoa->mutex); + } + if (aoa->stopped) { + // Stop immediately, do not process further event. + sc_mutex_unlock(&aoa->mutex); + break; + } + struct hid_event event; + bool non_empty = cbuf_take(&aoa->queue, &event); + assert(non_empty); + (void) non_empty; + sc_mutex_unlock(&aoa->mutex); + bool ok = process_hid_event(aoa, &event); + hid_event_destroy(&event); + if (!ok) { + LOGW("Could not send HID event to USB device"); + } + } + return 0; +} + +bool aoa_thread_start(struct aoa *aoa) { + LOGD("Starting aoa thread"); + + bool ok = sc_thread_create(&aoa->thread, run_aoa_thread, "aoa_thread", aoa); + + if (!ok) { + LOGC("Could not start aoa thread"); + return false; + } + + return true; +} + +void aoa_thread_stop(struct aoa *aoa) { + sc_mutex_lock(&aoa->mutex); + aoa->stopped = true; + sc_cond_signal(&aoa->event_cond); + sc_mutex_unlock(&aoa->mutex); +} + +void aoa_thread_join(struct aoa *aoa) { + sc_thread_join(&aoa->thread, NULL); +} + +void aoa_destroy(struct aoa *aoa) { + libusb_close(aoa->usb_handle); + libusb_unref_device(aoa->usb_device); + libusb_exit(aoa->usb_context); + sc_cond_destroy(&aoa->event_cond); + sc_mutex_destroy(&aoa->mutex); +} diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.h new file mode 100644 index 0000000000..226210264e --- /dev/null +++ b/app/src/aoa_hid.h @@ -0,0 +1,58 @@ +#ifndef AOA_HID_H +#define AOA_HID_H + +#include +#include + +#include + +#include "scrcpy.h" +#include "util/cbuf.h" +#include "util/thread.h" + +struct report_desc { + uint16_t from_accessory_id; + unsigned char *buffer; + uint16_t size; +}; + +struct hid_event { + uint16_t from_accessory_id; + unsigned char *buffer; + uint16_t size; +}; + +struct hid_event_queue CBUF(struct hid_event, 64); + +struct aoa { + libusb_context *usb_context; + libusb_device *usb_device; + libusb_device_handle *usb_handle; + sc_thread thread; + sc_mutex mutex; + sc_cond event_cond; + bool stopped; + uint16_t next_accessories_id; + struct hid_event_queue queue; +}; + +void hid_event_log(const struct hid_event *event); +void hid_event_destroy(struct hid_event *event); +bool aoa_init(struct aoa *aoa, const struct scrcpy_options *options); +// Generate a different accessory ID. +uint16_t aoa_get_new_accessory_id(struct aoa *aoa); +int +aoa_register_hid(struct aoa *aoa, const uint16_t accessory_id, + uint16_t report_desc_size); +int +aoa_set_hid_report_desc(struct aoa *aoa, const struct report_desc *report_desc); +int +aoa_send_hid_event(struct aoa *aoa, const struct hid_event *event); +int aoa_unregister_hid(struct aoa *aoa, const uint16_t accessory_id); +bool aoa_push_hid_event(struct aoa *aoa, const struct hid_event *event); +bool aoa_thread_start(struct aoa *aoa); +void aoa_thread_stop(struct aoa *aoa); +void aoa_thread_join(struct aoa *aoa); +void aoa_destroy(struct aoa *aoa); + +#endif diff --git a/app/src/cli.c b/app/src/cli.c index d22096cafa..ea42202be7 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -79,6 +79,15 @@ scrcpy_print_usage(const char *arg0) { " -h, --help\n" " Print this help.\n" "\n" + " -i, --input-mode mode\n" + " Select input mode for keyboard events.\n" + " Possible values are \"hid\" and \"inject\".\n" + " \"hid\" uses Android's USB HID over AOAv2 feature to\n" + " simulate physical keyboard's events, which provides better\n" + " experience for IME users if supported by you device.\n" + " \"inject\" is the legacy scrcpy way by injecting keycode\n" + " events on Android, works on most devices and is the default.\n" + "\n" " --legacy-paste\n" " Inject computer clipboard text as a sequence of key events\n" " on Ctrl+v (like MOD+Shift+v).\n" @@ -673,6 +682,20 @@ parse_record_format(const char *optarg, enum sc_record_format *format) { return false; } +static bool +parse_input_mode(const char *optarg, enum sc_input_mode *input_mode) { + if (!strcmp(optarg, "hid")) { + *input_mode = SC_INPUT_MODE_HID; + return true; + } else if (!strcmp(optarg, "inject")) { + *input_mode = SC_INPUT_MODE_INJECT; + return true; + } + LOGE("Unsupported input mode: %s (expected hid or inject)", optarg); + return false; +} + + static enum sc_record_format guess_record_format(const char *filename) { size_t len = strlen(filename); @@ -738,6 +761,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { OPT_FORWARD_ALL_CLICKS}, {"fullscreen", no_argument, NULL, 'f'}, {"help", no_argument, NULL, 'h'}, + {"input-mode", required_argument, NULL, 'i'}, {"legacy-paste", no_argument, NULL, OPT_LEGACY_PASTE}, {"lock-video-orientation", optional_argument, NULL, OPT_LOCK_VIDEO_ORIENTATION}, @@ -784,7 +808,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { optind = 0; // reset to start from the first argument in tests int c; - while ((c = getopt_long(argc, argv, "b:c:fF:hm:nNp:r:s:StTvV:w", + while ((c = getopt_long(argc, argv, "b:c:fF:hi:m:nNp:r:s:StTvV:w", long_options, NULL)) != -1) { switch (c) { case 'b': @@ -817,6 +841,11 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { case 'h': args->help = true; break; + case 'i': + if (!parse_input_mode(optarg, &opts->input_mode)) { + return false; + } + break; case OPT_MAX_FPS: if (!parse_max_fps(optarg, &opts->max_fps)) { return false; diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c new file mode 100644 index 0000000000..08d3f97e2b --- /dev/null +++ b/app/src/hid_keyboard.c @@ -0,0 +1,245 @@ +#include "util/log.h" +#include "hid_keyboard.h" + +/** + * For HID over AOAv2, we only need report descriptor. + * Normally a basic HID keyboard uses 8 bytes, + * `Modifier Reserved Key Key Key Key Key Key`. + * See Appendix B.1 Protocol 1 (Keyboard) and + * Appendix C: Keyboard Implementation in + * . + * But if we want to support media keys on keyboard, + * we need to use two reports, + * report id 1 and usage page key codes for basic keyboard, + * report id 2 and usage page consumer for media keys. + * See 8. Report Protocol in + * . + * The former byte is item type prefix, we only use short items here. + * For how to calculate an item, read 6.2.2 Report Descriptor in + * . + * For Consumer Page tags, see 15 Consumer Page (0x0C) in + * . + */ +/** + * You can dump your device's report descriptor with + * `sudo usbhid-dump -m vid:pid -e descriptor`. + * Change `vid:pid` to your device's vendor ID and product ID. + */ +unsigned char kb_report_desc_buffer[] = { + // Usage Page (Generic Desktop) + 0x05, 0x01, + // Usage (Keyboard) + 0x09, 0x06, + + // Collection (Application) + 0xA1, 0x01, + // Report ID (1) + 0x85, HID_KEYBOARD_REPORT_ID, + + // Usage Page (Key Codes) + 0x05, 0x07, + // Usage Minimum (224) + 0x19, 0xE0, + // Usage Maximum (231) + 0x29, 0xE7, + // Logical Minimum (0) + 0x15, 0x00, + // Logical Maximum (1) + 0x25, 0x01, + // Report Size (1) + 0x75, 0x01, + // Report Count (8) + 0x95, 0x08, + // Input (Data, Variable, Absolute): Modifier byte + 0x81, 0x02, + + // Report Size (8) + 0x75, 0x08, + // Report Count (1) + 0x95, 0x01, + // Input (Constant): Reserved byte + 0x81, 0x01, + + // Usage Page (LEDs) + 0x05, 0x08, + // Usage Minimum (1) + 0x19, 0x01, + // Usage Maximum (5) + 0x29, 0x05, + // Report Size (1) + 0x75, 0x01, + // Report Count (5) + 0x95, 0x05, + // Output (Data, Variable, Absolute): LED report + 0x91, 0x02, + + // Report Size (3) + 0x75, 0x03, + // Report Count (1) + 0x95, 0x01, + // Output (Constant): LED report padding + 0x91, 0x01, + + // Usage Page (Key Codes) + 0x05, 0x07, + // Usage Minimum (0) + 0x19, 0x00, + // Usage Maximum (101) + 0x29, HID_KEYBOARD_KEYS - 1, + // Logical Minimum (0) + 0x15, 0x00, + // Logical Maximum(101) + 0x25, HID_KEYBOARD_KEYS - 1, + // Report Size (8) + 0x75, 0x08, + // Report Count (6) + 0x95, HID_KEYBOARD_MAX_KEYS, + // Input (Data, Array): Keys + 0x81, 0x00, + + // End Collection + 0xC0 +}; + +static unsigned char *create_hid_keyboard_event(void) { + unsigned char *buffer = malloc(sizeof(*buffer) * HID_KEYBOARD_EVENT_SIZE); + if (!buffer) { + return NULL; + } + buffer[0] = HID_KEYBOARD_REPORT_ID; + buffer[1] = HID_KEYBOARD_MODIFIER_NONE; + buffer[2] = HID_KEYBOARD_RESERVED; + memset(buffer + HID_KEYBOARD_MODIFIER_INDEX + 2, + HID_KEYBOARD_RESERVED, HID_KEYBOARD_MAX_KEYS); + return buffer; +} + +bool +hid_keyboard_init(struct hid_keyboard *kb, struct aoa *aoa) { + kb->aoa = aoa; + kb->accessory_id = aoa_get_new_accessory_id(aoa); + + struct report_desc report_desc = { + kb->accessory_id, + kb_report_desc_buffer, + sizeof(kb_report_desc_buffer) / sizeof(kb_report_desc_buffer[0]) + }; + + if (aoa_register_hid(aoa, kb->accessory_id, report_desc.size) < 0) { + LOGW("Register HID for keyboard failed"); + return false; + } + + if (aoa_set_hid_report_desc(aoa, &report_desc) < 0) { + LOGW("Set HID report desc for keyboard failed"); + return false; + } + + // Reset all states. + memset(kb->keys, false, HID_KEYBOARD_KEYS); + return true; +} + +inline static unsigned char sdl_keymod_to_hid_modifiers(SDL_Keymod mod) { + unsigned char modifiers = HID_KEYBOARD_MODIFIER_NONE; + // Not so cool, but more readable, and does not rely on actual value. + if (mod & KMOD_LCTRL) { + modifiers |= HID_KEYBOARD_MODIFIER_LEFT_CONTROL; + } + if (mod & KMOD_LSHIFT) { + modifiers |= HID_KEYBOARD_MODIFIER_LEFT_SHIFT; + } + if (mod & KMOD_LALT) { + modifiers |= HID_KEYBOARD_MODIFIER_LEFT_ALT; + } + if (mod & KMOD_LGUI) { + modifiers |= HID_KEYBOARD_MODIFIER_LEFT_GUI; + } + if (mod & KMOD_RCTRL) { + modifiers |= HID_KEYBOARD_MODIFIER_RIGHT_CONTROL; + } + if (mod & KMOD_RSHIFT) { + modifiers |= HID_KEYBOARD_MODIFIER_RIGHT_SHIFT; + } + if (mod & KMOD_RALT) { + modifiers |= HID_KEYBOARD_MODIFIER_RIGHT_ALT; + } + if (mod & KMOD_RGUI) { + modifiers |= HID_KEYBOARD_MODIFIER_RIGHT_GUI; + } + return modifiers; +} + +inline static bool +convert_hid_keyboard_event(struct hid_keyboard *kb, struct hid_event *hid_event, + const SDL_KeyboardEvent *event) { + hid_event->buffer = create_hid_keyboard_event(); + if (!hid_event->buffer) { + return false; + } + hid_event->size = HID_KEYBOARD_EVENT_SIZE; + + unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->keysym.mod); + + SDL_Scancode scancode = event->keysym.scancode; + // SDL also generates event when only modifiers are pressed, + // we cannot ignore them totally, for example press `a` first then + // press `Control`, if we ignore `Control` event, only `a` is sent. + if (scancode >= 0 && scancode < HID_KEYBOARD_KEYS) { + // Pressed is true and released is false. + kb->keys[scancode] = (event->type == SDL_KEYDOWN); + LOGV("keys[%02x] = %s", scancode, + kb->keys[scancode] ? "true" : "false"); + } + + // Re-calculate pressed keys every time. + int keys_pressed_count = 0; + for (int i = 0; i < HID_KEYBOARD_KEYS; ++i) { + // USB HID protocol says that if keys exceeds report count, + // a phantom state should be report. + if (keys_pressed_count > HID_KEYBOARD_MAX_KEYS) { + // Pantom state is made of `ReportID, Modifiers, Reserved, ErrorRollOver, ErrorRollOver, ErrorRollOver, ErrorRollOver, ErrorRollOver, ErrorRollOver`. + memset(hid_event->buffer + HID_KEYBOARD_MODIFIER_INDEX + 2, + HID_KEYBOARD_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS); + // But the modifiers should be report normally for phantom state. + hid_event->buffer[HID_KEYBOARD_MODIFIER_INDEX] = modifiers; + return true; + } + if (kb->keys[i]) { + hid_event->buffer[ + HID_KEYBOARD_MODIFIER_INDEX + 2 + keys_pressed_count] = i; + ++keys_pressed_count; + } + } + hid_event->buffer[HID_KEYBOARD_MODIFIER_INDEX] = modifiers; + + return true; +} + +bool +hid_keyboard_convert_event(struct hid_keyboard *kb, + struct hid_event *hid_event, const SDL_KeyboardEvent *event) { + LOGV( + "Type: %s, Repeat: %s, Modifiers: %02x, Key: %02x", + event->type == SDL_KEYDOWN ? "down" : "up", + event->repeat != 0 ? "true" : "false", + sdl_keymod_to_hid_modifiers(event->keysym.mod), + event->keysym.scancode + ); + + hid_event->from_accessory_id = kb->accessory_id; + + if (event->keysym.scancode >= 0 && + event->keysym.scancode <= SDL_SCANCODE_MODE) { + // Usage Page 0x07 (Keyboard). + return convert_hid_keyboard_event(kb, hid_event, event); + } else { + // Others. + return false; + } +} + +void hid_keyboard_destroy(struct hid_keyboard *kb) { + // Unregister HID keyboard so the soft keyboard shows again on Android. + aoa_unregister_hid(kb->aoa, kb->accessory_id); +} diff --git a/app/src/hid_keyboard.h b/app/src/hid_keyboard.h new file mode 100644 index 0000000000..25dfa67385 --- /dev/null +++ b/app/src/hid_keyboard.h @@ -0,0 +1,73 @@ +#ifndef HID_KEYBOARD_H +#define HID_KEYBOARD_H + +#include + +#include + +#include "aoa_hid.h" + +/** + * Because of dual-report, when we send hid events, we need to add report id + * as prefix, so keyboard keys looks like + * `0x01 Modifier Reserved Key Key Key Key Key Key` and media keys looks like + * `0x02 MediaMask` (one key per bit for media keys). + */ +#define HID_REPORT_ID_INDEX 0 +#define HID_KEYBOARD_MODIFIER_INDEX (HID_REPORT_ID_INDEX + 1) +#define HID_KEYBOARD_MODIFIER_NONE 0x00 +#define HID_KEYBOARD_MODIFIER_LEFT_CONTROL (1 << 0) +#define HID_KEYBOARD_MODIFIER_LEFT_SHIFT (1 << 1) +#define HID_KEYBOARD_MODIFIER_LEFT_ALT (1 << 2) +#define HID_KEYBOARD_MODIFIER_LEFT_GUI (1 << 3) +#define HID_KEYBOARD_MODIFIER_RIGHT_CONTROL (1 << 4) +#define HID_KEYBOARD_MODIFIER_RIGHT_SHIFT (1 << 5) +#define HID_KEYBOARD_MODIFIER_RIGHT_ALT (1 << 6) +#define HID_KEYBOARD_MODIFIER_RIGHT_GUI (1 << 7) +// USB HID protocol says 6 keys in an event is the requirement for BIOS +// keyboard support, though OS could support more keys via modifying the report +// desc, I think 6 is enough for us. +#define HID_KEYBOARD_MAX_KEYS 6 +#define HID_KEYBOARD_EVENT_SIZE (3 + HID_KEYBOARD_MAX_KEYS) +#define HID_KEYBOARD_REPORT_ID 0x01 +#define HID_KEYBOARD_RESERVED 0x00 +#define HID_KEYBOARD_ERROR_ROLL_OVER 0x01 + +// See "SDL2/SDL_scancode.h". +// Maybe SDL_Keycode is used by most people, +// but SDL_Scancode is taken from USB HID protocol so I perfer this. +// 0x65 is Application, typically AT-101 Keyboard ends here. +#define HID_KEYBOARD_KEYS 0x66 + +/** + * HID keyboard events are sequence-based, every time keyboard state changes + * it sends an array of currently pressed keys, the host is responsible for + * compare events and determine which key becomes pressed and which key becomes + * released. In order to convert SDL_KeyboardEvent to HID events, we first use + * an array of keys to save each keys' state. And when a SDL_KeyboardEvent was + * emitted, we updated our state, this is done by hid_keyboard_update_state(), + * and then we use a loop to generate HID events, the sequence of array elements + * is unimportant and when too much keys pressed at the same time (more than + * report count), we should generate phantom state. This is done by + * hid_keyboard_get_hid_event(). Don't forget that modifiers should be updated + * too, even for phantom state. + */ +struct hid_keyboard { + struct aoa *aoa; + uint16_t accessory_id; + bool keys[HID_KEYBOARD_KEYS]; +}; + +bool +hid_keyboard_init(struct hid_keyboard *kb, struct aoa *aoa); +/** + * Return false if unsupported keys is received, + * and be safe to ignore the HID event. + * In fact we are not only convert events, we also UPDATE internal key states. + */ +bool +hid_keyboard_convert_event(struct hid_keyboard *kb, + struct hid_event *hid_event, const SDL_KeyboardEvent *event); +void hid_keyboard_destroy(struct hid_keyboard *kb); + +#endif diff --git a/app/src/input_manager.c b/app/src/input_manager.c index a5d0ad07a0..54836c4157 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -4,6 +4,8 @@ #include #include "event_converter.h" +#include "aoa_hid.h" +#include "hid_keyboard.h" #include "util/log.h" static const int ACTION_DOWN = 1; @@ -53,12 +55,16 @@ is_shortcut_mod(struct input_manager *im, uint16_t sdl_mod) { void input_manager_init(struct input_manager *im, struct controller *controller, - struct screen *screen, + struct screen *screen, struct aoa *aoa, + struct hid_keyboard *hid_keyboard, const struct scrcpy_options *options) { im->controller = controller; im->screen = screen; im->repeat = 0; + im->aoa = aoa; + im->hid_keyboard = hid_keyboard; + im->control = options->control; im->forward_key_repeat = options->forward_key_repeat; im->prefer_text = options->prefer_text; @@ -319,6 +325,11 @@ rotate_client_right(struct screen *screen) { static void input_manager_process_text_input(struct input_manager *im, const SDL_TextInputEvent *event) { + // We don't need this if HID over AOAv2 is used. + if (im->hid_keyboard) { + return; + } + if (is_shortcut_mod(im, SDL_GetModState())) { // A shortcut must never generate text events return; @@ -396,6 +407,30 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to, return true; } +static void +input_manager_process_key_inject(struct input_manager *im, + const SDL_KeyboardEvent *event) { + struct controller *controller = im->controller; + struct control_msg msg; + if (convert_input_key(event, &msg, im->prefer_text, im->repeat)) { + if (!controller_push_msg(controller, &msg)) { + LOGW("Could not request 'inject keycode'"); + } + } +} + +static void +input_manager_process_key_hid(struct input_manager *im, + const SDL_KeyboardEvent *event) { + struct hid_event hid_event; + // Not all keys are supported, just ignore unsupported keys. + if (hid_keyboard_convert_event(im->hid_keyboard, &hid_event, event)) { + if (!aoa_push_hid_event(im->aoa, &hid_event)) { + LOGW("Could request HID event"); + } + } +} + static void input_manager_process_key(struct input_manager *im, const SDL_KeyboardEvent *event) { @@ -541,7 +576,6 @@ input_manager_process_key(struct input_manager *im, } return; } - return; } @@ -550,7 +584,9 @@ input_manager_process_key(struct input_manager *im, } if (event->repeat) { - if (!im->forward_key_repeat) { + // In USB HID protocol, key repeat is handle by host + // (Android in this case), so just ignore key repeat here. + if (im->hid_keyboard || !im->forward_key_repeat) { return; } ++im->repeat; @@ -558,6 +594,7 @@ input_manager_process_key(struct input_manager *im, im->repeat = 0; } + // FIXME: Seems not work properly on Samsung Galaxy S9+? if (ctrl && !shift && keycode == SDLK_v && down && !repeat) { if (im->legacy_paste) { // inject the text as input events @@ -569,11 +606,10 @@ input_manager_process_key(struct input_manager *im, set_device_clipboard(controller, false); } - struct control_msg msg; - if (convert_input_key(event, &msg, im->prefer_text, im->repeat)) { - if (!controller_push_msg(controller, &msg)) { - LOGW("Could not request 'inject keycode'"); - } + if (im->hid_keyboard) { + input_manager_process_key_hid(im, event); + } else { + input_manager_process_key_inject(im, event); } } diff --git a/app/src/input_manager.h b/app/src/input_manager.h index 1dd7825f51..6e841846d3 100644 --- a/app/src/input_manager.h +++ b/app/src/input_manager.h @@ -11,11 +11,16 @@ #include "fps_counter.h" #include "scrcpy.h" #include "screen.h" +#include "hid_keyboard.h" struct input_manager { struct controller *controller; struct screen *screen; + struct aoa *aoa; + // If NULL, fallback to inject mode, else prefer HID mode. + struct hid_keyboard *hid_keyboard; + // SDL reports repeated events as a boolean, but Android expects the actual // number of repetitions. This variable keeps track of the count. unsigned repeat; @@ -43,7 +48,9 @@ struct input_manager { void input_manager_init(struct input_manager *im, struct controller *controller, - struct screen *screen, const struct scrcpy_options *options); + struct screen *screen, struct aoa *aoa, + struct hid_keyboard *hid_keyboard, + const struct scrcpy_options *options); bool input_manager_handle_event(struct input_manager *im, SDL_Event *event); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 6a2857884c..50d3bd9990 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -17,6 +17,7 @@ #include "decoder.h" #include "events.h" #include "file_handler.h" +#include "hid_keyboard.h" #include "input_manager.h" #include "recorder.h" #include "screen.h" @@ -40,6 +41,8 @@ struct scrcpy { #endif struct controller controller; struct file_handler file_handler; + struct hid_keyboard hid_keyboard; + struct aoa aoa; struct input_manager input_manager; }; @@ -257,8 +260,11 @@ scrcpy(const struct scrcpy_options *options) { bool v4l2_sink_initialized = false; #endif bool stream_started = false; + bool hid_keyboard_initialized = false; bool controller_initialized = false; bool controller_started = false; + bool aoa_initialized = false; + bool aoa_thread_started = false; bool screen_initialized = false; bool record = !!options->record_filename; @@ -412,7 +418,33 @@ scrcpy(const struct scrcpy_options *options) { } stream_started = true; - input_manager_init(&s->input_manager, &s->controller, &s->screen, options); + // We don't need HID over AOAv2 support if no control. + if (options->control) { + if (options->input_mode == SC_INPUT_MODE_HID) { + LOGD("Starting in HID over AOAv2 mode because of --input-mode=hid"); + aoa_initialized = aoa_init(&s->aoa, options); + if (aoa_initialized) { + hid_keyboard_initialized = hid_keyboard_init(&s->hid_keyboard, + &s->aoa); + } + // Init HID keyboard before starting thread, this is thread safe. + if (hid_keyboard_initialized) { + aoa_thread_started = aoa_thread_start(&s->aoa); + } + if (aoa_thread_started) { + LOGD("Successfully set up HID over AOAv2 mode"); + } else { + LOGW("Failed to set up HID over AOAv2 mode"); + goto end; + } + } else { + LOGD("Starting in inject mode"); + } + } + + input_manager_init(&s->input_manager, &s->controller, &s->screen, + aoa_thread_started ? &s->aoa : NULL, + hid_keyboard_initialized ? &s->hid_keyboard : NULL, options); ret = event_loop(s, options); LOGD("quit..."); @@ -422,6 +454,16 @@ scrcpy(const struct scrcpy_options *options) { screen_hide_window(&s->screen); end: + if (aoa_thread_started) { + aoa_thread_stop(&s->aoa); + } + if (hid_keyboard_initialized) { + hid_keyboard_destroy(&s->hid_keyboard); + } + if (aoa_initialized) { + aoa_destroy(&s->aoa); + } + // The stream is not stopped explicitly, because it will stop by itself on // end-of-stream if (controller_started) { diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index 8b76fb25a2..b5cf51b447 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -44,6 +44,11 @@ enum sc_shortcut_mod { SC_MOD_RSUPER = 1 << 5, }; +enum sc_input_mode { + SC_INPUT_MODE_HID, + SC_INPUT_MODE_INJECT +}; + struct sc_shortcut_mods { unsigned data[SC_MAX_SHORTCUT_MODS]; unsigned count; @@ -68,6 +73,7 @@ struct scrcpy_options { const char *v4l2_device; enum sc_log_level log_level; enum sc_record_format record_format; + enum sc_input_mode input_mode; struct sc_port_range port_range; struct sc_shortcut_mods shortcut_mods; uint16_t max_size; @@ -112,6 +118,7 @@ struct scrcpy_options { .v4l2_device = NULL, \ .log_level = SC_LOG_LEVEL_INFO, \ .record_format = SC_RECORD_FORMAT_AUTO, \ + .input_mode = SC_INPUT_MODE_INJECT, \ .port_range = { \ .first = DEFAULT_LOCAL_PORT_RANGE_FIRST, \ .last = DEFAULT_LOCAL_PORT_RANGE_LAST, \ diff --git a/cross_win32.txt b/cross_win32.txt index 4db17be71d..cb4ac03ac1 100644 --- a/cross_win32.txt +++ b/cross_win32.txt @@ -18,3 +18,4 @@ endian = 'little' prebuilt_ffmpeg_shared = 'ffmpeg-4.3.1-win32-shared' prebuilt_ffmpeg_dev = 'ffmpeg-4.3.1-win32-dev' prebuilt_sdl2 = 'SDL2-2.0.16/i686-w64-mingw32' +prebuilt_libusb = 'libusb-1.0.24/MinGW32' diff --git a/cross_win64.txt b/cross_win64.txt index d03f02722a..13cba3df4d 100644 --- a/cross_win64.txt +++ b/cross_win64.txt @@ -18,3 +18,4 @@ endian = 'little' prebuilt_ffmpeg_shared = 'ffmpeg-4.3.1-win64-shared' prebuilt_ffmpeg_dev = 'ffmpeg-4.3.1-win64-dev' prebuilt_sdl2 = 'SDL2-2.0.16/x86_64-w64-mingw32' +prebuilt_libusb = 'libusb-1.0.24/MinGW64' diff --git a/prebuilt-deps/Makefile b/prebuilt-deps/Makefile index dced047cbd..04d517c0d8 100644 --- a/prebuilt-deps/Makefile +++ b/prebuilt-deps/Makefile @@ -3,11 +3,14 @@ prepare-ffmpeg-dev-win32 \ prepare-ffmpeg-shared-win64 \ prepare-ffmpeg-dev-win64 \ + prepare-libusb \ prepare-sdl2 \ prepare-adb -prepare-win32: prepare-sdl2 prepare-ffmpeg-shared-win32 prepare-ffmpeg-dev-win32 prepare-adb -prepare-win64: prepare-sdl2 prepare-ffmpeg-shared-win64 prepare-ffmpeg-dev-win64 prepare-adb +LIBUSB_DIR := libusb-1.0.24 + +prepare-win32: prepare-sdl2 prepare-ffmpeg-shared-win32 prepare-ffmpeg-dev-win32 prepare-libusb prepare-adb +prepare-win64: prepare-sdl2 prepare-ffmpeg-shared-win64 prepare-ffmpeg-dev-win64 prepare-libusb prepare-adb prepare-ffmpeg-shared-win32: @./prepare-dep https://github.com/Genymobile/scrcpy/releases/download/v1.16/ffmpeg-4.3.1-win32-shared.zip \ @@ -29,6 +32,17 @@ prepare-ffmpeg-dev-win64: 2e8038242cf8e1bd095c2978f196ff0462b122cc6ef7e74626a6af15459d8b81 \ ffmpeg-4.3.1-win64-dev +prepare-libusb: + # libusb put all things under the root of 7z file, so we pass the last + # argument which means creating extract dir manually. + @./prepare-dep https://github.com/libusb/libusb/releases/download/v1.0.24/libusb-1.0.24.7z \ + 620cec4dbe4868202949294157da5adb75c9fbb4f04266146fc833eef85f90fb \ + "$(LIBUSB_DIR)" \ + 1 + # Our code expects include dir under architechture dir. + cp -a "$(LIBUSB_DIR)"/include "$(LIBUSB_DIR)"/MinGW32 + cp -a "$(LIBUSB_DIR)"/include "$(LIBUSB_DIR)"/MinGW64 + prepare-sdl2: @./prepare-dep https://libsdl.org/release/SDL2-devel-2.0.16-mingw.tar.gz \ 2bfe48628aa9635c12eac7d421907e291525de1d0b04b3bca4a5bd6e6c881a6f \ diff --git a/prebuilt-deps/prepare-dep b/prebuilt-deps/prepare-dep index f152e6cf65..179d52ee70 100755 --- a/prebuilt-deps/prepare-dep +++ b/prebuilt-deps/prepare-dep @@ -3,6 +3,7 @@ set -e url="$1" sum="$2" dir="$3" +create_extract_dir="$4" checksum() { local file="$1" @@ -27,13 +28,32 @@ get_file() { extract() { local file="$1" + local create_extract_dir="$2" echo "Extracting $file..." if [[ "$file" == *.zip ]] then - unzip -q "$file" + if [[ -n "$create_extract_dir" ]] + then + unzip -q "$file" -d "$dir" + else + unzip -q "$file" + fi elif [[ "$file" == *.tar.gz ]] then - tar xf "$file" + if [[ -n "$create_extract_dir" ]] + then + tar xf "$file" --one-top-level="$dir" + else + tar xf "$file" + fi + elif [[ "$file" == *.7z ]] + then + if [[ -n "$create_extract_dir" ]] + then + 7z x "$file" -o"$dir" + else + 7z x "$file" + fi else echo "Unsupported file: $file" return 1 @@ -44,6 +64,7 @@ get_dep() { local url="$1" local sum="$2" local dir="$3" + local create_extract_dir="$4" local file="${url##*/}" if [[ -d "$dir" ]] then @@ -51,8 +72,8 @@ get_dep() { else echo "$dir: not found" get_file "$url" "$file" "$sum" - extract "$file" + extract "$file" "$create_extract_dir" fi } -get_dep "$url" "$sum" "$dir" +get_dep "$url" "$sum" "$dir" "$create_extract_dir" diff --git a/release.mk b/release.mk index e327654cfd..5359a5eebc 100644 --- a/release.mk +++ b/release.mk @@ -99,6 +99,7 @@ dist-win32: build-server build-win32 cp prebuilt-deps/ffmpeg-4.3.1-win32-shared/bin/avformat-58.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp prebuilt-deps/ffmpeg-4.3.1-win32-shared/bin/swresample-3.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp prebuilt-deps/ffmpeg-4.3.1-win32-shared/bin/swscale-5.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp prebuilt-deps/libusb-1.0.24/MinGW32/dll/libusb-1.0.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp prebuilt-deps/platform-tools/adb.exe "$(DIST)/$(WIN32_TARGET_DIR)/" cp prebuilt-deps/platform-tools/AdbWinApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp prebuilt-deps/platform-tools/AdbWinUsbApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" @@ -115,6 +116,7 @@ dist-win64: build-server build-win64 cp prebuilt-deps/ffmpeg-4.3.1-win64-shared/bin/avformat-58.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp prebuilt-deps/ffmpeg-4.3.1-win64-shared/bin/swresample-3.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp prebuilt-deps/ffmpeg-4.3.1-win64-shared/bin/swscale-5.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp prebuilt-deps/libusb-1.0.24/MinGW64/dll/libusb-1.0.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp prebuilt-deps/platform-tools/adb.exe "$(DIST)/$(WIN64_TARGET_DIR)/" cp prebuilt-deps/platform-tools/AdbWinApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp prebuilt-deps/platform-tools/AdbWinUsbApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/"