From 8251ee974dd7146e647b8a8d796e99197a79a88b Mon Sep 17 00:00:00 2001 From: Ken Soh Date: Fri, 29 Mar 2019 22:51:11 +0800 Subject: [PATCH] #370 - New keyboard step for advance keyboard entries and modifiers (#371) Building on the series of enhancements around Sikuli integration #368, #361 and #365 for greater control of mouse, creating and sending a PR that adds a new `keyboard` step. This step lets user send low-level keyboard keystrokes to the operating system user interface, including the special keys and modifier keys. Normal letter characters and numbers can also be entered. Prior to this, users can only use 1. `type page.png as text` and limited to [enter] and [clear], or 2. use `vision` step to send custom commands to perform complex keyboard actions. Below are some examples. **macOS** ``` keyboard [cmd][space] keyboard safari[enter] keyboard [cmd]c keyboard [cmd]v keyboard testing 123 ``` **Windows** ``` keyboard [ctrl][home] keyboard [printscreen] keyboard [ctrl]c keyboard v[ctrl] keyboard testing 456 ``` List of modifier keys - [shift] [ctrl] [alt] [cmd] [win] [meta] List of special keys - [clear] [space] [enter] [backspace] [tab] [esc] [up] [down] [left] [right] [pageup] [pagedown] [delete] [home] [end] [insert] [f1] .. [f15] [printscreen] [scrolllock] [pause] [capslock] [numlock] --- src/tagui.sikuli/tagui.py | 100 +++++++++++++++++++++++++++++-- src/tagui_header.js | 8 +++ src/tagui_parse.php | 8 +++ src/test/positive_test | 15 +++++ src/test/positive_test.signature | 74 +++++++++++++++++++++++ 5 files changed, 200 insertions(+), 5 deletions(-) diff --git a/src/tagui.sikuli/tagui.py b/src/tagui.sikuli/tagui.py index 415f9736..33748abf 100644 --- a/src/tagui.sikuli/tagui.py +++ b/src/tagui.sikuli/tagui.py @@ -32,6 +32,72 @@ def x_coordinate ( input_locator ): def y_coordinate ( input_locator ): return int(input_locator[input_locator.find(',')+1:-1]) +# function to map modifier keys to unicode for use in type() +def modifiers_map ( input_keys ): + modifier_keys = 0 + if '[shift]' in input_keys or '[SHIFT]' in input_keys: modifier_keys = modifier_keys + KeyModifier.SHIFT + if '[ctrl]' in input_keys or '[CTRL]' in input_keys: modifier_keys = modifier_keys + KeyModifier.CTRL + if '[alt]' in input_keys or '[ALT]' in input_keys: modifier_keys = modifier_keys + KeyModifier.ALT + if '[meta]' in input_keys or '[META]' in input_keys: modifier_keys = modifier_keys + KeyModifier.META + if '[cmd]' in input_keys or '[CMD]' in input_keys: modifier_keys = modifier_keys + KeyModifier.CMD + if '[win]' in input_keys or '[WIN]' in input_keys: modifier_keys = modifier_keys + KeyModifier.WIN + return modifier_keys + +# function to map special keys to unicode for use in type() +def keyboard_map ( input_keys ): + input_keys = input_keys.replace('[clear]','\b').replace('[CLEAR]','\b') + input_keys = input_keys.replace('[space]',' ').replace('[SPACE]',' ') + input_keys = input_keys.replace('[enter]','\n').replace('[ENTER]','\n') + input_keys = input_keys.replace('[backspace]','\b').replace('[BACKSPACE]','\b') + input_keys = input_keys.replace('[tab]','\t').replace('[TAB]','\t') + input_keys = input_keys.replace('[esc]',u'\u001b').replace('[ESC]',u'\u001b') + input_keys = input_keys.replace('[up]',u'\ue000').replace('[UP]',u'\ue000') + input_keys = input_keys.replace('[right]',u'\ue001').replace('[RIGHT]',u'\ue001') + input_keys = input_keys.replace('[down]',u'\ue002').replace('[DOWN]',u'\ue002') + input_keys = input_keys.replace('[left]',u'\ue003').replace('[LEFT]',u'\ue003') + input_keys = input_keys.replace('[pageup]',u'\ue004').replace('[PAGEUP]',u'\ue004') + input_keys = input_keys.replace('[pagedown]',u'\ue005').replace('[PAGEDOWN]',u'\ue005') + input_keys = input_keys.replace('[delete]',u'\ue006').replace('[DELETE]',u'\ue006') + input_keys = input_keys.replace('[end]',u'\ue007').replace('[END]',u'\ue007') + input_keys = input_keys.replace('[home]',u'\ue008').replace('[HOME]',u'\ue008') + input_keys = input_keys.replace('[insert]',u'\ue009').replace('[INSERT]',u'\ue009') + input_keys = input_keys.replace('[f1]',u'\ue011').replace('[F1]',u'\ue011') + input_keys = input_keys.replace('[f2]',u'\ue012').replace('[F2]',u'\ue012') + input_keys = input_keys.replace('[f3]',u'\ue013').replace('[F3]',u'\ue013') + input_keys = input_keys.replace('[f4]',u'\ue014').replace('[F4]',u'\ue014') + input_keys = input_keys.replace('[f5]',u'\ue015').replace('[F5]',u'\ue015') + input_keys = input_keys.replace('[f6]',u'\ue016').replace('[F6]',u'\ue016') + input_keys = input_keys.replace('[f7]',u'\ue017').replace('[F7]',u'\ue017') + input_keys = input_keys.replace('[f8]',u'\ue018').replace('[F8]',u'\ue018') + input_keys = input_keys.replace('[f9]',u'\ue019').replace('[F9]',u'\ue019') + input_keys = input_keys.replace('[f10]',u'\ue01A').replace('[F10]',u'\ue01A') + input_keys = input_keys.replace('[f11]',u'\ue01B').replace('[F11]',u'\ue01B') + input_keys = input_keys.replace('[f12]',u'\ue01C').replace('[F12]',u'\ue01C') + input_keys = input_keys.replace('[f13]',u'\ue01D').replace('[F13]',u'\ue01D') + input_keys = input_keys.replace('[f14]',u'\ue01E').replace('[F14]',u'\ue01E') + input_keys = input_keys.replace('[f15]',u'\ue01F').replace('[F15]',u'\ue01F') + input_keys = input_keys.replace('[printscreen]',u'\ue024').replace('[PRINTSCREEN]',u'\ue024') + input_keys = input_keys.replace('[scrolllock]',u'\ue025').replace('[SCROLLLOCK]',u'\ue025') + input_keys = input_keys.replace('[pause]',u'\ue026').replace('[PAUSE]',u'\ue026') + input_keys = input_keys.replace('[capslock]',u'\ue027').replace('[CAPSLOCK]',u'\ue027') + input_keys = input_keys.replace('[numlock]',u'\ue03B').replace('[NUMLOCK]',u'\ue03B') + + # if modifier key is the only input, treat as a keystroke instead of a modifier + if input_keys == '[shift]' or input_keys == '[SHIFT]': input_keys = u'\ue020' + elif input_keys == '[ctrl]' or input_keys == '[CTRL]': input_keys = u'\ue021' + elif input_keys == '[alt]' or input_keys == '[ALT]': input_keys = u'\ue022' + elif input_keys == '[meta]' or input_keys == '[META]': input_keys = u'\ue023' + elif input_keys == '[cmd]' or input_keys == '[CMD]': input_keys = u'\ue023' + elif input_keys == '[win]' or input_keys == '[WIN]': input_keys = u'\ue042' + + input_keys = input_keys.replace('[shift]','').replace('[SHIFT]','') + input_keys = input_keys.replace('[ctrl]','').replace('[CTRL]','') + input_keys = input_keys.replace('[alt]','').replace('[ALT]','') + input_keys = input_keys.replace('[meta]','').replace('[META]','') + input_keys = input_keys.replace('[cmd]','').replace('[CMD]','') + input_keys = input_keys.replace('[win]','').replace('[WIN]','') + return input_keys + # function to output sikuli text to tagui def output_sikuli_text ( output_text ): import codecs @@ -89,14 +155,23 @@ def type_intent ( raw_intent ): param1 = params[:params.find(' as ')].strip() param2 = params[4+params.find(' as '):].strip() print '[tagui] ACTION - type ' + param1 + ' as ' + param2 - param2 = param2.replace('[enter]','\n') - param2 = param2.replace('[clear]','\b') + modifier_keys = modifiers_map(param2) + param2 = keyboard_map(param2) if param1.endswith('page.png') or param1.endswith('page.bmp'): - return type(param2) + if modifier_keys == 0: + return type(param2) + else: + return type(param2,modifier_keys) elif is_coordinates(param1): - return type(Location(x_coordinate(param1),y_coordinate(param1)),param2) + if modifier_keys == 0: + return type(Location(x_coordinate(param1),y_coordinate(param1)),param2) + else: + return type(Location(x_coordinate(param1),y_coordinate(param1)),param2,modifier_keys) elif exists(param1): - return type(param1,param2) + if modifier_keys == 0: + return type(param1,param2) + else: + return type(param1,param2,modifier_keys) else: return 0 @@ -185,6 +260,17 @@ def snap_intent ( raw_intent ): else: return 0 +# function for low-level keyboard control +def keyboard_intent ( raw_intent ): + params = (raw_intent + ' ')[1+(raw_intent + ' ').find(' '):].strip() + print '[tagui] ACTION - keyboard ' + params + modifier_keys = modifiers_map(params) + params = keyboard_map(params) + if modifier_keys == 0: + return type(params) + else: + return type(params,modifier_keys) + # function for low-level mouse control def mouse_intent ( raw_intent ): params = (raw_intent + ' ')[1+(raw_intent + ' ').find(' '):].strip() @@ -240,6 +326,8 @@ def get_intent ( raw_intent ): return 'save' if raw_intent[:5].lower() == 'snap ': return 'snap' + if raw_intent[:9].lower() == 'keyboard ': + return 'keyboard' if raw_intent[:6].lower() == 'mouse ': return 'mouse' if raw_intent[:7].lower() == 'vision ': @@ -271,6 +359,8 @@ def parse_intent ( script_line ): return save_intent(script_line) elif intent_type == 'snap': return snap_intent(script_line) + elif intent_type == 'keyboard': + return keyboard_intent(script_line) elif intent_type == 'mouse': return mouse_intent(script_line) elif intent_type == 'vision': diff --git a/src/tagui_header.js b/src/tagui_header.js index c18f75fb..120fa67c 100644 --- a/src/tagui_header.js +++ b/src/tagui_header.js @@ -706,6 +706,7 @@ case 'table': return table_intent(live_line); break; case 'wait': return wait_intent(live_line); break; case 'live': return live_intent(live_line); break; case 'ask': return ask_intent(live_line); break; +case 'keyboard': return keyboard_intent(live_line); break; case 'mouse': return mouse_intent(live_line); break; case 'check': return check_intent(live_line); break; case 'test': return test_intent(live_line); break; @@ -759,6 +760,7 @@ if (lc_raw_intent.substr(0,6) == 'table ') return 'table'; if (lc_raw_intent.substr(0,5) == 'wait ') return 'wait'; if (lc_raw_intent.substr(0,5) == 'live ') return 'live'; if (lc_raw_intent.substr(0,4) == 'ask ') return 'ask'; +if (lc_raw_intent.substr(0,9) == 'keyboard ') return 'keyboard'; if (lc_raw_intent.substr(0,6) == 'mouse ') return 'mouse'; if (lc_raw_intent.substr(0,6) == 'check ') return 'check'; if (lc_raw_intent.substr(0,5) == 'test ') return 'test'; @@ -795,6 +797,7 @@ if (lc_raw_intent == 'table') return 'table'; if (lc_raw_intent == 'wait') return 'wait'; if (lc_raw_intent == 'live') return 'live'; if (lc_raw_intent == 'ask') return 'ask'; +if (lc_raw_intent == 'keyboard') return 'keyboard'; if (lc_raw_intent == 'mouse') return 'mouse'; if (lc_raw_intent == 'check') return 'check'; if (lc_raw_intent == 'test') return 'test'; @@ -1091,6 +1094,11 @@ return "this.echo('ERROR - you are already in live mode, type done to quit live function ask_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables return "this.echo('ERROR - step is not relevant in live mode, set ask_result directly')";} +function keyboard_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables +var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim(); +if (params == '') return "this.echo('ERROR - keys to type missing for " + raw_intent + "')"; +else return call_sikuli(raw_intent,params);} + function mouse_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim(); if (params == '') return "this.echo('ERROR - up / down missing for " + raw_intent + "')"; diff --git a/src/tagui_parse.php b/src/tagui_parse.php index 9f71056c..cab4ad8f 100755 --- a/src/tagui_parse.php +++ b/src/tagui_parse.php @@ -389,6 +389,7 @@ function process_intent($intent_type, $script_line) { case "wait": return wait_intent($script_line); break; case "live": return live_intent($script_line); break; case "ask": return ask_intent($script_line); break; +case "keyboard": return keyboard_intent($script_line); break; case "mouse": return mouse_intent($script_line); break; case "check": return check_intent($script_line); break; case "test": return test_intent($script_line); break; @@ -440,6 +441,7 @@ function get_intent($raw_intent) {$lc_raw_intent = strtolower($raw_intent); if (substr($lc_raw_intent,0,5)=="wait ") return "wait"; if (substr($lc_raw_intent,0,5)=="live ") return "live"; if (substr($lc_raw_intent,0,4)=="ask ") return "ask"; +if (substr($lc_raw_intent,0,9)=="keyboard ") return "keyboard"; if (substr($lc_raw_intent,0,6)=="mouse ") return "mouse"; if (substr($lc_raw_intent,0,6)=="check ") {$GLOBALS['test_automation']++; return "check";} if (substr($lc_raw_intent,0,5)=="test ") return "test"; @@ -476,6 +478,7 @@ function get_intent($raw_intent) {$lc_raw_intent = strtolower($raw_intent); if ($lc_raw_intent=="wait") return "wait"; if ($lc_raw_intent=="live") return "live"; if ($lc_raw_intent=="ask") return "ask"; +if ($lc_raw_intent=="keyboard") return "keyboard"; if ($lc_raw_intent=="mouse") return "mouse"; if ($lc_raw_intent=="check") {$GLOBALS['test_automation']++; return "check";} if ($lc_raw_intent=="test") return "test"; @@ -842,6 +845,11 @@ function ask_intent($raw_intent) { // ask user for input during automation and s "{ask_result = ''; var sys = require('system');\nthis.echo('".$params." '); ". "ask_result = sys.stdin.readLine();}".end_fi()."});"."\n\n";} +function keyboard_intent($raw_intent) { +$params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," "))); +if ($params == "") echo "ERROR - " . current_line() . " keys to type missing for " . $raw_intent . "\n"; +return "casper.then(function() {".call_sikuli($raw_intent,$params);} + function mouse_intent($raw_intent) { $params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," "))); if ($params == "") echo "ERROR - " . current_line() . " up / down missing for " . $raw_intent . "\n"; diff --git a/src/test/positive_test b/src/test/positive_test index bbd71772..535024fc 100644 --- a/src/test/positive_test +++ b/src/test/positive_test @@ -413,6 +413,21 @@ wait 7.5 seconds // test live live +// test keyboard +keyboard ls -lrt[enter] +keyboard ls -lrt[ENTER] +keyboard [pageup] +keyboard [PAGEDOWN] +keyboard 123[enter]456[ENTER] +keyboard [home] +keyboard [end] +keyboard [ctrl][home] +keyboard [ctrl][end] +keyboard [win] +keyboard e[win] +keyboard [win]e +keyboard [cmd][space] + // test mouse mouse down mouse up diff --git a/src/test/positive_test.signature b/src/test/positive_test.signature index 57fe487c..1bd5584a 100644 --- a/src/test/positive_test.signature +++ b/src/test/positive_test.signature @@ -733,6 +733,7 @@ case 'table': return table_intent(live_line); break; case 'wait': return wait_intent(live_line); break; case 'live': return live_intent(live_line); break; case 'ask': return ask_intent(live_line); break; +case 'keyboard': return keyboard_intent(live_line); break; case 'mouse': return mouse_intent(live_line); break; case 'check': return check_intent(live_line); break; case 'test': return test_intent(live_line); break; @@ -786,6 +787,7 @@ if (lc_raw_intent.substr(0,6) == 'table ') return 'table'; if (lc_raw_intent.substr(0,5) == 'wait ') return 'wait'; if (lc_raw_intent.substr(0,5) == 'live ') return 'live'; if (lc_raw_intent.substr(0,4) == 'ask ') return 'ask'; +if (lc_raw_intent.substr(0,9) == 'keyboard ') return 'keyboard'; if (lc_raw_intent.substr(0,6) == 'mouse ') return 'mouse'; if (lc_raw_intent.substr(0,6) == 'check ') return 'check'; if (lc_raw_intent.substr(0,5) == 'test ') return 'test'; @@ -822,6 +824,7 @@ if (lc_raw_intent == 'table') return 'table'; if (lc_raw_intent == 'wait') return 'wait'; if (lc_raw_intent == 'live') return 'live'; if (lc_raw_intent == 'ask') return 'ask'; +if (lc_raw_intent == 'keyboard') return 'keyboard'; if (lc_raw_intent == 'mouse') return 'mouse'; if (lc_raw_intent == 'check') return 'check'; if (lc_raw_intent == 'test') return 'test'; @@ -1118,6 +1121,11 @@ return "this.echo('ERROR - you are already in live mode, type done to quit live function ask_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables return "this.echo('ERROR - step is not relevant in live mode, set ask_result directly')";} +function keyboard_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables +var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim(); +if (params == '') return "this.echo('ERROR - keys to type missing for " + raw_intent + "')"; +else return call_sikuli(raw_intent,params);} + function mouse_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim(); if (params == '') return "this.echo('ERROR - up / down missing for " + raw_intent + "')"; @@ -2621,6 +2629,72 @@ while (true) {live_input = sys.stdin.readLine(); // evaluate input in casperjs c if (live_input.indexOf('done') == 0) break; try {eval(tagui_parse(live_input));} catch(e) {this.echo('ERROR - ' + e.message.charAt(0).toLowerCase() + e.message.slice(1));}}}}); +// test keyboard +casper.then(function() {{techo('keyboard ls -lrt[enter]'); var fs = require('fs'); +if (!sikuli_step('keyboard ls -lrt[enter]')) if (!fs.exists('ls -lrt[enter]')) +this.echo('ERROR - cannot find image file ls -lrt[enter]').exit(); else +this.echo('ERROR - cannot find ls -lrt[enter] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard ls -lrt[ENTER]'); var fs = require('fs'); +if (!sikuli_step('keyboard ls -lrt[ENTER]')) if (!fs.exists('ls -lrt[ENTER]')) +this.echo('ERROR - cannot find image file ls -lrt[ENTER]').exit(); else +this.echo('ERROR - cannot find ls -lrt[ENTER] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [pageup]'); var fs = require('fs'); +if (!sikuli_step('keyboard [pageup]')) if (!fs.exists('[pageup]')) +this.echo('ERROR - cannot find image file [pageup]').exit(); else +this.echo('ERROR - cannot find [pageup] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [PAGEDOWN]'); var fs = require('fs'); +if (!sikuli_step('keyboard [PAGEDOWN]')) if (!fs.exists('[PAGEDOWN]')) +this.echo('ERROR - cannot find image file [PAGEDOWN]').exit(); else +this.echo('ERROR - cannot find [PAGEDOWN] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard 123[enter]456[ENTER]'); var fs = require('fs'); +if (!sikuli_step('keyboard 123[enter]456[ENTER]')) if (!fs.exists('123[enter]456[ENTER]')) +this.echo('ERROR - cannot find image file 123[enter]456[ENTER]').exit(); else +this.echo('ERROR - cannot find 123[enter]456[ENTER] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [home]'); var fs = require('fs'); +if (!sikuli_step('keyboard [home]')) if (!fs.exists('[home]')) +this.echo('ERROR - cannot find image file [home]').exit(); else +this.echo('ERROR - cannot find [home] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [end]'); var fs = require('fs'); +if (!sikuli_step('keyboard [end]')) if (!fs.exists('[end]')) +this.echo('ERROR - cannot find image file [end]').exit(); else +this.echo('ERROR - cannot find [end] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [ctrl][home]'); var fs = require('fs'); +if (!sikuli_step('keyboard [ctrl][home]')) if (!fs.exists('[ctrl][home]')) +this.echo('ERROR - cannot find image file [ctrl][home]').exit(); else +this.echo('ERROR - cannot find [ctrl][home] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [ctrl][end]'); var fs = require('fs'); +if (!sikuli_step('keyboard [ctrl][end]')) if (!fs.exists('[ctrl][end]')) +this.echo('ERROR - cannot find image file [ctrl][end]').exit(); else +this.echo('ERROR - cannot find [ctrl][end] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [win]'); var fs = require('fs'); +if (!sikuli_step('keyboard [win]')) if (!fs.exists('[win]')) +this.echo('ERROR - cannot find image file [win]').exit(); else +this.echo('ERROR - cannot find [win] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard e[win]'); var fs = require('fs'); +if (!sikuli_step('keyboard e[win]')) if (!fs.exists('e[win]')) +this.echo('ERROR - cannot find image file e[win]').exit(); else +this.echo('ERROR - cannot find e[win] on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [win]e'); var fs = require('fs'); +if (!sikuli_step('keyboard [win]e')) if (!fs.exists('[win]e')) +this.echo('ERROR - cannot find image file [win]e').exit(); else +this.echo('ERROR - cannot find [win]e on screen').exit(); this.wait(0);}}); + +casper.then(function() {{techo('keyboard [cmd][space]'); var fs = require('fs'); +if (!sikuli_step('keyboard [cmd][space]')) if (!fs.exists('[cmd][space]')) +this.echo('ERROR - cannot find image file [cmd][space]').exit(); else +this.echo('ERROR - cannot find [cmd][space] on screen').exit(); this.wait(0);}}); + // test mouse casper.then(function() {{techo('mouse down'); var fs = require('fs'); if (!sikuli_step('mouse down')) if (!fs.exists('down'))