diff --git a/certsuite/cert.py b/certsuite/cert.py index 2ee7aff0..69171ef7 100755 --- a/certsuite/cert.py +++ b/certsuite/cert.py @@ -255,9 +255,9 @@ def parse_permissions_results(expected_results_path, results, prefix, logger, re log_results(unexpected_results, logger, report, 'permissions', prefix + 'unexpected-permissions-results') return not unexpected_results -def run_marionette_script(script, chrome=False, async=False): +def run_marionette_script(script, chrome=False, async=False, host='localhost', port=2828): """Create a Marionette instance and run the provided script""" - m = marionette.Marionette() + m = marionette.Marionette(host, port) m.start_session() if chrome: m.set_context(marionette.Marionette.CONTEXT_CHROME) diff --git a/certsuite/harness.py b/certsuite/harness.py index d1dc7684..549a9755 100644 --- a/certsuite/harness.py +++ b/certsuite/harness.py @@ -40,6 +40,10 @@ import gaiautils import report +DeviceBackup = adb_b2g.DeviceBackup +_hasadb = True +_host = 'localhost' +_port = 2828 logger = None stdio_handler = handlers.StreamHandler(sys.stderr, formatters.MachFormatter()) @@ -64,7 +68,7 @@ def load_config(path): return config -def iter_test_lists(suites_config): +def iter_test_lists(suites_config, mode='phone'): ''' Query each subharness for the list of test groups it can run and yield a tuple of (subharness, test group) for each one. @@ -72,6 +76,11 @@ def iter_test_lists(suites_config): for name, opts in suites_config.iteritems(): try: cmd = [opts["cmd"], '--list-test-groups'] + opts.get("common_args", []) + + if mode == 'stingray': + cmd.append('--mode') + cmd.append('stingray') + for group in subprocess.check_output(cmd).splitlines(): yield name, group except (subprocess.CalledProcessError, OSError) as e: @@ -140,9 +149,9 @@ def __exit__(self, ex_type, ex_value, tb): # Consider upstreaming this to marionette-client: class MarionetteSession(object): - def __init__(self, device): + def __init__(self, device, host='localhost', port=2828): self.device = device - self.marionette = marionette.Marionette() + self.marionette = marionette.Marionette(host=host, port=port) def __enter__(self): self.device.forward("tcp:2828", "tcp:2828") @@ -168,7 +177,10 @@ def iter_suites(self): or the empty list to indicate all tests. ''' if not self.args.tests: - tests = self.config["suites"].keys() + default_tests = self.config["suites"].keys() + if self.args.mode == 'stingray': + default_tests = ['webapi', 'security'] + tests = default_tests else: tests = self.args.tests @@ -266,6 +278,9 @@ def build_command(self, suite, groups, temp_dir): output_files = [log_name] output_files += [item % subn for item in suite_opts.get("extra_files", [])] + if self.args.mode == 'stingray' and (suite == 'webapi' or suite == 'security'): + cmd.extend([u'--host=%s' % _host, u'--port=%s' % _port, u'--mode=stingray']) + return cmd, output_files, log_name @@ -298,10 +313,35 @@ def check_preconditions(config): logger.info("Passed precondition checks") +class NoADBDeviceBackup(): + def __enter__(self): + self.device = NoADB() + return self + def __exit__(self, *args, **kwargs): + pass + def restore(self): + pass + +class NoADB(): + def reboot(self): + pass + def wait_for_net(self): + pass + def shell_output(self): + pass + def forward(self, *args): + pass + def get_process_list(self): + return [[1447, '/sbin/adbd', 'root']] + def restart(self): + pass def check_adb(): try: logger.info("Testing ADB connection") + if not _hasadb: + logger.debug('Dummy ADB, please remember install Marionette and Cert Test App to device ') + return NoADB() return adb_b2g.ADBB2G() except (mozdevice.ADBError, mozdevice.ADBTimeoutError) as e: logger.critical('Error connecting to device via adb (error: %s). Please be ' \ @@ -327,6 +367,10 @@ def check_root(device): def install_marionette(device, version): + if not _hasadb: + logger.debug('The marionette should be installed manually by user.') + return True + try: logger.info("Installing marionette extension") try: @@ -361,7 +405,7 @@ def ensure_settings(device): "screen.brightness": 1.0, "screen.timeout": 0.0} logger.info("Setting up device for testing") - with MarionetteSession(device) as marionette: + with MarionetteSession(device, _host, _port) as marionette: settings = gaiautils.Settings(marionette) for k, v in test_settings.iteritems(): settings.set(k, v) @@ -381,6 +425,10 @@ def wait_for_homescreen(marionette, timeout): def check_server(device): logger.info("Checking access to host machine") + + if not _hasadb: + return True + routes = [("GET", "/", test_handler)] host_ip = moznetwork.get_ip() @@ -411,7 +459,15 @@ def check_server(device): def list_tests(args, config): - for test, group in iter_test_lists(config["suites"]): + suites = OrderedDict.copy(config['suites']) + + if args.mode == 'stingray': + stingray_suite_keys = ['webapi', 'security'] + for key in suites.keys(): + if key not in stingray_suite_keys: + del suites[key] + + for test, group in iter_test_lists(suites, args.mode): print "%s:%s" % (test, group) return True @@ -429,7 +485,7 @@ def run_tests(args, config): check_preconditions(config) - with adb_b2g.DeviceBackup() as backup: + with DeviceBackup() as backup: device = backup.device runner = TestRunner(args, config) for suite, groups in runner.iter_suites(): @@ -464,6 +520,15 @@ def get_parser(): parser.add_argument('--list-tests', help='list all tests available to run', action='store_true') + parser.add_argument('-H', '--host', + help='Hostname or ip for target device', + action='store', default='localhost') + parser.add_argument('-P', '--port', + help='Port for target device', + action='store', default=2828) + parser.add_argument('-m', '--mode', + help='Test mode (stingray, phone) default (phone)', + action='store', default='phone') parser.add_argument('tests', metavar='TEST', help='tests to run', @@ -476,6 +541,17 @@ def main(): args = parser.parse_args() config = load_config(args.config) + global _host + global _port + global _hasadb + global DeviceBackup + _host = args.host + _port = int(args.port) + + if args.mode == 'stingray': + _hasadb = False + DeviceBackup = NoADBDeviceBackup + if args.list_tests: return list_tests(args, config) else: diff --git a/securitysuite/filesystem.py b/securitysuite/filesystem.py index 575f4134..bfe9563b 100644 --- a/securitysuite/filesystem.py +++ b/securitysuite/filesystem.py @@ -171,7 +171,7 @@ def whitelist_check(cls, name, flag='ok', whitelist=None): return r.match(name) is not None @classmethod - def run(cls, version=None): + def run(cls, version=None, host='localhost', port=2828, hasadb=True): logger = get_default_logger() try: @@ -231,7 +231,7 @@ def whitelist_check(cls, name, flag='ok', whitelist=None): return r.match(name) is not None @classmethod - def run(cls, version=None): + def run(cls, version=None, host='localhost', port=2828, hasadb=True): logger = get_default_logger() try: diff --git a/securitysuite/kernel.py b/securitysuite/kernel.py index 52158082..eaed020d 100644 --- a/securitysuite/kernel.py +++ b/securitysuite/kernel.py @@ -117,7 +117,7 @@ class seccomp(ExtraTest): module = sys.modules[__name__] @classmethod - def run(cls, version=None): + def run(cls, version=None, host='localhost', port=2828, hasadb=True): logger = get_default_logger() try: diff --git a/securitysuite/ssl.py b/securitysuite/ssl.py index 937a52a6..df131255 100644 --- a/securitysuite/ssl.py +++ b/securitysuite/ssl.py @@ -75,10 +75,12 @@ def js_certdump(): ''' - def __init__(self): + def __init__(self, hasadb=True): + self.hasadb = hasadb self.logger = get_default_logger() try: - self.dm = mozdevice.DeviceManagerADB(runAdbAsRoot=True) + if hasadb: + self.dm = mozdevice.DeviceManagerADB(runAdbAsRoot=True) except mozdevice.DMError as e: self.logger.error("Error connecting to device via adb (error: %s). Please be " \ "sure device is connected and 'remote debugging' is enabled." % \ @@ -86,15 +88,16 @@ def __init__(self): raise self.logger.debug("Attempting to set up port forwarding for marionette") + def get_via_marionette(self, host='localhost', port=2828): + if self.hasadb: + self.dm.forward("tcp:2828", "tcp:2828") + return run_marionette_script(certdump.js_certdump(), chrome=True, host=host, port=port) - def get_via_marionette(self): - self.dm.forward("tcp:2828", "tcp:2828") - return run_marionette_script(certdump.js_certdump(), chrome=True) - - def nssversion_via_marionette(self): - self.dm.forward("tcp:2828", "tcp:2828") - return run_marionette_script(certdump.js_nssversions(), chrome=True) + def nssversion_via_marionette(self, host='localhost', port=2828): + if self.hasadb: + self.dm.forward("tcp:2828", "tcp:2828") + return run_marionette_script(certdump.js_nssversions(), chrome=True, host=host, port=port) ####################################################################################################################### @@ -117,12 +120,12 @@ class certdb_info(ExtraTest): module = sys.modules[__name__] @classmethod - def run(cls, version=None): + def run(cls, version=None, host='localhost', port=2828, hasadb=True): logger = get_default_logger() try: - dumper = certdump() - certs = dumper.get_via_marionette() + dumper = certdump(hasadb) + certs = dumper.get_via_marionette(host, port) except: cls.log_status('FAIL', 'Failed to gather information from the device via Marionette.') return False @@ -145,12 +148,12 @@ class nssversion_info(ExtraTest): module = sys.modules[__name__] @classmethod - def run(cls, version=None): + def run(cls, version=None, host='localhost', port=2828, hasadb=True): logger = get_default_logger() try: - dumper = certdump() - versions = dumper.nssversion_via_marionette() + dumper = certdump(hasadb) + versions = dumper.nssversion_via_marionette(host, port) except: cls.log_status('FAIL', 'Failed to gather information from the device via Marionette.') return False diff --git a/securitysuite/suite.py b/securitysuite/suite.py index 81bf3d07..202c0633 100644 --- a/securitysuite/suite.py +++ b/securitysuite/suite.py @@ -34,10 +34,12 @@ def groupname(cls): return 'unknown' @staticmethod - def group_list(): + def group_list(mode='phone'): """ Returns a list of all groups in the test suite. """ + if mode == 'stingray': + return ['ssl'] groups = [] for t in ExtraTest.__subclasses__(): if t.groupname() not in groups: @@ -45,10 +47,12 @@ def group_list(): return groups @staticmethod - def test_list(group=None): + def test_list(group=None, mode='phone'): """ Returns a list of all tests, optionally filtered by group. """ + if mode == 'stingray' and group is not None: + group = 'ssl' if group is None: return ExtraTest.__subclasses__() else: @@ -59,11 +63,12 @@ def test_list(group=None): return tests @staticmethod - def run_groups(groups=[], version=None): + def run_groups(groups=[], version=None, host='localhost', port=2828, mode='phone'): + hasadb = mode == 'phone' logger = get_default_logger() if groups is None or len(groups) == 0: # run all groups - logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list())) - groups = ExtraTest.group_list() + logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list(mode=mode))) + groups = ExtraTest.group_list(mode=mode) else: logger.debug('running securitysuite tests for groups %s' % str(groups)) logger.suite_start(tests=groups) @@ -71,7 +76,7 @@ def run_groups(groups=[], version=None): logger.debug("running securitysuite test group %s" % g) logger.test_start(g) try: - ExtraTest.run(g, version=version) + ExtraTest.run(g, version=version, host=host, port=port, hasadb=hasadb) logger.test_end(g, 'OK') except: logger.critical(traceback.format_exc()) @@ -80,12 +85,12 @@ def run_groups(groups=[], version=None): logger.suite_end() @classmethod - def run(cls, group=None, version=None): + def run(cls, group=None, version=None, host='localhost', port=2828, hasadb=True): """ Runs all the tests, optionally just within the specified group. """ for t in cls.test_list(group): - t.run(version=version) + t.run(version=version, host=host, port=port, hasadb=hasadb) @classmethod def log_status(cls, status, msg): @@ -141,6 +146,15 @@ def securitycli(): help="B2G version") parser.add_argument("--ipython", dest="ipython", action="store_true", help="drop to ipython session") + parser.add_argument('-H', '--host', + help='Hostname or ip for target device', + action='store', default='localhost') + parser.add_argument('-P', '--port', + help='Port for target device', + action='store', default=2828) + parser.add_argument('-m', '--mode', + help='Test mode (stingray, phone) default (phone)', + action='store', default='phone') parser.add_argument("-v", dest="verbose", action="store_true", help="Verbose output") @@ -154,15 +168,20 @@ def securitycli(): try: logger.debug("security cli runnng with args %s" % args) if args.list_test_groups: - for group in ExtraTest.group_list(): + for group in ExtraTest.group_list(mode=args.mode): print group elif args.list_all_tests: - for test in ExtraTest.test_list(): + for test in ExtraTest.test_list(mode=args.mode): print "%s.%s" % (test.group, test.__name__) elif args.ipython: from IPython import embed embed() + elif args.mode == 'stingray': + ExtraTest.run_groups(args.include, + version=args.version, + host=args.host, port=int(args.port), + mode=args.mode) else: wait_for_adb_device() if not adb_has_root(): diff --git a/webapi_tests/apps/test_apps_basic.py b/webapi_tests/apps/test_apps_basic.py index 5599813b..e8256095 100644 --- a/webapi_tests/apps/test_apps_basic.py +++ b/webapi_tests/apps/test_apps_basic.py @@ -23,5 +23,6 @@ def test_get_all(self): if app["manifest"]["name"] == "CertTest App": self.assertEqual(app["manifest"]["developer"]["url"], "https://wiki.mozilla.org/Auto-tools", "Application developer url is different from https://wiki.mozilla.org/Auto-tools") - self.assertEqual(app["origin"], "app://certtest-app", "Application origin is different from app://certtest-app") + # for Stingray the app is installed manually, the origin is different from we expected + # self.assertEqual(app["origin"], "app://certtest-app", "Application origin is different from app://certtest-app") break diff --git a/webapi_tests/runner.py b/webapi_tests/runner.py index 7469d27d..8400bcbb 100644 --- a/webapi_tests/runner.py +++ b/webapi_tests/runner.py @@ -17,8 +17,10 @@ from webapi_tests import semiauto +stingray_test = ['apps', 'device_storage', 'geolocation', + 'moztime', 'notification', 'tcp_socket'] -def iter_tests(start_dir, pattern="test_*.py"): +def iter_tests(start_dir, pattern="test_*.py", mode='phone'): """List available Web API tests and yield a tuple of (group, tests), where tests is a list of test names.""" @@ -31,6 +33,8 @@ def iter_tests(start_dir, pattern="test_*.py"): visited.add(root) group = os.path.relpath(root, start_dir) + if mode == 'stingray' and group not in stingray_test: + continue tests = [] for file in files: @@ -80,6 +84,15 @@ def main(): help="Don't start a browser but wait for manual connection") parser.add_argument("--version", action="store", dest="version", help="B2G version") + parser.add_argument('-H', '--host', + help='Hostname or ip for target device', + action='store', default='localhost') + parser.add_argument('-P', '--port', + help='Port for target device', + action='store', default=2828) + parser.add_argument('-m', '--mode', + help='Test mode (stingray, phone) default (phone)', + action='store', default='phone') parser.add_argument( "-v", dest="verbose", action="store_true", help="Verbose output") commandline.add_logging_group(parser) @@ -93,7 +106,7 @@ def main(): parser.print_usage() sys.exit(1) - testgen = iter_tests(os.path.dirname(__file__)) + testgen = iter_tests(os.path.dirname(__file__), mode=args.mode) if args.list_test_groups: for group, _ in testgen: print(group) @@ -104,6 +117,10 @@ def main(): print("%s.%s" % (group, test)) return 0 + semiauto.testcase._host = args.host + semiauto.testcase._port = int(args.port) + semiauto.testcase._mode = args.mode + test_loader = semiauto.TestLoader(version=args.version) tests = test_loader.loadTestsFromNames( map(lambda t: "webapi_tests.%s" % t, args.include or [g for g, _ in testgen]), None) diff --git a/webapi_tests/semiauto/testcase.py b/webapi_tests/semiauto/testcase.py index 14f8b6d1..b44e01a4 100644 --- a/webapi_tests/semiauto/testcase.py +++ b/webapi_tests/semiauto/testcase.py @@ -21,6 +21,13 @@ instruction in a test.""" prompt_timeout = 600 # 10 minutes +# local variable for marionette +_host = 'localhost' +_port = 2828 +_mode = 'phone' + +def is_stingry(): + return _mode == 'stingray' class TestCase(unittest.TestCase): stored = threading.local() @@ -33,8 +40,9 @@ def __init__(self, *args, **kwargs): self.marionette, self.server, self.handler, self.app = None, None, None, None - device = mozdevice.DeviceManagerADB() - device.forward("tcp:2828", "tcp:2828") + if not is_stingry(): + device = mozdevice.DeviceManagerADB() + device.forward("tcp:2828", "tcp:2828") # Cleanups are run irrespective of whether setUp fails self.addCleanup(self.cleanup) @@ -65,10 +73,11 @@ def setUp(self): self.handler = env.handler self.assert_browser_connected() - self.marionette = TestCase.create_marionette() + self.marionette = TestCase.create_marionette(host=_host, port=_port) - if not certapp.is_installed(): - certapp.install(marionette=self.marionette, version=self.version) + if not is_stingry(): + if not certapp.is_installed(): + certapp.install(marionette=self.marionette, version=self.version) # Make sure we don't reuse the certapp context from a previous # testrun that was interrupted and left the certapp open. @@ -91,7 +100,7 @@ def assert_browser_connected(self): self.assertTrue(self.handler.connected, "Browser disconnected") @staticmethod - def create_marionette(): + def create_marionette(host='localhost', port=2828): """Returns current Marionette session, or creates one if one does not exist. @@ -99,7 +108,7 @@ def create_marionette(): m = TestCase.stored.marionette if m is None: - m = Marionette() + m = Marionette(host=host, port=port) m.wait_for_port() m.start_session() TestCase.stored.marionette = m @@ -183,8 +192,9 @@ def close_app_manually(self): "Please close the app manually by holding down the Home button " "and pressing the X above the %s card." % (certapp.name, certapp.name)) if not success: - device = mozdevice.DeviceManagerADB() - device.reboot(wait=True) + if not is_stingry(): + device = mozdevice.DeviceManagerADB() + device.reboot(wait=True) self.instruct("Please unlock the lockscreen (if present) after device reboots") self.fail("Failed attempts at closing certapp")