From c07f3c0a7e6147f0005d1e59b24f80cd042623cd Mon Sep 17 00:00:00 2001 From: Jerjou Cheng Date: Thu, 31 Mar 2016 14:46:17 -0700 Subject: [PATCH] Auto-detect directories to test. --- .coveragerc | 1 + TESTING.md | 40 +++++++------ .../{test_datastore.py => datastore_test.py} | 0 .../{test_env_vars.py => env_vars_test.py} | 0 .../{test_login.py => login_test.py} | 0 .../{test_mail.py => mail_test.py} | 0 ...{test_task_queue.py => task_queue_test.py} | 0 nox.py | 59 ++++++------------- 8 files changed, 43 insertions(+), 57 deletions(-) rename appengine/localtesting/{test_datastore.py => datastore_test.py} (100%) rename appengine/localtesting/{test_env_vars.py => env_vars_test.py} (100%) rename appengine/localtesting/{test_login.py => login_test.py} (100%) rename appengine/localtesting/{test_mail.py => mail_test.py} (100%) rename appengine/localtesting/{test_task_queue.py => task_queue_test.py} (100%) diff --git a/.coveragerc b/.coveragerc index b332653b7270..45ede645eb7a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,6 +8,7 @@ include = datastore/* managed_vms/* monitoring/* + speech/* storage/* [report] exclude_lines = diff --git a/TESTING.md b/TESTING.md index 0d57ff240c2f..fc04d1fdb596 100644 --- a/TESTING.md +++ b/TESTING.md @@ -4,10 +4,10 @@ The tests in this repository are system tests and run against live services, the Before you can run tests locally you must have: -* The latest [tox](https://tox.readthedocs.org/en/latest/), +* The latest [nox](https://nox.readthedocs.org/en/latest/), [pip](https://pypi.python.org/pypi/pip), and [gcp-python-repo-tools](https://pypi.python.org/pypi/gcp-python-repo-tools) installed. - $ sudo pip install --upgrade tox pip gcp-python-repo-tools + $ sudo pip install --upgrade nox-automation pip gcp-python-repo-tools * The [Google Cloud SDK](https://cloud.google.com/sdk/) installed. You can do so with the following command: @@ -50,29 +50,35 @@ If you want to run the Google App Engine tests, you will need: ### Test environments -We use [tox](https://tox.readthedocs.org/en/latest/) to configure -multiple python environments: +We use [nox](https://nox.readthedocs.org/en/latest/) to configure +multiple python sessions: -* ``py27`` and ``py34`` contains tests for samples that run in a normal Python 2.7 pr 3.4 environment. This is everything outside of the ``appengine`` directory that isn't slow or flaky. -* ``py27-all`` and ``py34-all`` runs all tests except for App Engine tests. This can time some time and some tests are flaky. -* ``gae`` contains tests for samples that run only in Google App Engine. This is (mostly) everything in the ``appengine`` directory. -* ``pep8`` just runs the linter. +* ``tests`` contains tests for samples that run in a normal Python 2.7 or 3.4 + environment. This is everything outside of the ``appengine`` directory. It's + parameterized to run all the tests using the 2.7 and 3.4 interpreters. +* ``gae`` contains tests for samples that run only in Google App Engine. This is + (mostly) everything in the ``appengine`` directory. +* ``lint`` just runs the linter. -To run tests for a particular environment, invoke tox with the ``-e`` -flag: +To see a list of the available sessions: - tox -e py27 + nox -l -To run one particular test suite or provide additional parameters to -``py.test``, invoke tox like this: +To run tests for a particular session, with a particular parameter, invoke nox +with the ``-s`` flag: - toxe -e py27 -- storage/api + nox -s "tests(interpreter='python2.7')" -### Adding new tests +To run one particular session or provide additional parameters to ``py.test``, +invoke nox like this: + + nox -s tests -- storage/api -When adding a new top-level directory, be sure to edit ``.coveragerc`` and ``tox.ini`` to include it in tests and coverage reporting. +### Adding new tests +When adding a new top-level directory, be sure to edit ``.coveragerc`` to +include it in coverage reporting. To add new tests that require Google App Engine, please place them in the ``appengine`` directory if possible. If you place them elsewhere, -you will need to modify ``tox.ini`` to make the environments +you will need to modify ``nox.py`` to make the environments appropriately run or ignore your test. diff --git a/appengine/localtesting/test_datastore.py b/appengine/localtesting/datastore_test.py similarity index 100% rename from appengine/localtesting/test_datastore.py rename to appengine/localtesting/datastore_test.py diff --git a/appengine/localtesting/test_env_vars.py b/appengine/localtesting/env_vars_test.py similarity index 100% rename from appengine/localtesting/test_env_vars.py rename to appengine/localtesting/env_vars_test.py diff --git a/appengine/localtesting/test_login.py b/appengine/localtesting/login_test.py similarity index 100% rename from appengine/localtesting/test_login.py rename to appengine/localtesting/login_test.py diff --git a/appengine/localtesting/test_mail.py b/appengine/localtesting/mail_test.py similarity index 100% rename from appengine/localtesting/test_mail.py rename to appengine/localtesting/mail_test.py diff --git a/appengine/localtesting/test_task_queue.py b/appengine/localtesting/task_queue_test.py similarity index 100% rename from appengine/localtesting/test_task_queue.py rename to appengine/localtesting/task_queue_test.py diff --git a/nox.py b/nox.py index a3d3f0a0e15c..6772a97d4bff 100644 --- a/nox.py +++ b/nox.py @@ -25,44 +25,7 @@ '-x', '--no-success-flaky-report', '--cov', '--cov-config', '.coveragerc', '--cov-append', '--cov-report='] -SAMPLES = [ - 'bigquery/api', - 'blog/introduction_to_data_models_in_cloud_datastore', - 'cloud_logging/api', - 'compute/api', - 'compute/autoscaler/demo', - 'datastore/api', - 'managed_vms/cloudsql', - 'managed_vms/datastore', - 'managed_vms/disk', - 'managed_vms/extending_runtime', - 'managed_vms/hello_world', - 'managed_vms/hello_world_compat', - 'managed_vms/memcache', - 'managed_vms/pubsub', - 'managed_vms/static_files', - 'managed_vms/storage', - 'monitoring/api', - 'storage/api', -] - -GAE_SAMPLES = [ - 'appengine/app_identity/signing', - 'appengine/bigquery', - 'appengine/blobstore', - 'appengine/cloudsql', - 'appengine/images', - 'appengine/localtesting', - 'appengine/logging/reading_logs', - 'appengine/logging/writing_logs', - 'appengine/mailgun', - 'appengine/memcache/guestbook', - 'appengine/multitenancy', - 'appengine/ndb/modeling', - 'appengine/ndb/overview', - 'appengine/ndb/transactions', - 'appengine/storage', -] +SESSION_TESTS_BLACKLIST = set(('appengine', 'testing')) def session_lint(session): @@ -94,6 +57,21 @@ def session_reqcheck(session): session.run('gcprepotools', command, reqfile) +def collect_sample_dirs(start_dir, blacklist=set()): + """Recursively collects a list of dirs that contain tests.""" + # Collect all the directories that have tests in them. + for parent, subdirs, files in os.walk(start_dir): + if any(f for f in files if f[-8:] == '_test.py'): + # Don't recurse further, since py.test will do that. + del subdirs[:] + # This dir has tests in it. yield it. + yield parent + else: + # Filter out dirs we don't want to recurse into + subdirs[:] = [s for s in subdirs + if s[0].isalpha() and s not in blacklist] + + @nox.parametrize('interpreter', ['python2.7', 'python3.4']) def session_tests(session, interpreter, extra_pytest_args=None): session.interpreter = interpreter @@ -106,7 +84,8 @@ def session_tests(session, interpreter, extra_pytest_args=None): # session.posargs is any leftover arguments from the command line, which # allows users to run a particular test instead of all of them. - for sample in (session.posargs or SAMPLES): + for sample in (session.posargs or + collect_sample_dirs('.', SESSION_TESTS_BLACKLIST)): session.run( 'py.test', sample, *pytest_args, @@ -130,7 +109,7 @@ def session_gae(session, extra_pytest_args=None): pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or []) - for sample in (session.posargs or GAE_SAMPLES): + for sample in (session.posargs or collect_sample_dirs('appengine')): session.run( 'py.test', sample, *pytest_args,