diff --git a/.kokoro/continuous/compliance.cfg b/.kokoro/continuous/compliance.cfg new file mode 100644 index 00000000..03f702f9 --- /dev/null +++ b/.kokoro/continuous/compliance.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "compliance" +} diff --git a/.kokoro/continuous/prerelease.cfg b/.kokoro/continuous/prerelease.cfg new file mode 100644 index 00000000..00bc8678 --- /dev/null +++ b/.kokoro/continuous/prerelease.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease" +} diff --git a/.kokoro/presubmit/compliance.cfg b/.kokoro/presubmit/compliance.cfg new file mode 100644 index 00000000..03f702f9 --- /dev/null +++ b/.kokoro/presubmit/compliance.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "compliance" +} diff --git a/.kokoro/presubmit/prerelease.cfg b/.kokoro/presubmit/prerelease.cfg new file mode 100644 index 00000000..00bc8678 --- /dev/null +++ b/.kokoro/presubmit/prerelease.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease" +} diff --git a/noxfile.py b/noxfile.py index ce847c57..b50a212c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import os import pathlib +import re import shutil import nox @@ -37,11 +38,10 @@ # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ - "lint", "unit", - "cover", "system", - "compliance", + "cover", + "lint", "lint_setup_py", "blacken", "docs", @@ -183,7 +183,77 @@ def system(session): ) -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "alembic", + "geoalchemy2", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "sqlalchemy", + "shapely", + # These are transitive dependencies, but we'd still like to know if a + # change in a prerelease there breaks this connector. + "google-cloud-core", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-testutils", + "mock", + "psutil", + "pytest", + "pytest-cov", + "pytz", + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) def compliance(session): """Run the SQLAlchemy dialect-compliance system tests""" constraints_path = str( @@ -193,8 +263,6 @@ def compliance(session): if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping") - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip("Credentials must be set via environment variable") if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": session.install("pyopenssl") if not os.path.exists(system_test_folder_path): @@ -204,7 +272,9 @@ def compliance(session): session.install( "mock", - "pytest", + # TODO: Allow latest version of pytest once SQLAlchemy 1.4.28+ is supported. + # See: https://github.com/googleapis/python-bigquery-sqlalchemy/issues/413 + "pytest<=7.0.0dev", "pytest-rerunfailures", "google-cloud-testutils", "-c", diff --git a/owlbot.py b/owlbot.py index 6f3ccbf0..cd3a7226 100644 --- a/owlbot.py +++ b/owlbot.py @@ -65,6 +65,12 @@ '"sqlalchemy_bigquery"', ) +s.replace( + ["noxfile.py"], + r"import shutil", + "import re\nimport shutil", +) + s.replace( ["noxfile.py"], "--cov=google", "--cov=sqlalchemy_bigquery", ) @@ -89,28 +95,84 @@ def place_before(path, text, *before_text, escape=None): "nox.options.stop_on_first_error = True", ) -old_sessions = ''' - "unit", - "system", - "cover", - "lint", -''' +prerelease = r''' +@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "alembic", + "geoalchemy2", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "sqlalchemy", + "shapely", + # These are transitive dependencies, but we'd still like to know if a + # change in a prerelease there breaks this connector. + "google-cloud-core", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-testutils", + "mock", + "psutil", + "pytest", + "pytest-cov", + "pytz", + ) -new_sessions = ''' - "lint", - "unit", - "cover", - "system", - "compliance", -''' + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) -s.replace( ["noxfile.py"], old_sessions, new_sessions) + +''' # Maybe we can get rid of this when we don't need pytest-rerunfailures, # which we won't need when BQ retries itself: # https://github.com/googleapis/python-bigquery/pull/837 compliance = ''' -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) def compliance(session): """Run the SQLAlchemy dialect-compliance system tests""" constraints_path = str( @@ -120,8 +182,6 @@ def compliance(session): if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping") - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip("Credentials must be set via environment variable") if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": session.install("pyopenssl") if not os.path.exists(system_test_folder_path): @@ -131,7 +191,9 @@ def compliance(session): session.install( "mock", - "pytest", + # TODO: Allow latest version of pytest once SQLAlchemy 1.4.28+ is supported. + # See: https://github.com/googleapis/python-bigquery-sqlalchemy/issues/413 + "pytest<=7.0.0dev", "pytest-rerunfailures", "google-cloud-testutils", "-c", @@ -166,7 +228,7 @@ def compliance(session): "noxfile.py", "@nox.session(python=DEFAULT_PYTHON_VERSION)\n" "def cover(session):", - compliance, + prerelease + compliance, escape="()", )