From 8395ebba57a07e1ad63ec211c2eb31da7e166477 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 1 Aug 2023 18:14:29 +0530 Subject: [PATCH 1/8] Run test for Python3.11 --- .circleci/config.yml | 44 ++++++++++++++++++- .circleci/scripts/pre_commit_readme_extra.py | 2 +- ..._commit_setup_cfg_python_3_11_all_extra.py | 36 +++++++++++++++ .pre-commit-config.yaml | 10 ++--- setup.cfg | 21 +++++++++ tests/core/triggers/test_external_task.py | 3 +- 6 files changed, 106 insertions(+), 10 deletions(-) create mode 100644 .circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ed1bd286..e0d48e2c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,6 +42,10 @@ workflows: <<: *main_and_release_branches - build-docs: <<: *all_branches_and_version_tag + - test-python3-11: + <<: *all_branches_and_version_tag + requires: + - markdown-link-check - test: name: test-python<< matrix.python_version >> matrix: @@ -72,7 +76,7 @@ executors: parameters: python_version: type: string - default: "3.8" + default: "3.7" docker: - image: cimg/python:<> @@ -92,7 +96,7 @@ jobs: - run: name: Run pre-commit command: | - pip install "pre-commit>=3.0.0" + pip install pre-commit pre-commit run markdown-link-check --all-files || { git --no-pager diff && false ; } - save_cache: key: v1-pc-cache-{{ checksum "pre-commit-cache-key.txt" }}- @@ -164,6 +168,42 @@ jobs: - ~/.pyenv/versions/ key: docs-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum ".readthedocs.yaml" }} + # This run unit tests for Python3.11 only and exclude Apache Hive test since + # currently, Airflow Hive provider excluded from Python3.11 + # Check the issue https://github.com/cloudera/python-sasl/issues/30 for detail + # PR to bring back Airflow Apache Hive provider https://github.com/apache/airflow/pull/32607 + test-python3-11: + description: "Test Python-3.11" + executor: + name: docker-executor + python_version: "3.11" + parallelism: 4 + steps: + - checkout + - restore_cache: + keys: + - deps-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }} + - deps-main-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }} + - run: + name: Install Dependencies + command: pip install -U -e .[test_python_3_11,tests] + - run: + name: Run tests + command: | + set -e + TEST_FILES=$(circleci tests \ + glob "tests/**/test_*.py" | \ + sed '/tests\/apache\/hive/d' | \ + circleci tests split --split-by=timings \ + ) + pytest --junit-xml=test-report/report.xml $TEST_FILES + - store_test_results: + path: test-report + - save_cache: + paths: + - ~/.cache/pip + - ~/.pyenv/versions/ + key: deps-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }} test: parameters: diff --git a/.circleci/scripts/pre_commit_readme_extra.py b/.circleci/scripts/pre_commit_readme_extra.py index 94618fed9..f62307970 100755 --- a/.circleci/scripts/pre_commit_readme_extra.py +++ b/.circleci/scripts/pre_commit_readme_extra.py @@ -10,7 +10,7 @@ config.read(repo_dir / "setup.cfg") all_extra = [] -extra_to_exclude = {"tests", "mypy", "docs"} +extra_to_exclude = {"tests", "mypy", "docs", "test_python_3_11"} all_extras = set(config["options.extras_require"].keys()) - extra_to_exclude readme_path = repo_dir / "README.rst" diff --git a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py new file mode 100644 index 000000000..961a8d262 --- /dev/null +++ b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +""" +Pre-commit hook to sync a "test_python_3_11" extra in setup.cfg. +It will contain all the dependencies apart from tests and mypy. +""" +import configparser +from pathlib import Path + +repo_dir = Path(__file__).parent.parent.parent + +config = configparser.ConfigParser(strict=False) +config.read(repo_dir / "setup.cfg") + +all_extra = set() +extra_to_exclude = {"tests", "mypy", "docs", "all", "test_python_3_11", "apache.hive"} +for key, extra_value in config["options.extras_require"].items(): + if key in extra_to_exclude: + continue + reqs = extra_value.split() + all_extra |= set(reqs) + +expected_test_python_3_11_extra = all_extra +found_test_python_3_11_extra = set(config["options.extras_require"].get("test_python_3_11", "").split()) +if not found_test_python_3_11_extra: + raise SystemExit("Missing 'test_python_3_11' extra in setup.cfg") + +""" +Use XOR operator ^ to find the missing dependencies instead of set A - set B +set A - set B will only show difference of set A from set B, but we want see overall diff +""" +diff_extras = expected_test_python_3_11_extra ^ found_test_python_3_11_extra +if diff_extras: + diff_extras_str = "\n \t" + "\n \t".join(sorted(diff_extras)) + raise SystemExit( + f"'test_python_3_11' extra in setup.cfg is missing some dependencies:\n {diff_extras_str}" + ) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c6a53ebb7..fbb1797bc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -98,11 +98,11 @@ repos: args: ["-c", "./pyproject.toml"] additional_dependencies: [".[toml]"] - - repo: https://github.com/rhysd/actionlint - rev: v1.6.25 - hooks: - - id: actionlint - args: ["-ignore", "shellcheck"] + # - repo: https://github.com/rhysd/actionlint + # rev: v1.6.25 + # hooks: + # - id: actionlint + # args: ["-ignore", "shellcheck"] - repo: local hooks: diff --git a/setup.cfg b/setup.cfg index 5d9f8f19b..c8a478234 100644 --- a/setup.cfg +++ b/setup.cfg @@ -138,6 +138,27 @@ all = paramiko snowflake-sqlalchemy>=1.4.4 # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4. +test_python_3_11 = + aiobotocore>=2.1.1 + apache-airflow-providers-amazon>=3.0.0 + apache-airflow-providers-apache-livy + apache-airflow-providers-cncf-kubernetes>=4 + apache-airflow-providers-databricks>=2.2.0 + apache-airflow-providers-google>=8.1.0 + apache-airflow-providers-http + apache-airflow-providers-snowflake + apache-airflow-providers-sftp + apache-airflow-providers-microsoft-azure + asyncssh>=2.12.0 + databricks-sql-connector>=2.0.4;python_version>='3.10' + apache-airflow-providers-dbt-cloud>=2.1.0 + gcloud-aio-bigquery + gcloud-aio-storage + kubernetes_asyncio + openlineage-airflow>=0.12.0 + paramiko + snowflake-sqlalchemy>=1.4.4 # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4. + [options.packages.find] include = astronomer.* diff --git a/tests/core/triggers/test_external_task.py b/tests/core/triggers/test_external_task.py index f2171bf99..416c90dc4 100644 --- a/tests/core/triggers/test_external_task.py +++ b/tests/core/triggers/test_external_task.py @@ -2,7 +2,6 @@ from unittest import mock from unittest.mock import AsyncMock -import asynctest import pytest from airflow import AirflowException from airflow.operators.empty import EmptyOperator @@ -222,7 +221,7 @@ async def test_deployment_task_exception(self, mock_run): assert TriggerEvent({"state": "error", "message": "Test exception"}) == actual @pytest.mark.asyncio - @asynctest.patch("astronomer.providers.http.hooks.http.HttpHookAsync.run") + @mock.patch("astronomer.providers.http.hooks.http.HttpHookAsync.run") async def test_deployment_complete(self, mock_run): """Assert ExternalDeploymentTaskTrigger runs and complete the run in success state""" mock.AsyncMock(HttpHookAsync) From ea171473e2798406c69162b769f9b46e1f03225b Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 1 Aug 2023 20:39:23 +0530 Subject: [PATCH 2/8] Enable action lint --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fbb1797bc..c6a53ebb7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -98,11 +98,11 @@ repos: args: ["-c", "./pyproject.toml"] additional_dependencies: [".[toml]"] - # - repo: https://github.com/rhysd/actionlint - # rev: v1.6.25 - # hooks: - # - id: actionlint - # args: ["-ignore", "shellcheck"] + - repo: https://github.com/rhysd/actionlint + rev: v1.6.25 + hooks: + - id: actionlint + args: ["-ignore", "shellcheck"] - repo: local hooks: From 4f9950ed699a09f7f39ef1aa52fd82af127d66cd Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 1 Aug 2023 20:48:41 +0530 Subject: [PATCH 3/8] Update pre-commit --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e0d48e2c0..494ae83ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: - run: name: Run pre-commit command: | - pip install pre-commit + pip install "pre-commit>=3.0.0" pre-commit run markdown-link-check --all-files || { git --no-pager diff && false ; } - save_cache: key: v1-pc-cache-{{ checksum "pre-commit-cache-key.txt" }}- From 60665b9a3a6d2a0c8a2ec4ff9f7fea73f5534a54 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 1 Aug 2023 20:57:09 +0530 Subject: [PATCH 4/8] Update pre-commit --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 494ae83ca..76f8c6a8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,7 +76,7 @@ executors: parameters: python_version: type: string - default: "3.7" + default: "3.8" docker: - image: cimg/python:<> From ac598df03d2b3e11e97bc40bf3ceb1a0dde77032 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Wed, 2 Aug 2023 11:58:01 +0530 Subject: [PATCH 5/8] Apply review suggestion --- .../pre_commit_setup_cfg_python_3_11_all_extra.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py index 961a8d262..92646b498 100644 --- a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py +++ b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py @@ -11,15 +11,14 @@ config = configparser.ConfigParser(strict=False) config.read(repo_dir / "setup.cfg") -all_extra = set() extra_to_exclude = {"tests", "mypy", "docs", "all", "test_python_3_11", "apache.hive"} -for key, extra_value in config["options.extras_require"].items(): - if key in extra_to_exclude: - continue - reqs = extra_value.split() - all_extra |= set(reqs) +expected_test_python_3_11_extra = { + req + for key, extra_value in config["options.extras_require"].items() + for req in extra_value.split() + if key not in extra_to_exclude +} -expected_test_python_3_11_extra = all_extra found_test_python_3_11_extra = set(config["options.extras_require"].get("test_python_3_11", "").split()) if not found_test_python_3_11_extra: raise SystemExit("Missing 'test_python_3_11' extra in setup.cfg") From a3667e537f383d19fe6501e32f90053e62019150 Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Wed, 2 Aug 2023 12:33:12 +0530 Subject: [PATCH 6/8] Apply review suggestion Co-authored-by: Wei Lee --- .circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py index 92646b498..b40bce4de 100644 --- a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py +++ b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py @@ -19,7 +19,6 @@ if key not in extra_to_exclude } -found_test_python_3_11_extra = set(config["options.extras_require"].get("test_python_3_11", "").split()) if not found_test_python_3_11_extra: raise SystemExit("Missing 'test_python_3_11' extra in setup.cfg") From e6db42ff0b3558d3aaf611d69049ee69e152aa07 Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Wed, 2 Aug 2023 12:36:47 +0530 Subject: [PATCH 7/8] Apply review suggestion --- .circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py index b40bce4de..95957172b 100644 --- a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py +++ b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py @@ -18,7 +18,7 @@ for req in extra_value.split() if key not in extra_to_exclude } - +found_test_python_3_11_extra = set(config["options.extras_require"].get("test_python_3_11", "").split()) if not found_test_python_3_11_extra: raise SystemExit("Missing 'test_python_3_11' extra in setup.cfg") From bd32dc5ab8fc3b0361a1777cc01884ab97860744 Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Wed, 2 Aug 2023 15:29:40 +0530 Subject: [PATCH 8/8] Update .circleci/config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 76f8c6a8a..905543c5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -168,8 +168,8 @@ jobs: - ~/.pyenv/versions/ key: docs-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum ".readthedocs.yaml" }} - # This run unit tests for Python3.11 only and exclude Apache Hive test since - # currently, Airflow Hive provider excluded from Python3.11 + # This runs unit tests for Python3.11 only and excludes the Apache Hive test since + # Currently, the Airflow Hive provider is excluded from Python3.11 in Airflow # Check the issue https://github.com/cloudera/python-sasl/issues/30 for detail # PR to bring back Airflow Apache Hive provider https://github.com/apache/airflow/pull/32607 test-python3-11: