From 44adbbada4b9d575cec7b247d39a8f9f1c6fa887 Mon Sep 17 00:00:00 2001 From: Jed Cunningham Date: Fri, 9 Aug 2024 15:23:14 -0600 Subject: [PATCH 1/3] Remove support for deprecated imports like operators/hooks/sensors --- airflow/executors/__init__.py | 49 -------- airflow/hooks/__init__.py | 73 ------------ airflow/macros/__init__.py | 10 -- airflow/operators/__init__.py | 177 ---------------------------- airflow/sensors/__init__.py | 55 --------- newsfragments/41368.significant.rst | 12 ++ 6 files changed, 12 insertions(+), 364 deletions(-) create mode 100644 newsfragments/41368.significant.rst diff --git a/airflow/executors/__init__.py b/airflow/executors/__init__.py index 7c936a16b864..13a83393a912 100644 --- a/airflow/executors/__init__.py +++ b/airflow/executors/__init__.py @@ -14,52 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Executors.""" - -from __future__ import annotations - -from airflow.utils.deprecation_tools import add_deprecated_classes - -__deprecated_classes = { - "celery_executor": { - "app": "airflow.providers.celery.executors.celery_executor_utils.app", - "CeleryExecutor": "airflow.providers.celery.executors.celery_executor.CeleryExecutor", - }, - "celery_kubernetes_executor": { - "CeleryKubernetesExecutor": "airflow.providers.celery.executors." - "celery_kubernetes_executor.CeleryKubernetesExecutor", - }, - "dask_executor": { - "DaskExecutor": "airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor", - }, - "kubernetes_executor": { - "KubernetesExecutor": "airflow.providers.cncf.kubernetes." - "executors.kubernetes_executor.KubernetesExecutor", - }, - "kubernetes_executor_types": { - "ALL_NAMESPACES": "airflow.providers.cncf.kubernetes." - "executors.kubernetes_executor_types.ALL_NAMESPACES", - "POD_EXECUTOR_DONE_KEY": "airflow.providers.cncf.kubernetes." - "executors.kubernetes_executor_types.POD_EXECUTOR_DONE_KEY", - }, - "kubernetes_executor_utils": { - "AirflowKubernetesScheduler": "airflow.providers.cncf.kubernetes." - "executors.kubernetes_executor_utils.AirflowKubernetesScheduler", - "KubernetesJobWatcher": "airflow.providers.cncf.kubernetes." - "executors.kubernetes_executor_utils.KubernetesJobWatcher", - "ResourceVersion": "airflow.providers.cncf.kubernetes." - "executors.kubernetes_executor_utils.ResourceVersion", - }, - "local_kubernetes_executor": { - "LocalKubernetesExecutor": "airflow.providers.cncf.kubernetes.executors.LocalKubernetesExecutor", - }, -} - -add_deprecated_classes( - __deprecated_classes, - __name__, - {}, - "For Celery executors, the `celery` provider should be >= 3.3.0. " - "For Kubernetes executors, the `cncf.kubernetes` provider should be >= 7.4.0 for that. " - "For Dask executors, any version of `daskexecutor` provider is needed.", -) diff --git a/airflow/hooks/__init__.py b/airflow/hooks/__init__.py index 1baff07c0bb1..f1ad214d7148 100644 --- a/airflow/hooks/__init__.py +++ b/airflow/hooks/__init__.py @@ -16,76 +16,3 @@ # specific language governing permissions and limitations # under the License. # fmt:, off -"""Hooks.""" - -from __future__ import annotations - -from airflow.utils.deprecation_tools import add_deprecated_classes - -__deprecated_classes = { - "S3_hook": { - "S3Hook": "airflow.providers.amazon.aws.hooks.s3.S3Hook", - "provide_bucket_name": "airflow.providers.amazon.aws.hooks.s3.provide_bucket_name", - }, - "base_hook": { - "BaseHook": "airflow.hooks.base.BaseHook", - }, - "dbapi_hook": { - "DbApiHook": "airflow.providers.common.sql.hooks.sql.DbApiHook", - }, - "docker_hook": { - "DockerHook": "airflow.providers.docker.hooks.docker.DockerHook", - }, - "druid_hook": { - "DruidDbApiHook": "airflow.providers.apache.druid.hooks.druid.DruidDbApiHook", - "DruidHook": "airflow.providers.apache.druid.hooks.druid.DruidHook", - }, - "hive_hooks": { - "HIVE_QUEUE_PRIORITIES": "airflow.providers.apache.hive.hooks.hive.HIVE_QUEUE_PRIORITIES", - "HiveCliHook": "airflow.providers.apache.hive.hooks.hive.HiveCliHook", - "HiveMetastoreHook": "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook", - "HiveServer2Hook": "airflow.providers.apache.hive.hooks.hive.HiveServer2Hook", - }, - "http_hook": { - "HttpHook": "airflow.providers.http.hooks.http.HttpHook", - }, - "jdbc_hook": { - "JdbcHook": "airflow.providers.jdbc.hooks.jdbc.JdbcHook", - "jaydebeapi": "airflow.providers.jdbc.hooks.jdbc.jaydebeapi", - }, - "mssql_hook": { - "MsSqlHook": "airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook", - }, - "mysql_hook": { - "MySqlHook": "airflow.providers.mysql.hooks.mysql.MySqlHook", - }, - "oracle_hook": { - "OracleHook": "airflow.providers.oracle.hooks.oracle.OracleHook", - }, - "pig_hook": { - "PigCliHook": "airflow.providers.apache.pig.hooks.pig.PigCliHook", - }, - "postgres_hook": { - "PostgresHook": "airflow.providers.postgres.hooks.postgres.PostgresHook", - }, - "presto_hook": { - "PrestoHook": "airflow.providers.presto.hooks.presto.PrestoHook", - }, - "samba_hook": { - "SambaHook": "airflow.providers.samba.hooks.samba.SambaHook", - }, - "slack_hook": { - "SlackHook": "airflow.providers.slack.hooks.slack.SlackHook", - }, - "sqlite_hook": { - "SqliteHook": "airflow.providers.sqlite.hooks.sqlite.SqliteHook", - }, - "webhdfs_hook": { - "WebHDFSHook": "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook", - }, - "zendesk_hook": { - "ZendeskHook": "airflow.providers.zendesk.hooks.zendesk.ZendeskHook", - }, -} - -add_deprecated_classes(__deprecated_classes, __name__) diff --git a/airflow/macros/__init__.py b/airflow/macros/__init__.py index b18984b94e5a..be4554818acf 100644 --- a/airflow/macros/__init__.py +++ b/airflow/macros/__init__.py @@ -29,20 +29,10 @@ from babel.dates import LC_TIME, format_datetime import airflow.utils.yaml as yaml # noqa: F401 -from airflow.utils.deprecation_tools import add_deprecated_classes if TYPE_CHECKING: from pendulum import DateTime -__deprecated_classes = { - "hive": { - "closest_ds_partition": "airflow.providers.apache.hive.macros.hive.closest_ds_partition", - "max_partition": "airflow.providers.apache.hive.macros.hive.max_partition", - }, -} - -add_deprecated_classes(__deprecated_classes, __name__) - def ds_add(ds: str, days: int) -> str: """ diff --git a/airflow/operators/__init__.py b/airflow/operators/__init__.py index f3195123b6a4..77fcc8c0c0e9 100644 --- a/airflow/operators/__init__.py +++ b/airflow/operators/__init__.py @@ -16,180 +16,3 @@ # specific language governing permissions and limitations # under the License. # fmt: off -""" -Operators. - -:sphinx-autoapi-skip: -""" - -from __future__ import annotations - -from airflow.utils.deprecation_tools import add_deprecated_classes - -__deprecated_classes = { - "bash_operator": { - "BashOperator": "airflow.operators.bash.BashOperator", - }, - "branch_operator": { - "BaseBranchOperator": "airflow.operators.branch.BaseBranchOperator", - }, - "check_operator": { - "SQLCheckOperator": "airflow.providers.common.sql.operators.sql.SQLCheckOperator", - "SQLIntervalCheckOperator": "airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator", - "SQLThresholdCheckOperator": "airflow.providers.common.sql.operators.sql.SQLThresholdCheckOperator", - "SQLValueCheckOperator": "airflow.providers.common.sql.operators.sql.SQLValueCheckOperator", - "CheckOperator": "airflow.providers.common.sql.operators.sql.SQLCheckOperator", - "IntervalCheckOperator": "airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator", - "ThresholdCheckOperator": "airflow.providers.common.sql.operators.sql.SQLThresholdCheckOperator", - "ValueCheckOperator": "airflow.providers.common.sql.operators.sql.SQLValueCheckOperator", - }, - "dagrun_operator": { - "TriggerDagRunLink": "airflow.operators.trigger_dagrun.TriggerDagRunLink", - "TriggerDagRunOperator": "airflow.operators.trigger_dagrun.TriggerDagRunOperator", - }, - "docker_operator": { - "DockerOperator": "airflow.providers.docker.operators.docker.DockerOperator", - }, - "druid_check_operator": { - "DruidCheckOperator": "airflow.providers.apache.druid.operators.druid_check.DruidCheckOperator", - }, - "dummy": { - "EmptyOperator": "airflow.operators.empty.EmptyOperator", - "DummyOperator": "airflow.operators.empty.EmptyOperator", - }, - "dummy_operator": { - "EmptyOperator": "airflow.operators.empty.EmptyOperator", - "DummyOperator": "airflow.operators.empty.EmptyOperator", - }, - "email_operator": { - "EmailOperator": "airflow.operators.email.EmailOperator", - }, - "gcs_to_s3": { - "GCSToS3Operator": "airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator", - }, - "google_api_to_s3_transfer": { - "GoogleApiToS3Operator": ( - "airflow.providers.amazon.aws.transfers.google_api_to_s3.GoogleApiToS3Operator" - ), - "GoogleApiToS3Transfer": ( - "airflow.providers.amazon.aws.transfers.google_api_to_s3.GoogleApiToS3Operator" - ), - }, - "hive_operator": { - "HiveOperator": "airflow.providers.apache.hive.operators.hive.HiveOperator", - }, - "hive_stats_operator": { - "HiveStatsCollectionOperator": ( - "airflow.providers.apache.hive.operators.hive_stats.HiveStatsCollectionOperator" - ), - }, - "hive_to_druid": { - "HiveToDruidOperator": "airflow.providers.apache.druid.transfers.hive_to_druid.HiveToDruidOperator", - "HiveToDruidTransfer": "airflow.providers.apache.druid.transfers.hive_to_druid.HiveToDruidOperator", - }, - "hive_to_mysql": { - "HiveToMySqlOperator": "airflow.providers.apache.hive.transfers.hive_to_mysql.HiveToMySqlOperator", - "HiveToMySqlTransfer": "airflow.providers.apache.hive.transfers.hive_to_mysql.HiveToMySqlOperator", - }, - "hive_to_samba_operator": { - "HiveToSambaOperator": "airflow.providers.apache.hive.transfers.hive_to_samba.HiveToSambaOperator", - }, - "http_operator": { - "SimpleHttpOperator": "airflow.providers.http.operators.http.SimpleHttpOperator", - }, - "jdbc_operator": { - "JdbcOperator": "airflow.providers.jdbc.operators.jdbc.JdbcOperator", - }, - "latest_only_operator": { - "LatestOnlyOperator": "airflow.operators.latest_only.LatestOnlyOperator", - }, - "mssql_operator": { - "MsSqlOperator": "airflow.providers.microsoft.mssql.operators.mssql.MsSqlOperator", - }, - "mssql_to_hive": { - "MsSqlToHiveOperator": "airflow.providers.apache.hive.transfers.mssql_to_hive.MsSqlToHiveOperator", - "MsSqlToHiveTransfer": "airflow.providers.apache.hive.transfers.mssql_to_hive.MsSqlToHiveOperator", - }, - "mysql_operator": { - "MySqlOperator": "airflow.providers.mysql.operators.mysql.MySqlOperator", - }, - "mysql_to_hive": { - "MySqlToHiveOperator": "airflow.providers.apache.hive.transfers.mysql_to_hive.MySqlToHiveOperator", - "MySqlToHiveTransfer": "airflow.providers.apache.hive.transfers.mysql_to_hive.MySqlToHiveOperator", - }, - "oracle_operator": { - "OracleOperator": "airflow.providers.oracle.operators.oracle.OracleOperator", - }, - "papermill_operator": { - "PapermillOperator": "airflow.providers.papermill.operators.papermill.PapermillOperator", - }, - "pig_operator": { - "PigOperator": "airflow.providers.apache.pig.operators.pig.PigOperator", - }, - "postgres_operator": { - "Mapping": "airflow.providers.postgres.operators.postgres.Mapping", - "PostgresOperator": "airflow.providers.postgres.operators.postgres.PostgresOperator", - }, - "presto_check_operator": { - "SQLCheckOperator": "airflow.providers.common.sql.operators.sql.SQLCheckOperator", - "SQLIntervalCheckOperator": "airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator", - "SQLValueCheckOperator": "airflow.providers.common.sql.operators.sql.SQLValueCheckOperator", - "PrestoCheckOperator": "airflow.providers.common.sql.operators.sql.SQLCheckOperator", - "PrestoIntervalCheckOperator": "airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator", - "PrestoValueCheckOperator": "airflow.providers.common.sql.operators.sql.SQLValueCheckOperator", - }, - "presto_to_mysql": { - "PrestoToMySqlOperator": "airflow.providers.mysql.transfers.presto_to_mysql.PrestoToMySqlOperator", - "PrestoToMySqlTransfer": "airflow.providers.mysql.transfers.presto_to_mysql.PrestoToMySqlOperator", - }, - "python_operator": { - "BranchPythonOperator": "airflow.operators.python.BranchPythonOperator", - "PythonOperator": "airflow.operators.python.PythonOperator", - "PythonVirtualenvOperator": "airflow.operators.python.PythonVirtualenvOperator", - "ShortCircuitOperator": "airflow.operators.python.ShortCircuitOperator", - }, - "redshift_to_s3_operator": { - "RedshiftToS3Operator": "airflow.providers.amazon.aws.transfers.redshift_to_s3.RedshiftToS3Operator", - "RedshiftToS3Transfer": "airflow.providers.amazon.aws.transfers.redshift_to_s3.RedshiftToS3Operator", - }, - "s3_file_transform_operator": { - "S3FileTransformOperator": ( - "airflow.providers.amazon.aws.operators.s3_file_transform.S3FileTransformOperator" - ), - }, - "s3_to_hive_operator": { - "S3ToHiveOperator": "airflow.providers.apache.hive.transfers.s3_to_hive.S3ToHiveOperator", - "S3ToHiveTransfer": "airflow.providers.apache.hive.transfers.s3_to_hive.S3ToHiveOperator", - }, - "s3_to_redshift_operator": { - "S3ToRedshiftOperator": "airflow.providers.amazon.aws.transfers.s3_to_redshift.S3ToRedshiftOperator", - "S3ToRedshiftTransfer": "airflow.providers.amazon.aws.transfers.s3_to_redshift.S3ToRedshiftOperator", - }, - "slack_operator": { - "SlackAPIOperator": "airflow.providers.slack.operators.slack.SlackAPIOperator", - "SlackAPIPostOperator": "airflow.providers.slack.operators.slack.SlackAPIPostOperator", - }, - "sql": { - "BaseSQLOperator": "airflow.providers.common.sql.operators.sql.BaseSQLOperator", - "BranchSQLOperator": "airflow.providers.common.sql.operators.sql.BranchSQLOperator", - "SQLCheckOperator": "airflow.providers.common.sql.operators.sql.SQLCheckOperator", - "SQLColumnCheckOperator": "airflow.providers.common.sql.operators.sql.SQLColumnCheckOperator", - "SQLIntervalCheckOperator": "airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator", - "SQLTableCheckOperator": "airflow.providers.common.sql.operators.sql.SQLTableCheckOperator", - "SQLThresholdCheckOperator": "airflow.providers.common.sql.operators.sql.SQLThresholdCheckOperator", - "SQLValueCheckOperator": "airflow.providers.common.sql.operators.sql.SQLValueCheckOperator", - "_convert_to_float_if_possible": ( - "airflow.providers.common.sql.operators.sql._convert_to_float_if_possible" - ), - "parse_boolean": "airflow.providers.common.sql.operators.sql.parse_boolean", - }, - "sql_branch_operator": { - "BranchSQLOperator": "airflow.providers.common.sql.operators.sql.BranchSQLOperator", - "BranchSqlOperator": "airflow.providers.common.sql.operators.sql.BranchSQLOperator", - }, - "sqlite_operator": { - "SqliteOperator": "airflow.providers.sqlite.operators.sqlite.SqliteOperator", - }, -} - -add_deprecated_classes(__deprecated_classes, __name__) diff --git a/airflow/sensors/__init__.py b/airflow/sensors/__init__.py index 3d1627cc036a..77fcc8c0c0e9 100644 --- a/airflow/sensors/__init__.py +++ b/airflow/sensors/__init__.py @@ -16,58 +16,3 @@ # specific language governing permissions and limitations # under the License. # fmt: off -""" -Sensors. - -:sphinx-autoapi-skip: -""" -from __future__ import annotations - -from airflow.utils.deprecation_tools import add_deprecated_classes - -__deprecated_classes = { - "base_sensor_operator": { - "BaseSensorOperator": "airflow.sensors.base.BaseSensorOperator", - }, - "date_time_sensor": { - "DateTimeSensor": "airflow.sensors.date_time.DateTimeSensor", - }, - "external_task_sensor": { - "ExternalTaskMarker": "airflow.sensors.external_task.ExternalTaskMarker", - "ExternalTaskSensor": "airflow.sensors.external_task.ExternalTaskSensor", - "ExternalTaskSensorLink": "airflow.sensors.external_task.ExternalTaskSensorLink", - }, - "hive_partition_sensor": { - "HivePartitionSensor": "airflow.providers.apache.hive.sensors.hive_partition.HivePartitionSensor", - }, - "http_sensor": { - "HttpSensor": "airflow.providers.http.sensors.http.HttpSensor", - }, - "metastore_partition_sensor": { - "MetastorePartitionSensor": ( - "airflow.providers.apache.hive.sensors.metastore_partition.MetastorePartitionSensor" - ), - }, - "named_hive_partition_sensor": { - "NamedHivePartitionSensor": ( - "airflow.providers.apache.hive.sensors.named_hive_partition.NamedHivePartitionSensor" - ), - }, - "s3_key_sensor": { - "S3KeySensor": "airflow.providers.amazon.aws.sensors.s3.S3KeySensor", - }, - "sql": { - "SqlSensor": "airflow.providers.common.sql.sensors.sql.SqlSensor", - }, - "sql_sensor": { - "SqlSensor": "airflow.providers.common.sql.sensors.sql.SqlSensor", - }, - "time_delta_sensor": { - "TimeDeltaSensor": "airflow.sensors.time_delta.TimeDeltaSensor", - }, - "web_hdfs_sensor": { - "WebHdfsSensor": "airflow.providers.apache.hdfs.sensors.web_hdfs.WebHdfsSensor", - }, -} - -add_deprecated_classes(__deprecated_classes, __name__) diff --git a/newsfragments/41368.significant.rst b/newsfragments/41368.significant.rst new file mode 100644 index 000000000000..26b76b0f3a1e --- /dev/null +++ b/newsfragments/41368.significant.rst @@ -0,0 +1,12 @@ +Support for deprecated core imports removed + +Support for importing classes etc from the following locations was deprecated at various times during Airflow 2s lifecycle, and has been removed: + +- ``airflow.executors`` +- ``airflow.hooks`` +- ``airflow.macros`` +- ``airflow.operators`` +- ``airflow.sensors`` + +Instead, import from the right provider or more specific module instead. +For example, instead of ``from airflow.sensors import TimeDeltaSensor``, use ``from airflow.sensors.time_delta import TimeDeltaSensor``. From e8b632d8f1474e02560da34fa49e9ef71979d3d2 Mon Sep 17 00:00:00 2001 From: Jed Cunningham Date: Sat, 10 Aug 2024 00:04:03 -0600 Subject: [PATCH 2/3] Ignore operator and sensor autoapi index pages The individual operators and sensors are already enumerate elsewhere. --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 11560b0923f4..f406267ede88 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -199,8 +199,8 @@ exclude_patterns = [ # We only link to selected subpackages. "_api/airflow/index.rst", - # "_api/airflow/operators/index.rst", - # "_api/airflow/sensors/index.rst", + "_api/airflow/operators/index.rst", + "_api/airflow/sensors/index.rst", # Included in the cluster-policies doc "_api/airflow/policies/index.rst", "README.rst", From 68327c1fe1d6367e0bab4fd2544df7dd6b721748 Mon Sep 17 00:00:00 2001 From: Jed Cunningham Date: Tue, 13 Aug 2024 16:59:15 -0600 Subject: [PATCH 3/3] Fix toc warning --- airflow/operators/__init__.py | 5 +++++ airflow/sensors/__init__.py | 5 +++++ docs/conf.py | 2 -- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/airflow/operators/__init__.py b/airflow/operators/__init__.py index 77fcc8c0c0e9..7cedaa0ecfcf 100644 --- a/airflow/operators/__init__.py +++ b/airflow/operators/__init__.py @@ -16,3 +16,8 @@ # specific language governing permissions and limitations # under the License. # fmt: off +""" +Operators. + +:sphinx-autoapi-skip: +""" diff --git a/airflow/sensors/__init__.py b/airflow/sensors/__init__.py index 77fcc8c0c0e9..bc76ab08fc65 100644 --- a/airflow/sensors/__init__.py +++ b/airflow/sensors/__init__.py @@ -16,3 +16,8 @@ # specific language governing permissions and limitations # under the License. # fmt: off +""" +Sensors. + +:sphinx-autoapi-skip: +""" diff --git a/docs/conf.py b/docs/conf.py index f406267ede88..cbcdaa697015 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -199,8 +199,6 @@ exclude_patterns = [ # We only link to selected subpackages. "_api/airflow/index.rst", - "_api/airflow/operators/index.rst", - "_api/airflow/sensors/index.rst", # Included in the cluster-policies doc "_api/airflow/policies/index.rst", "README.rst",