AIP-47 - Migrate apache pig DAGs to new design #22439 (#24212)

apache · Jun 5, 2022 · 0046f12 · 0046f12
1 parent a2bfc0e
commit 0046f12
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 30 deletions.
diff --git a/airflow/providers/apache/pig/example_dags/__init__.py b/airflow/providers/apache/pig/example_dags/__init__.py
diff --git a/docs/apache-airflow-providers-apache-pig/index.rst b/docs/apache-airflow-providers-apache-pig/index.rst
@@ -37,7 +37,7 @@ Content
     :maxdepth: 1
     :caption: Resources
 
-    Example DAGs <https://github.com/apache/airflow/tree/main/airflow/providers/apache/pig/example_dags>
+    Example DAGs <https://github.com/apache/airflow/tree/main/tests/system/providers/apache/pig>
     PyPI Repository <https://pypi.org/project/apache-airflow-providers-apache-pig/>
     Installing from sources <installing-providers-from-sources>
 

diff --git a/docs/apache-airflow-providers-apache-pig/operators.rst b/docs/apache-airflow-providers-apache-pig/operators.rst
@@ -26,7 +26,7 @@ Pig programs are amenable to substantial parallelization, which in turns enables
 
 use the PigOperator to execute a pig script
 
-.. exampleinclude:: /../../airflow/providers/apache/pig/example_dags/example_pig.py
+.. exampleinclude:: /../../tests/system/providers/apache/pig/example_pig.py
     :language: python
     :start-after: [START create_pig]
     :end-before: [END create_pig]
diff --git a/...rs/apache/pig/example_dags/example_pig.py → ...ystem/providers/apache/pig/example_pig.py b/...rs/apache/pig/example_dags/example_pig.py → ...ystem/providers/apache/pig/example_pig.py
@@ -17,24 +17,34 @@
 # under the License.
 
 """Example DAG demonstrating the usage of the PigOperator."""
+
+import os
 from datetime import datetime
 
 from airflow import DAG
 from airflow.providers.apache.pig.operators.pig import PigOperator
 
-dag = DAG(
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_adf_run_pipeline"
+
+with DAG(
     dag_id='example_pig_operator',
     schedule_interval=None,
     start_date=datetime(2021, 1, 1),
     catchup=False,
     tags=['example'],
-)
-
-# [START create_pig]
-run_this = PigOperator(
-    task_id="run_example_pig_script",
-    pig="ls /;",
-    pig_opts="-x local",
-    dag=dag,
-)
-# [END create_pig]
+) as dag:
+
+    # [START create_pig]
+    run_this = PigOperator(
+        task_id="run_example_pig_script",
+        pig="ls /;",
+        pig_opts="-x local",
+    )
+    # [END create_pig]
+
+
+from tests.system.utils import get_test_run  # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)