diff --git a/docs/apache-airflow-providers-papermill/index.rst b/docs/apache-airflow-providers-papermill/index.rst index 7effd4b35ca7..54fe848e7014 100644 --- a/docs/apache-airflow-providers-papermill/index.rst +++ b/docs/apache-airflow-providers-papermill/index.rst @@ -38,7 +38,7 @@ Content :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-papermill/operators.rst b/docs/apache-airflow-providers-papermill/operators.rst index 274d83adb919..c760eecaf016 100644 --- a/docs/apache-airflow-providers-papermill/operators.rst +++ b/docs/apache-airflow-providers-papermill/operators.rst @@ -50,8 +50,15 @@ Example DAG Use the :class:`~airflow.providers.papermill.operators.papermill.PapermillOperator` to execute a jupyter notebook: -.. exampleinclude:: /../../airflow/providers/papermill/example_dags/example_papermill.py +.. exampleinclude:: /../../tests/system/providers/papermill/example_papermill.py :language: python :dedent: 4 :start-after: [START howto_operator_papermill] :end-before: [END howto_operator_papermill] + +Example DAG to Verify the message in the notebook: + +.. exampleinclude:: /../../tests/system/providers/papermill/example_papermill_verify.py + :language: python + :start-after: [START howto_verify_operator_papermill] + :end-before: [END howto_verify_operator_papermill] diff --git a/airflow/providers/papermill/example_dags/__init__.py b/tests/system/__init__.py similarity index 100% rename from airflow/providers/papermill/example_dags/__init__.py rename to tests/system/__init__.py diff --git a/tests/system/providers/__init__.py b/tests/system/providers/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/tests/system/providers/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/papermill/__init__.py b/tests/system/providers/papermill/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/tests/system/providers/papermill/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/papermill/example_papermill.py b/tests/system/providers/papermill/example_papermill.py new file mode 100644 index 000000000000..8f828e4ed11f --- /dev/null +++ b/tests/system/providers/papermill/example_papermill.py @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This DAG will use Papermill to run the notebook "hello_world", based on the execution date +it will create an output notebook "out-". All fields, including the keys in the parameters, are +templated. +""" +import os +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.providers.papermill.operators.papermill import PapermillOperator + +START_DATE = datetime(2021, 1, 1) +SCHEDULE_INTERVAL = '0 0 * * *' +DAGRUN_TIMEOUT = timedelta(minutes=60) +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_papermill_operator" + +with DAG( + dag_id=DAG_ID, + schedule_interval=SCHEDULE_INTERVAL, + start_date=START_DATE, + dagrun_timeout=DAGRUN_TIMEOUT, + tags=['example'], + catchup=False, +) as dag: + # [START howto_operator_papermill] + run_this = PapermillOperator( + task_id="run_example_notebook", + input_nb="/tmp/hello_world.ipynb", + output_nb="/tmp/out-{{ execution_date }}.ipynb", + parameters={"msgs": "Ran from Airflow at {{ execution_date }}!"}, + ) + # [END howto_operator_papermill] + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/papermill/example_dags/example_papermill.py b/tests/system/providers/papermill/example_papermill_verify.py similarity index 79% rename from airflow/providers/papermill/example_dags/example_papermill.py rename to tests/system/providers/papermill/example_papermill_verify.py index c49b7715794a..dd24fc51f636 100644 --- a/airflow/providers/papermill/example_dags/example_papermill.py +++ b/tests/system/providers/papermill/example_papermill_verify.py @@ -33,25 +33,11 @@ START_DATE = datetime(2021, 1, 1) SCHEDULE_INTERVAL = '0 0 * * *' DAGRUN_TIMEOUT = timedelta(minutes=60) - -with DAG( - dag_id='example_papermill_operator', - schedule_interval=SCHEDULE_INTERVAL, - start_date=START_DATE, - dagrun_timeout=DAGRUN_TIMEOUT, - tags=['example'], - catchup=False, -) as dag_1: - # [START howto_operator_papermill] - run_this = PapermillOperator( - task_id="run_example_notebook", - input_nb="/tmp/hello_world.ipynb", - output_nb="/tmp/out-{{ execution_date }}.ipynb", - parameters={"msgs": "Ran from Airflow at {{ execution_date }}!"}, - ) - # [END howto_operator_papermill] +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_papermill_operator_verify" +# [START howto_verify_operator_papermill] @task def check_notebook(inlets, execution_date): """ @@ -68,12 +54,12 @@ def check_notebook(inlets, execution_date): with DAG( - dag_id='example_papermill_operator_2', + dag_id='example_papermill_operator_verify', schedule_interval=SCHEDULE_INTERVAL, start_date=START_DATE, dagrun_timeout=DAGRUN_TIMEOUT, catchup=False, -) as dag_2: +) as dag: run_this = PapermillOperator( task_id="run_example_notebook", @@ -83,3 +69,9 @@ def check_notebook(inlets, execution_date): ) run_this >> check_notebook(inlets=AUTO, execution_date="{{ execution_date }}") +# [END howto_verify_operator_papermill] + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/papermill/example_dags/input_notebook.ipynb b/tests/system/providers/papermill/input_notebook.ipynb similarity index 100% rename from airflow/providers/papermill/example_dags/input_notebook.ipynb rename to tests/system/providers/papermill/input_notebook.ipynb diff --git a/tests/www/api/experimental/test_endpoints.py b/tests/www/api/experimental/test_endpoints.py index 9710b38c6895..8d5516a409f6 100644 --- a/tests/www/api/experimental/test_endpoints.py +++ b/tests/www/api/experimental/test_endpoints.py @@ -312,7 +312,7 @@ def test_dagrun_status(self): class TestLineageApiExperimental(TestBase): - PAPERMILL_EXAMPLE_DAGS = os.path.join(ROOT_FOLDER, "airflow", "providers", "papermill", "example_dags") + PAPERMILL_EXAMPLE_DAGS = os.path.join(ROOT_FOLDER, "tests", "system", "providers", "papermill") @pytest.fixture(scope="class", autouse=True) def _populate_db(self):