Add (optional) dependency between dbt-cloud and openlineage providers (…

…#39366)

* Add (optional) minimum dependency between dbt-cloud and OpenLineage provider

Since the change #38033 was merged, `airflow-providers-dbt-cloud>=1.7.0` depend on `airflow-providers-openlineage>=1.7.0`. However, since this dependency was not declared anywhere.

This is the error users face if they use `airflow-providers-dbt-cloud>=1.7.0` and `airflow-providers-openlineage<1.7.0`:
```
2024-05-01, 10:17:39 UTC] {base.py:147} ERROR - OpenLineage provider method failed to import OpenLineage integration. This should not happen.
Traceback (most recent call last):
  File /usr/local/lib/python3.9/site-packages/airflow/providers/openlineage/extractors/base.py, line 137, in _get_openlineage_facets
    facets: OperatorLineage = get_facets_method(*args)
  File /usr/local/lib/python3.9/site-packages/airflow/providers/dbt/cloud/operators/dbt.py, line 249, in get_openlineage_facets_on_complete
    return generate_openlineage_events_from_dbt_cloud_run(operator=self, task_instance=task_instance)
  File /usr/local/lib/python3.9/site-packages/airflow/providers/dbt/cloud/utils/openlineage.py, line 50, in generate_openlineage_events_from_dbt_cloud_run
    from airflow.providers.openlineage.conf import namespace
ModuleNotFoundError: No module named 'airflow.providers.openlineage.conf'
```

Given that the dependency between both is optional, this PR introduces additional-extras to the dbt provider, solving the dependency issue for users who install using .

* Refactor dbt-cloud provider to raise a user-friendly exception if using incompatible openlineage provider

airflow/providers/dbt/cloud/provider.yaml

@@ -57,6 +57,13 @@ dependencies:
   - asgiref
   - aiohttp>=3.9.2
 
+additional-extras:
+  # pip install apache-airflow-providers-dbt-cloud[openlineage]
+  - name: openlineage
+    description: Install compatible OpenLineage dependencies
+    dependencies:
+      - apache-airflow-providers-openlineage>=1.7.0
+
 integrations:
   - integration-name: dbt Cloud
     external-doc-url: https://docs.getdbt.com/docs/dbt-cloud/cloud-overview

airflow/providers/dbt/cloud/utils/openlineage.py

@@ -47,7 +47,14 @@ def generate_openlineage_events_from_dbt_cloud_run(
     """
     from openlineage.common.provider.dbt import DbtCloudArtifactProcessor, ParentRunMetadata
 
-    from airflow.providers.openlineage.conf import namespace
+    try:
+        from airflow.providers.openlineage.conf import namespace
+    except ModuleNotFoundError as e:
+        from airflow.exceptions import AirflowOptionalProviderFeatureException
+
+        msg = "Please install `apache-airflow-providers-openlineage>=1.7.0`"
+        raise AirflowOptionalProviderFeatureException(e, msg)
+
     from airflow.providers.openlineage.extractors import OperatorLineage
     from airflow.providers.openlineage.plugins.adapter import (
         _PRODUCER,

tests/providers/dbt/cloud/utils/test_openlineage.py

@@ -19,6 +19,9 @@
 import json
 from unittest.mock import MagicMock, patch
 
+import pytest
+
+from airflow.exceptions import AirflowOptionalProviderFeatureException
 from airflow.providers.dbt.cloud.hooks.dbt import DbtCloudHook
 from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator
 from airflow.providers.dbt.cloud.utils.openlineage import generate_openlineage_events_from_dbt_cloud_run
@@ -77,6 +80,26 @@ def get_dbt_artifact(*args, **kwargs):
     return None
 
 
+def test_previous_version_openlineage_provider():
+    """When using OpenLineage, the dbt-cloud provider now depends on openlineage provider >= 1.7"""
+    original_import = __import__
+
+    def custom_import(name, *args, **kwargs):
+        if name == "airflow.providers.openlineage.conf":
+            raise ModuleNotFoundError("No module named 'airflow.providers.openlineage.conf")
+        else:
+            return original_import(name, *args, **kwargs)
+
+    mock_operator = MagicMock()
+    mock_task_instance = MagicMock()
+
+    with patch("builtins.__import__", side_effect=custom_import):
+        with pytest.raises(AirflowOptionalProviderFeatureException) as exc:
+            generate_openlineage_events_from_dbt_cloud_run(mock_operator, mock_task_instance)
+    assert str(exc.value.args[0]) == "No module named 'airflow.providers.openlineage.conf"
+    assert str(exc.value.args[1]) == "Please install `apache-airflow-providers-openlineage>=1.7.0`"
+
+
 class TestGenerateOpenLineageEventsFromDbtCloudRun:
     @patch("airflow.providers.openlineage.plugins.listener.get_openlineage_listener")
     @patch("airflow.providers.openlineage.plugins.adapter.OpenLineageAdapter.build_task_instance_run_id")