Source Amazon Ads #8388 PR: fix SAT and bump version

airbytehq · Dec 28, 2021 · 0e1073d · 0e1073d
1 parent 039a1aa
commit 0e1073d
Show file tree

Hide file tree

Showing 13 changed files with 32 additions and 86 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -15,7 +15,7 @@
 - name: Amazon Ads
   sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
   dockerRepository: airbyte/source-amazon-ads
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads
   icon: amazonads.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -87,7 +87,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-ads:0.1.2"
+- dockerImage: "airbyte/source-amazon-ads:0.1.3"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-ads"
     connectionSpecification:
@@ -133,7 +133,7 @@
           - "2022-10-22"
           type: "string"
         region:
-          description: "Region to pull data from (EU/NA/FE/SANDBOX)"
+          description: "An enumeration."
           default: "NA"
           name: "Region"
           title: "AmazonAdsRegion"

diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-amazon-ads
diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json
@@ -40,16 +40,6 @@
       "destination_sync_mode": "overwrite",
       "source_defined_cursor": false
     },
-    {
-      "stream": {
-        "name": "sponsored_display_product_ads",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite",
-      "source_defined_cursor": false
-    },
     {
       "stream": {
         "name": "sponsored_display_targetings",
@@ -100,16 +90,6 @@
       "destination_sync_mode": "overwrite",
       "source_defined_cursor": false
     },
-    {
-      "stream": {
-        "name": "sponsored_product_ads",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite",
-      "source_defined_cursor": false
-    },
     {
       "stream": {
         "name": "sponsored_product_targetings",

diff --git a/...tegrations/connectors/source-amazon-ads/integration_tests/configured_catalog_display.json b/...tegrations/connectors/source-amazon-ads/integration_tests/configured_catalog_display.json
@@ -30,16 +30,6 @@
       "destination_sync_mode": "overwrite",
       "source_defined_cursor": false
     },
-    {
-      "stream": {
-        "name": "sponsored_display_product_ads",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite",
-      "source_defined_cursor": false
-    },
     {
       "stream": {
         "name": "sponsored_display_targetings",

diff --git a/...tions/connectors/source-amazon-ads/integration_tests/configured_catalog_full_refresh.json b/...tions/connectors/source-amazon-ads/integration_tests/configured_catalog_full_refresh.json
@@ -30,16 +30,6 @@
       "destination_sync_mode": "overwrite",
       "source_defined_cursor": false
     },
-    {
-      "stream": {
-        "name": "sponsored_display_product_ads",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite",
-      "source_defined_cursor": false
-    },
     {
       "stream": {
         "name": "sponsored_display_targetings",
@@ -90,16 +80,6 @@
       "destination_sync_mode": "overwrite",
       "source_defined_cursor": false
     },
-    {
-      "stream": {
-        "name": "sponsored_product_ads",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite",
-      "source_defined_cursor": false
-    },
     {
       "stream": {
         "name": "sponsored_product_targetings",

diff --git a/...egrations/connectors/source-amazon-ads/integration_tests/configured_catalog_products.json b/...egrations/connectors/source-amazon-ads/integration_tests/configured_catalog_products.json
@@ -50,16 +50,6 @@
       "destination_sync_mode": "overwrite",
       "source_defined_cursor": false
     },
-    {
-      "stream": {
-        "name": "sponsored_product_ads",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite",
-      "source_defined_cursor": false
-    },
     {
       "stream": {
         "name": "sponsored_product_targetings",

diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/no_profiles_catalog.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/no_profiles_catalog.json
@@ -20,16 +20,6 @@
       "destination_sync_mode": "overwrite",
       "source_defined_cursor": false
     },
-    {
-      "stream": {
-        "name": "sponsored_display_product_ads",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite",
-      "source_defined_cursor": false
-    },
     {
       "stream": {
         "name": "sponsored_display_targetings",

diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json
@@ -40,7 +40,7 @@
         "type": "string"
       },
       "region": {
-        "description": "Region to pull data from (EU/NA/FE/SANDBOX)",
+        "description": "An enumeration.",
         "default": "NA",
         "name": "Region",
         "title": "AmazonAdsRegion",

diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.12", "requests_oauthlib~=1.3.0", "pytz~=2021.1", "pendulum~=1.5.1"]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.46", "requests_oauthlib~=1.3.0", "pytz~=2021.1", "pendulum~=1.5.1"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",

diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py
@@ -14,11 +14,17 @@ class Config:
 
     client_id: str = Field(
         name="Client ID",
-        description='Oauth client id <a href="https://advertising.amazon.com/API/docs/en-us/setting-up/step-1-create-lwa-app">How to create your Login with Amazon</a>',
+        description=(
+            'Oauth client id <a href="https://advertising.amazon.com/API/docs/en-us/setting-up/step-1-create-lwa-app">'
+            "How to create your Login with Amazon</a>"
+        ),
     )
     client_secret: str = Field(
         name="Client secret",
-        description='Oauth client secret <a href="https://advertising.amazon.com/API/docs/en-us/setting-up/step-1-create-lwa-app">How to create your Login with Amazon</a>',
+        description=(
+            'Oauth client secret <a href="https://advertising.amazon.com/API/docs/en-us/setting-up/step-1-create-lwa-app">'
+            "How to create your Login with Amazon</a>"
+        ),
         airbyte_secret=True,
     )
 
@@ -30,11 +36,17 @@ class Config:
         examples=[
             "cpc_advertising:campaign_management",
         ],
-        description="By default its advertising::campaign_management, but customers may need to set scope to cpc_advertising:campaign_management.",
+        description=(
+            "By default its advertising::campaign_management,"
+            " but customers may need to set scope to cpc_advertising:campaign_management."
+        ),
     )
     refresh_token: str = Field(
         name="Oauth refresh token",
-        description='Oauth 2.0 refresh_token, <a href="https://developer.amazon.com/docs/login-with-amazon/conceptual-overview.html">read details here</a>',
+        description=(
+            'Oauth 2.0 refresh_token, <a href="https://developer.amazon.com/docs/login-with-amazon/conceptual-overview.html">'
+            "read details here</a>"
+        ),
         airbyte_secret=True,
     )
 
@@ -61,9 +73,9 @@ def schema(cls, **kvargs):
         # filter out it from the jsonschema output
         schema["properties"] = {name: desc for name, desc in schema["properties"].items() if not name.startswith("_")}
         # Transform pydantic generated enum for region
-        schema["definitions"]["AmazonAdsRegion"].pop("description")
-        schema["properties"]["region"].update(schema["definitions"]["AmazonAdsRegion"])
-        schema["properties"]["region"].pop("allOf", None)
-        schema["properties"]["region"].pop("$ref", None)
-        del schema["definitions"]
+        definitions = schema.pop("definitions", None)
+        if definitions:
+            schema["properties"]["region"].update(definitions["AmazonAdsRegion"])
+            schema["properties"]["region"].pop("allOf", None)
+            schema["properties"]["region"].pop("$ref", None)
         return schema
diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py
@@ -9,6 +9,7 @@
 import requests
 from airbyte_cdk.sources.streams.core import Stream
 from airbyte_cdk.sources.streams.http import HttpStream
+from airbyte_cdk.sources.utils.schema_helpers import expand_refs
 from pydantic import BaseModel, ValidationError
 from source_amazon_ads.constants import URL_MAPPING
 from source_amazon_ads.schemas import CatalogModel
@@ -86,7 +87,9 @@ def model(self) -> CatalogModel:
         """
 
     def get_json_schema(self):
-        return self.model.schema()
+        schema = self.model.schema()
+        expand_refs(schema)
+        return schema
 
 
 # Basic full refresh stream

diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md
@@ -76,6 +76,7 @@ Start date used for generating reports starting from the specified start date. S
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| `0.1.3` | 2021-12-28 | [\#8388](https://github.com/airbytehq/airbyte/pull/8388) | `Add retry if recoverable error  occured for reporting stream processing` |
 | `0.1.2` | 2021-10-01 | [\#6367](https://github.com/airbytehq/airbyte/pull/6461) | `Add option to pull data for different regions. Add option to choose profiles we want to pull data. Add lookback` |
 | `0.1.1` | 2021-09-22 | [\#6367](https://github.com/airbytehq/airbyte/pull/6367) | `Add seller and vendor filters to profiles stream` |
 | `0.1.0` | 2021-08-13 | [\#5023](https://github.com/airbytehq/airbyte/pull/5023) | `Initial version` |