From 90451eb0752fa167540ad17b5d083e872bef826c Mon Sep 17 00:00:00 2001 From: Yevhenii <34103125+yevhenii-ldv@users.noreply.github.com> Date: Wed, 3 Nov 2021 09:57:56 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Hubspot:=20Migrate=20Hu?= =?UTF-8?q?bspot=20source=20to=20CDK=20structure=20(#7562)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Migrate Hubspot source to CDK structure --- .../36c891d9-4bd9-43ac-bad2-10e12756272c.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-hubspot/.dockerignore | 4 +- .../connectors/source-hubspot/.gitignore | 1 - .../connectors/source-hubspot/Dockerfile | 42 ++++++++++++++----- .../connectors/source-hubspot/build.gradle | 5 --- .../source-hubspot/{main_dev.py => main.py} | 2 +- .../source-hubspot/requirements.txt | 2 - .../connectors/source-hubspot/setup.py | 2 - .../source-hubspot/source_hubspot/api.py | 4 +- .../source-hubspot/source_hubspot/client.py | 4 +- .../source-hubspot/source_hubspot/source.py | 2 +- .../unit_tests/test_field_type_converting.py | 9 ++-- docs/integrations/sources/hubspot.md | 1 + 14 files changed, 46 insertions(+), 36 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-hubspot/.gitignore rename airbyte-integrations/connectors/source-hubspot/{main_dev.py => main.py} (83%) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json index 67f13cd7fff5..cda2735c33c8 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", "name": "Hubspot", "dockerRepository": "airbyte/source-hubspot", - "dockerImageTag": "0.1.21", + "dockerImageTag": "0.1.22", "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", "icon": "hubspot.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 3a715c5dcd57..27c53deb2b14 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -236,7 +236,7 @@ - name: Hubspot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.21 + dockerImageTag: 0.1.22 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-hubspot/.dockerignore b/airbyte-integrations/connectors/source-hubspot/.dockerignore index 461b1bb7ee9e..85586eba85c5 100644 --- a/airbyte-integrations/connectors/source-hubspot/.dockerignore +++ b/airbyte-integrations/connectors/source-hubspot/.dockerignore @@ -1,8 +1,6 @@ * !Dockerfile -!Dockerfile.test +!main.py !source_hubspot !setup.py !secrets -!acceptance-test-config.yml -!acceptance-test.sh diff --git a/airbyte-integrations/connectors/source-hubspot/.gitignore b/airbyte-integrations/connectors/source-hubspot/.gitignore deleted file mode 100644 index 29fffc6a50cc..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/.gitignore +++ /dev/null @@ -1 +0,0 @@ -NEW_SOURCE_CHECKLIST.md diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index e2313033cfb0..40fcc091a121 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -1,18 +1,38 @@ -FROM airbyte/integration-base-python:0.1.1 +FROM python:3.7.11-alpine3.14 as base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code -ENV CODE_PATH="source_hubspot" -ENV AIRBYTE_IMPL_MODULE="source_hubspot" -ENV AIRBYTE_IMPL_PATH="SourceHubspot" +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install . -ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_hubspot ./source_hubspot + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.21 +LABEL io.airbyte.version=0.1.22 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/build.gradle b/airbyte-integrations/connectors/source-hubspot/build.gradle index bdea51cec8a8..259bffb74a61 100644 --- a/airbyte-integrations/connectors/source-hubspot/build.gradle +++ b/airbyte-integrations/connectors/source-hubspot/build.gradle @@ -7,8 +7,3 @@ plugins { airbytePython { moduleDirectory 'source_hubspot' } - -dependencies { - implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) - implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-hubspot/main_dev.py b/airbyte-integrations/connectors/source-hubspot/main.py similarity index 83% rename from airbyte-integrations/connectors/source-hubspot/main_dev.py rename to airbyte-integrations/connectors/source-hubspot/main.py index 6649d86ac662..2d902157fd45 100644 --- a/airbyte-integrations/connectors/source-hubspot/main_dev.py +++ b/airbyte-integrations/connectors/source-hubspot/main.py @@ -5,7 +5,7 @@ import sys -from base_python.entrypoint import launch +from airbyte_cdk.entrypoint import launch from source_hubspot import SourceHubspot if __name__ == "__main__": diff --git a/airbyte-integrations/connectors/source-hubspot/requirements.txt b/airbyte-integrations/connectors/source-hubspot/requirements.txt index e74f41a28ce1..7be17a56d745 100644 --- a/airbyte-integrations/connectors/source-hubspot/requirements.txt +++ b/airbyte-integrations/connectors/source-hubspot/requirements.txt @@ -1,5 +1,3 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/airbyte-protocol --e ../../bases/base-python -e ../../bases/source-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index d15d84cb5506..7c4c01fa6b73 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -7,8 +7,6 @@ MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.1", - "airbyte-protocol", - "base-python", "backoff==1.11.1", "pendulum==2.1.2", "requests==2.26.0", diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index 4ee4657241be..48ec93e7b781 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -14,8 +14,8 @@ import backoff import pendulum as pendulum import requests +from airbyte_cdk.entrypoint import logger from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator -from base_python.entrypoint import logger from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout # The value is obtained experimentally, Hubspot allows the URL length up to ~16300 symbols, @@ -374,7 +374,7 @@ def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iter 'message': 'This hapikey (....) does not have proper permissions! (requires any of [automation-access])', 'correlationId': '111111-2222-3333-4444-55555555555'} """ - logger.warn(f"Stream `{self.entity}` cannot be procced. {response.get('message')}") + logger.warning(f"Stream `{self.entity}` cannot be procced. {response.get('message')}") return if response.get(self.data_field) is None: diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py index 6f6a2cf865ff..6dd6ffb0c1dc 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py @@ -5,8 +5,8 @@ from typing import Any, Callable, Iterator, Mapping, Optional, Tuple -from airbyte_protocol import AirbyteStream -from base_python import BaseClient +from airbyte_cdk.models import AirbyteStream +from airbyte_cdk.sources.deprecated.client import BaseClient from requests import HTTPError from source_hubspot.api import ( API, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 660980307cfb..deed1d336c23 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -3,7 +3,7 @@ # -from base_python import BaseSource +from airbyte_cdk.sources.deprecated.base_source import BaseSource from .client import Client diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py index 113bc557df64..f55391f49fec 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py @@ -36,14 +36,15 @@ def test_field_type_format_converting(field_type, expected): (1, {"type": ["null", "string"]}), ], ) -def test_bad_field_type_converting(field_type, expected, capsys): +def test_bad_field_type_converting(field_type, expected, caplog, capsys): assert Stream._get_field_props(field_type=field_type) == expected - logs = capsys.readouterr().out + logs = caplog.records - assert '"WARN"' in logs - assert f"Unsupported type {field_type} found" in logs + assert logs + assert logs[0].levelname == "WARNING" + assert logs[0].msg == f"Unsupported type {field_type} found" @pytest.mark.parametrize( diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 2836f6b08ed0..35b64b637c8b 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -96,6 +96,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.22 | 2021-11-03 | [7562](https://github.com/airbytehq/airbyte/pull/7562) | Migrate Hubspot source to CDK structure | | 0.1.21 | 2021-10-27 | [7405](https://github.com/airbytehq/airbyte/pull/7405) | Change of package `import` from `urllib` to `urllib.parse` | | 0.1.20 | 2021-10-26 | [7393](https://github.com/airbytehq/airbyte/pull/7393) | Hotfix for `split_properties` function, add the length of separator symbol `,`(`%2C` in HTTP format) to the checking of the summary URL length | | 0.1.19 | 2021-10-26 | [6954](https://github.com/airbytehq/airbyte/pull/6954) | Fix issue with getting `414` HTTP error for streams |