🎉 Source Hubspot: Migrate Hubspot source to CDK structure (#7562)

* Migrate Hubspot source to CDK structure

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c",
   "name": "Hubspot",
   "dockerRepository": "airbyte/source-hubspot",
-  "dockerImageTag": "0.1.21",
+  "dockerImageTag": "0.1.22",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot",
   "icon": "hubspot.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -236,7 +236,7 @@
 - name: Hubspot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.21
+  dockerImageTag: 0.1.22
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

airbyte-integrations/connectors/source-hubspot/.dockerignore

@@ -1,8 +1,6 @@
 *
 !Dockerfile
-!Dockerfile.test
+!main.py
 !source_hubspot
 !setup.py
 !secrets
-!acceptance-test-config.yml
-!acceptance-test.sh

airbyte-integrations/connectors/source-hubspot/Dockerfile

@@ -1,18 +1,38 @@
-FROM airbyte/integration-base-python:0.1.1
+FROM python:3.7.11-alpine3.14 as base
 
-# Bash is installed for more convenient debugging.
-RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
+# build and load all requirements
+FROM base as builder
+WORKDIR /airbyte/integration_code
 
-ENV CODE_PATH="source_hubspot"
-ENV AIRBYTE_IMPL_MODULE="source_hubspot"
-ENV AIRBYTE_IMPL_PATH="SourceHubspot"
+# upgrade pip to the latest version
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
 
-WORKDIR /airbyte/integration_code
-COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
-RUN pip install .
 
-ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
+# install necessary packages to a temporary folder
+RUN pip install --prefix=/install .
+
+# build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code
+
+# copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# copy payload code only
+COPY main.py ./
+COPY source_hubspot ./source_hubspot
+
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.21
+LABEL io.airbyte.version=0.1.22
 LABEL io.airbyte.name=airbyte/source-hubspot

airbyte-integrations/connectors/source-hubspot/build.gradle

@@ -7,8 +7,3 @@ plugins {
 airbytePython {
     moduleDirectory 'source_hubspot'
 }
-
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs)
-    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
-}

airbyte-integrations/connectors/source-hubspot/main_dev.py → airbyte-integrations/connectors/source-hubspot/main.py

@@ -5,7 +5,7 @@
 
 import sys
 
-from base_python.entrypoint import launch
+from airbyte_cdk.entrypoint import launch
 from source_hubspot import SourceHubspot
 
 if __name__ == "__main__":

airbyte-integrations/connectors/source-hubspot/requirements.txt

@@ -1,5 +1,3 @@
 # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
--e ../../bases/airbyte-protocol
--e ../../bases/base-python
 -e ../../bases/source-acceptance-test
 -e .

airbyte-integrations/connectors/source-hubspot/setup.py

@@ -7,8 +7,6 @@
 
 MAIN_REQUIREMENTS = [
     "airbyte-cdk~=0.1",
-    "airbyte-protocol",
-    "base-python",
     "backoff==1.11.1",
     "pendulum==2.1.2",
     "requests==2.26.0",

airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py

@@ -14,8 +14,8 @@
 import backoff
 import pendulum as pendulum
 import requests
+from airbyte_cdk.entrypoint import logger
 from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator
-from base_python.entrypoint import logger
 from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout
 
 # The value is obtained experimentally, Hubspot allows the URL length up to ~16300 symbols,
@@ -374,7 +374,7 @@ def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iter
                     'message': 'This hapikey (....) does not have proper permissions! (requires any of [automation-access])',
                     'correlationId': '111111-2222-3333-4444-55555555555'}
                 """
-                logger.warn(f"Stream `{self.entity}` cannot be procced. {response.get('message')}")
+                logger.warning(f"Stream `{self.entity}` cannot be procced. {response.get('message')}")
                 return
 
             if response.get(self.data_field) is None:

airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py

@@ -5,8 +5,8 @@
 
 from typing import Any, Callable, Iterator, Mapping, Optional, Tuple
 
-from airbyte_protocol import AirbyteStream
-from base_python import BaseClient
+from airbyte_cdk.models import AirbyteStream
+from airbyte_cdk.sources.deprecated.client import BaseClient
 from requests import HTTPError
 from source_hubspot.api import (
     API,

airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py

@@ -3,7 +3,7 @@
 #
 
 
-from base_python import BaseSource
+from airbyte_cdk.sources.deprecated.base_source import BaseSource
 
 from .client import Client
 

airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py

@@ -36,14 +36,15 @@ def test_field_type_format_converting(field_type, expected):
         (1, {"type": ["null", "string"]}),
     ],
 )
-def test_bad_field_type_converting(field_type, expected, capsys):
+def test_bad_field_type_converting(field_type, expected, caplog, capsys):
 
     assert Stream._get_field_props(field_type=field_type) == expected
 
-    logs = capsys.readouterr().out
+    logs = caplog.records
 
-    assert '"WARN"' in logs
-    assert f"Unsupported type {field_type} found" in logs
+    assert logs
+    assert logs[0].levelname == "WARNING"
+    assert logs[0].msg == f"Unsupported type {field_type} found"
 
 
 @pytest.mark.parametrize(

docs/integrations/sources/hubspot.md

@@ -96,6 +96,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.22 | 2021-11-03 | [7562](https://github.com/airbytehq/airbyte/pull/7562) | Migrate Hubspot source to CDK structure |
 | 0.1.21 | 2021-10-27 | [7405](https://github.com/airbytehq/airbyte/pull/7405) | Change of package `import` from `urllib` to `urllib.parse` |
 | 0.1.20 | 2021-10-26 | [7393](https://github.com/airbytehq/airbyte/pull/7393) | Hotfix for `split_properties` function, add the length of separator symbol `,`(`%2C` in HTTP format) to the checking of the summary URL length |
 | 0.1.19 | 2021-10-26 | [6954](https://github.com/airbytehq/airbyte/pull/6954) | Fix issue with getting `414` HTTP error for streams |