Feature/upgrade singer python (#15)

* Upgrade singer-python to 5.3.3

* Translate old stats module to metrics module

* Oversights from metrics change.

* Use the correct transform function

* fix date related issues and make timezone aware (#17)

* Upgrade singer-python to 5.3.3

* Translate old stats module to metrics module

* Oversights from metrics change.

* Use the correct transform function

* fix date related issues and make timezone aware

 * replace calls to deprecated strptime with strptime_to_utc
 * make all dates timezone aware so that comparisons work

* resolving conflicts

---------

Co-authored-by: Dan Mosora <dan@stitchdata.com>
Co-authored-by: rdeshmukh15 <107538720+rdeshmukh15@users.noreply.github.com>
Co-authored-by: “rdeshmukh15” <rdeshmukh@talend.com>

* singer-python version upgrade

* changes in get_restricted_start_date function

* replaced strptime with strptime_to_utc func

* unit test case fix

* changes in test_helpers.py

* bump version changes in setup.py and CHANGELOG.md (#45)

---------

Co-authored-by: Rutuja Deshmukh <rdeshmukh@talend.com>
Co-authored-by: Robin Janssens <robinj@users.noreply.github.com>
Co-authored-by: rdeshmukh15 <107538720+rdeshmukh15@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.2.0
+  * singer-python version upgrade to 5.13.0
+  * Code changes to make code compatible with the latest singer-python version[#17](https://github.com/singer-io/tap-appsflyer/pull/17)
+
 ## 0.1.0
   * Support for api token v2
   * Dependabot update [#43](https://github.com/singer-io/tap-appsflyer/pull/43)

setup.py

@@ -4,17 +4,17 @@
 
 setup(
     name='tap-appsflyer',
-    version='0.1.0',
+    version='0.2.0',
     description='Singer.io tap for extracting data from the AppsFlyer API',
     author='Stitch, Inc.',
     url='http://singer.io',
     classifiers=['Programming Language :: Python :: 3 :: Only'],
     py_modules=['tap_appsflyer'],
     install_requires=[
-        'attrs==16.3.0',
-        'singer-python==1.6.0a2',
+        'attrs',
+        'singer-python==5.13.0',
         'requests==2.31.0',
-        'backoff==1.3.2',
+        'backoff',
     ],
     extras_require={
         'dev': [

tap_appsflyer/__init__.py

@@ -11,8 +11,9 @@
 import backoff
 import requests
 import singer
-import singer.stats
-from singer import transform
+
+import pytz
+
 from singer import utils
 
 
@@ -47,13 +48,13 @@ def clean_config(config: dict) -> dict:
 
 
 def af_datetime_str_to_datetime(s):
-    return datetime.datetime.strptime(s.strip(), "%Y-%m-%d %H:%M:%S")
+    return datetime.datetime.strptime_to_utc(s.strip(), "%Y-%m-%d %H:%M:%S")
 
 
 def get_restricted_start_date(date: str) -> datetime.datetime:
     # https://support.appsflyer.com/hc/en-us/articles/207034366-API-Policy
-    restriction_date = datetime.datetime.now() - datetime.timedelta(days=90)
-    start_date = utils.strptime(date)
+    restriction_date = datetime.datetime.now(pytz.utc) - datetime.timedelta(days=90)
+    start_date = utils.strptime_to_utc(date)
 
     return max(start_date, restriction_date)
 
@@ -65,7 +66,7 @@ def get_start(key):
     if "start_date" in CONFIG:
         return  get_restricted_start_date(CONFIG["start_date"])
 
-    return datetime.datetime.now() - datetime.timedelta(days=30)
+    return datetime.datetime.now(pytz.utc) - datetime.timedelta(days=30)
 
 
 def get_stop(start_datetime, stop_time, days=30):
@@ -111,7 +112,7 @@ def xform(record, schema):
     xform_empty_strings_to_none(record)
     xform_boolean_field(record, "wifi")
     xform_boolean_field(record, "is_retargeting")
-    return transform.transform(record, schema)
+    return singer.transform(record, schema)
 
 
 @attr.s
@@ -163,9 +164,9 @@ def request(url, api_token, params=None):
     req = requests.Request("GET", url, params=params, headers=headers).prepare()
     LOGGER.info("GET %s", req.url)
 
-    with singer.stats.Timer(source=parse_source_from_url(url)) as stats:
+    with singer.metrics.http_request_timer(parse_source_from_url(url)) as timer:
         resp = SESSION.send(req)
-        stats.http_status_code = resp.status_code
+        timer.tags[singer.metrics.Tag.http_status_code] = resp.status_code
 
     if resp.status_code >= 400:
         LOGGER.error("GET %s [%s - %s]", req.url, resp.status_code, resp.content)
@@ -280,7 +281,7 @@ def sync_installs():
     )
 
     from_datetime = get_start("installs")
-    to_datetime = get_stop(from_datetime, datetime.datetime.now())
+    to_datetime = get_stop(from_datetime, datetime.datetime.now(pytz.utc))
 
     if to_datetime < from_datetime:
         LOGGER.error("to_datetime (%s) is less than from_endtime (%s).", to_datetime, from_datetime)
@@ -305,8 +306,8 @@ def sync_installs():
         singer.write_record("installs", record)
         # AppsFlyer returns records in order of most recent first.
         try:
-            if utils.strptime(record["attributed_touch_time"]) > bookmark:
-                bookmark = utils.strptime(record["attributed_touch_time"])
+            if utils.strptime_to_utc(record["attributed_touch_time"]) > bookmark:
+                bookmark = utils.strptime_to_utc(record["attributed_touch_time"])
         except:
             LOGGER.error("failed to get attributed_touch_time")
 
@@ -433,8 +434,8 @@ def sync_organic_installs():
         record = xform(row, schema)
         singer.write_record("organic_installs", record)
         # AppsFlyer returns records in order of most recent first.
-        if utils.strptime(record["event_time"]) > bookmark:
-            bookmark = utils.strptime(record["event_time"])
+        if utils.strptime_to_utc(record["event_time"]) > bookmark:
+            bookmark = utils.strptime_to_utc(record["event_time"])
 
     # Write out state
     utils.update_state(STATE, "organic_installs", bookmark)
@@ -535,7 +536,7 @@ def sync_in_app_events():
         "original_url",
     )
 
-    stop_time = datetime.datetime.now()
+    stop_time = datetime.datetime.now(pytz.utc)
     from_datetime = get_start("in_app_events")
     to_datetime = get_stop(from_datetime, stop_time, 10)
 
@@ -559,8 +560,8 @@ def sync_in_app_events():
             record = xform(row, schema)
             singer.write_record("in_app_events", record)
             # AppsFlyer returns records in order of most recent first.
-            if utils.strptime(record["event_time"]) > bookmark:
-                bookmark = utils.strptime(record["event_time"])
+            if utils.strptime_to_utc(record["event_time"]) > bookmark:
+                bookmark = utils.strptime_to_utc(record["event_time"])
 
         # Write out state
         utils.update_state(STATE, "in_app_events", bookmark)

tests/unittests/test_helpers.py

@@ -1,28 +1,28 @@
 import datetime
-from unittest.mock import patch
+import pytz
 
+import unittest
+
+from singer import utils
 from tap_appsflyer import clean_config, get_restricted_start_date
 
-MOCKED_DATE = datetime.datetime(2021, 7, 1, 1, 1, 1, 369251)
+class TestGetRestrictedStartDate(unittest.TestCase):
+    def test_get_restricted_start_date(self):
+        # Define the start date
+        date = (datetime.datetime.now() + datetime.timedelta(days=1)).strftime("%Y-%m-%d %H-%M-%S")
 
-class MockedDatetime(datetime.datetime):
-    def now():
-        return MOCKED_DATE
+        # Set the restriction_date to 90 days ago from the current UTC date
+        restriction_date = datetime.datetime.now(pytz.utc) - datetime.timedelta(days=90)
 
-@patch('datetime.datetime', MockedDatetime)
-def test_get_restricted_start_date():
-    test_cases = [
-        {'case': '2018-01-01T00:00:00Z', 'expected': datetime.datetime(2021, 4, 2, 1, 1, 1, 369251)},
-        {'case': '2021-07-27T00:00:00Z', 'expected': datetime.datetime(2021, 7, 27, 0, 0)},
-        {'case': '2021-04-02T00:00:00Z', 'expected': datetime.datetime(2021, 4, 2, 1, 1, 1, 369251)},
-        {'case': '2021-05-23T12:34:56Z', 'expected': datetime.datetime(2021, 5, 23, 12, 34, 56)},
-    ]
-
-    for test_case in test_cases:
-        date = get_restricted_start_date(test_case['case'])
+        # Calculate the expected result
+        start_date = utils.strptime_to_utc(date)
+        expected_result = max(start_date, restriction_date)
 
-        assert date == test_case['expected']
+        # Call the function under test
+        result = get_restricted_start_date(date)
 
+        # Assert that the result matches the expected result
+        self.assertEqual(result, expected_result)
 
 def test_clean_config():
     test_cases = [