Merge branch 'develop' into release/4.1.7

openbb_platform/core/openbb_core/provider/utils/helpers.py

@@ -316,7 +316,9 @@ def _filter(d: Data) -> bool:
     return list(filter(_filter, data))
 
 
-def safe_fromtimestamp(timestamp: float, tz: Optional[timezone] = None) -> datetime:
+def safe_fromtimestamp(
+    timestamp: Union[float, int], tz: Optional[timezone] = None
+) -> datetime:
     """datetime.fromtimestamp alternative which supports negative timestamps on Windows platform."""
     if os.name == "nt" and timestamp < 0:
         return datetime(1970, 1, 1, tzinfo=tz) + timedelta(seconds=timestamp)

openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py

@@ -2,7 +2,10 @@
 
 # pylint: disable=unused-argument
 
-from datetime import datetime
+from datetime import (
+    date as dateType,
+    timezone,
+)
 from typing import Any, Dict, List, Optional
 
 from openbb_core.provider.abstract.fetcher import Fetcher
@@ -11,9 +14,12 @@
     AnalystSearchQueryParams,
 )
 from openbb_core.provider.utils.errors import EmptyDataError
-from openbb_core.provider.utils.helpers import amake_request, get_querystring
+from openbb_core.provider.utils.helpers import (
+    amake_request,
+    get_querystring,
+    safe_fromtimestamp,
+)
 from pydantic import Field, field_validator, model_validator
-from pytz import UTC
 
 
 class BenzingaAnalystSearchQueryParams(AnalystSearchQueryParams):
@@ -358,12 +364,12 @@ class BenzingaAnalystSearchData(AnalystSearchData):
 
     @field_validator("last_updated", mode="before", check_fields=False)
     @classmethod
-    def validate_date(cls, v):
-        """Validate date."""
+    def validate_date(cls, v: float) -> Optional[dateType]:
+        """Validate last_updated."""
         if v:
-            dt = datetime.fromtimestamp(v, UTC)
+            dt = safe_fromtimestamp(v, tz=timezone.utc)
             return dt.date() if dt.time() == dt.min.time() else dt
-        return v
+        return None
 
     @model_validator(mode="before")
     @classmethod

openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py

@@ -17,9 +17,12 @@
 )
 from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS
 from openbb_core.provider.utils.errors import EmptyDataError
-from openbb_core.provider.utils.helpers import amake_requests, get_querystring
+from openbb_core.provider.utils.helpers import (
+    amake_requests,
+    get_querystring,
+    safe_fromtimestamp,
+)
 from pydantic import Field, field_validator, model_validator
-from pytz import UTC
 
 COVERAGE_DICT = {
     "downgrades": "Downgrades",
@@ -221,12 +224,12 @@ def parse_date(cls, v: str):
 
     @field_validator("last_updated", mode="before", check_fields=False)
     @classmethod
-    def validate_date(cls, v):
+    def validate_date(cls, v: float) -> Optional[dateType]:
         """Convert the Unix timestamp to a datetime object."""
         if v:
-            dt = datetime.fromtimestamp(v, UTC)
+            dt = safe_fromtimestamp(v, tz=timezone.utc)
             return dt.date() if dt.time() == dt.min.time() else dt
-        return v
+        return None
 
     @model_validator(mode="before")
     @classmethod

openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py

@@ -2,7 +2,10 @@
 
 # pylint: disable=unused-argument
 
-from datetime import datetime
+from datetime import (
+    datetime,
+    timezone,
+)
 from typing import Any, Dict, List, Optional
 
 from openbb_core.provider.abstract.fetcher import Fetcher
@@ -11,7 +14,7 @@
     CurrencySnapshotsQueryParams,
 )
 from openbb_core.provider.utils.errors import EmptyDataError
-from openbb_core.provider.utils.helpers import amake_request
+from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp
 from pandas import DataFrame, concat
 from pydantic import Field, field_validator
 
@@ -85,7 +88,6 @@ async def aextract_data(
         **kwargs: Any,
     ) -> List[Dict]:
         """Extract the data from the FMP endpoint."""
-
         api_key = credentials.get("fmp_api_key") if credentials else ""
 
         url = f"https://financialmodelingprep.com/api/v3/quotes/forex?apikey={api_key}"
@@ -99,7 +101,6 @@ def transform_data(
         **kwargs: Any,
     ) -> List[FMPCurrencySnapshotsData]:
         """Filter by the query parameters and validate the model."""
-
         if not data:
             raise EmptyDataError("No data was returned from the FMP endpoint.")
 
@@ -143,7 +144,7 @@ def transform_data(
             if len(temp) > 0:
                 # Convert the Unix timestamp to a datetime.
                 temp.timestamp = temp.timestamp.apply(
-                    lambda x: datetime.fromtimestamp(x)
+                    lambda x: safe_fromtimestamp(x, tz=timezone.utc)
                 )
                 new_df = concat([new_df, temp])
             if len(new_df) == 0:

openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py

@@ -1,8 +1,14 @@
 """FMP Equity Quote Model."""
 
+# pylint: disable=unused-argument
+
 import asyncio
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
+from datetime import (
+    date as dateType,
+    datetime,
+    timezone,
+)
+from typing import Any, Dict, List, Optional, Union
 from warnings import warn
 
 from openbb_core.provider.abstract.data import ForceInt
@@ -12,7 +18,7 @@
     EquityQuoteQueryParams,
 )
 from openbb_core.provider.utils.errors import EmptyDataError
-from openbb_core.provider.utils.helpers import amake_request
+from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp
 from openbb_fmp.utils.helpers import get_querystring, response_callback
 from pydantic import Field, field_validator
 
@@ -63,22 +69,22 @@ class FMPEquityQuoteData(EquityQuoteData):
 
     @field_validator("last_timestamp", mode="before", check_fields=False)
     @classmethod
-    def validate_last_timestamp(cls, v):  # pylint: disable=E0213
+    def validate_last_timestamp(cls, v: Union[str, int]) -> Optional[dateType]:
         """Return the date as a datetime object."""
         if v:
             v = int(v) if isinstance(v, str) else v
-            return datetime.fromtimestamp(int(v), tz=timezone.utc)
+            return safe_fromtimestamp(v, tz=timezone.utc)
         return None
 
     @field_validator("earnings_announcement", mode="before", check_fields=False)
     @classmethod
-    def timestamp_validate(cls, v):  # pylint: disable=E0213
+    def timestamp_validate(cls, v: str) -> Optional[dateType]:
         """Return the datetime string as a datetime object."""
         if v:
             dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f%z")
             dt = dt.replace(microsecond=0)
             timestamp = dt.timestamp()
-            return datetime.fromtimestamp(timestamp, tz=timezone.utc)
+            return safe_fromtimestamp(timestamp, tz=timezone.utc)
         return None
 
     @field_validator("change_percent", mode="after", check_fields=False)
@@ -115,7 +121,7 @@ async def aextract_data(
 
         symbols = query.symbol.split(",")
 
-        results = []
+        results: list = []
 
         async def get_one(symbol):
             """Get data for one symbol."""

openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py

@@ -1,13 +1,18 @@
 """FMP Key Executives Model."""
 
-from datetime import datetime
-from typing import Any, Dict, List, Optional
+# pylint: disable=unused-argument
+
+from datetime import (
+    date as dateType,
+)
+from typing import Any, Dict, List, Optional, Union
 
 from openbb_core.provider.abstract.fetcher import Fetcher
 from openbb_core.provider.standard_models.key_executives import (
     KeyExecutivesData,
     KeyExecutivesQueryParams,
 )
+from openbb_core.provider.utils.helpers import safe_fromtimestamp
 from openbb_fmp.utils.helpers import get_data_many
 from pydantic import field_validator
 
@@ -24,9 +29,12 @@ class FMPKeyExecutivesData(KeyExecutivesData):
 
     @field_validator("titleSince", mode="before", check_fields=False)
     @classmethod
-    def time_validate(cls, v):  # pylint: disable=E0213
+    def time_validate(cls, v: Union[float, int]) -> Optional[dateType]:
         """Return the date as a datetime object."""
-        return datetime.fromtimestamp(v / 1000)
+        if v:
+            v = v / 1000
+            return safe_fromtimestamp(v)
+        return v  # type: ignore
 
 
 class FMPKeyExecutivesFetcher(

openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py

@@ -1,5 +1,7 @@
 """FMP Market Indices Model."""
 
+# pylint: disable=unused-argument
+
 from datetime import datetime
 from typing import Any, Dict, List, Literal, Optional
 

openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py

@@ -1,8 +1,11 @@
 """FMP Market Snapshots Model."""
 
+# pylint: disable=unused-argument
+
 from datetime import (
     date as dateType,
     datetime,
+    timezone,
 )
 from typing import Any, Dict, List, Optional, Union
 
@@ -13,6 +16,7 @@
     MarketSnapshotsData,
     MarketSnapshotsQueryParams,
 )
+from openbb_core.provider.utils.helpers import safe_fromtimestamp
 from openbb_fmp.utils.definitions import EXCHANGES
 from openbb_fmp.utils.helpers import get_data
 from pydantic import Field, field_validator
@@ -87,14 +91,20 @@ class FMPMarketSnapshotsData(MarketSnapshotsData):
 
     @field_validator("last_price_timestamp", mode="before", check_fields=False)
     @classmethod
-    def validate_timestamp(cls, v):
+    def validate_timestamp(cls, v: Union[str, int, float]) -> Optional[dateType]:
         """Validate the timestamp."""
+        if isinstance(v, str):
+            try:
+                v = float(v)
+            except ValueError:
+                return None
+
         if isinstance(v, (int, float)) and v != 0:
             try:
-                v = datetime.fromtimestamp(v)
-                if v.hour == 0 and v.minute == 0 and v.second == 0:
-                    v = v.date()
-                return v
+                v = safe_fromtimestamp(v, tz=timezone.utc)  # type: ignore
+                if v.hour == 0 and v.minute == 0 and v.second == 0:  # type: ignore
+                    v = v.date()  # type: ignore
+                return v  # type: ignore
             except ValueError:
                 return None
         return None

openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py

@@ -7,6 +7,7 @@
 from datetime import (
     date as dateType,
     datetime,
+    timezone as datetime_timezone,
 )
 from io import BytesIO
 from typing import Any, Dict, List, Optional, Union
@@ -16,7 +17,7 @@
     MarketSnapshotsData,
     MarketSnapshotsQueryParams,
 )
-from openbb_core.provider.utils.helpers import amake_request
+from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp
 from pandas import DataFrame, notna, read_csv, to_datetime
 from pydantic import Field
 from pytz import timezone
@@ -105,7 +106,7 @@ def transform_query(params: Dict[str, Any]) -> IntrinioMarketSnapshotsQueryParam
                 dt = transformed_params["date"]
                 dt = dt.astimezone(tz=timezone("America/New_York"))
             if isinstance(transformed_params["date"], dateType):
-                dt = transformed_params["date"]
+                dt = transformed_params["date"]  # type: ignore
                 if isinstance(dt, dateType):
                     dt = datetime(
                         dt.year,
@@ -143,7 +144,6 @@ async def aextract_data(
         **kwargs: Any,
     ) -> List[Dict]:
         """Return the raw data from the Intrinio endpoint."""
-
         api_key = credentials.get("intrinio_api_key") if credentials else ""
 
         # This gets the URL to the actual file.
@@ -216,7 +216,7 @@ async def get_csv(url):
                     to_datetime(
                         df[col].apply(
                             lambda x: (
-                                datetime.fromtimestamp(x, tz=timezone("UTC"))
+                                safe_fromtimestamp(x, tz=datetime_timezone.utc)
                                 if notna(x)
                                 else x
                             )

openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py

@@ -3,7 +3,10 @@
 # pylint: disable=unused-argument,protected-access,line-too-long
 
 import warnings
-from datetime import datetime
+from datetime import (
+    datetime,
+    timezone,
+)
 from typing import Any, Dict, List, Literal, Optional
 
 from dateutil.relativedelta import relativedelta
@@ -18,14 +21,14 @@
     ClientResponse,
     ClientSession,
     amake_requests,
+    safe_fromtimestamp,
 )
 from pydantic import (
     Field,
     PositiveInt,
     PrivateAttr,
     model_validator,
 )
-from pytz import timezone
 
 _warn = warnings.warn
 
@@ -146,17 +149,18 @@ async def callback(
             data = await response.json()
 
             symbol = response.url.parts[4]
-            next_url = data.get("next_url", None)
-            results: list = data.get("results", [])
+            next_url = data.get("next_url", None)  # type: ignore
+            results: list = data.get("results", [])  # type: ignore
 
             while next_url:
                 url = f"{next_url}&apiKey={api_key}"
                 data = await session.get_json(url)
-                results.extend(data.get("results", []))
-                next_url = data.get("next_url", None)
+                results.extend(data.get("results", []))  # type: ignore
+                next_url = data.get("next_url", None)  # type: ignore
 
             for r in results:
-                r["t"] = datetime.fromtimestamp(r["t"] / 1000, tz=timezone("UTC"))
+                v = r["t"] / 1000  # milliseconds to seconds
+                r["t"] = safe_fromtimestamp(v, tz=timezone.utc)  # type: ignore[arg-type]
                 if query._timespan not in ["second", "minute", "hour"]:
                     r["t"] = r["t"].date().strftime("%Y-%m-%d")
                 else: