Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BugFix] Merge Fix FMP Intraday (#6229) From Main #6235

Merged
merged 14 commits into from
Mar 19, 2024
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,15 @@ def test_crypto_search(params, headers):
"end_date": "2023-01-02",
}
),
(
{
"interval": "1h",
"provider": "fmp",
"symbol": "BTCUSD,ETHUSD",
"start_date": None,
"end_date": None,
}
),
(
{
"interval": "1m",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,15 @@ def test_crypto_search(params, obb):
"end_date": "2023-01-02",
}
),
(
{
"interval": "1h",
"provider": "fmp",
"symbol": "BTCUSD,ETHUSD",
"start_date": None,
"end_date": None,
}
),
(
{
"interval": "1m",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,15 @@ def test_currency_search(params, headers):
"provider": "fmp",
}
),
(
{
"interval": "1h",
"provider": "fmp",
"symbol": "EURUSD,USDJPY",
"start_date": None,
"end_date": None,
}
),
(
{
"interval": "1m",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,15 @@ def test_currency_search(params, obb):
"provider": "fmp",
}
),
(
{
"interval": "1h",
"provider": "fmp",
"symbol": "EURUSD,USDJPY",
"start_date": None,
"end_date": None,
}
),
(
{
"interval": "1m",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -939,6 +939,15 @@ def test_equity_compare_groups(params, headers):
"interval": "1d",
}
),
(
{
"interval": "1h",
"provider": "fmp",
"symbol": "AAPL,MSFT",
"start_date": None,
"end_date": None,
}
),
(
{
"timezone": "UTC",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -878,6 +878,15 @@ def test_equity_compare_groups(params, obb):
"interval": "1d",
}
),
(
{
"interval": "1h",
"provider": "fmp",
"symbol": "AAPL,MSFT",
"start_date": None,
"end_date": None,
}
),
(
{
"timezone": "UTC",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ def test_index_constituents(params, headers):
"end_date": "2024-02-05",
}
),
(
{
"interval": "1h",
"provider": "fmp",
"symbol": "^DJI,^NDX",
"start_date": None,
"end_date": None,
}
),
(
{
"interval": "1m",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,15 @@ def test_index_constituents(params, obb):
"end_date": "2024-02-05",
}
),
(
{
"interval": "1h",
"provider": "fmp",
"symbol": "^DJI,^NDX",
"start_date": None,
"end_date": None,
}
),
(
{
"interval": "1m",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@

# pylint: disable=unused-argument

import warnings
import asyncio
from datetime import datetime
from typing import Any, Dict, List, Literal, Optional
from warnings import warn

from dateutil.relativedelta import relativedelta
from openbb_core.provider.abstract.fetcher import Fetcher
Expand All @@ -16,16 +17,14 @@
DATA_DESCRIPTIONS,
QUERY_DESCRIPTIONS,
)
from openbb_core.provider.utils.errors import EmptyDataError
from openbb_core.provider.utils.helpers import (
ClientResponse,
amake_requests,
amake_request,
get_querystring,
)
from openbb_fmp.utils.helpers import get_interval
from pydantic import Field

_warn = warnings.warn


class FMPCryptoHistoricalQueryParams(CryptoHistoricalQueryParams):
"""
Expand Down Expand Up @@ -104,29 +103,55 @@ def get_url_params(symbol: str) -> str:
url = f"{base_url}/historical-price-full/crypto/{url_params}"
return url

# if there are more than 20 symbols, we need to increase the timeout
if len(query.symbol.split(",")) > 20:
kwargs.update({"preferences": {"request_timeout": 30}})
symbols = query.symbol.split(",")

results = []
messages = []

async def get_one(symbol):
"""Get data for one symbol."""

url = get_url_params(symbol)

data = []

response = await amake_request(url, **kwargs)

async def callback(response: ClientResponse, _: Any) -> List[Dict]:
data = await response.json()
symbol = response.url.parts[-1]
results = []
if not data:
_warn(f"No data found the the symbol: {symbol}")
return results
if isinstance(response, dict) and response.get("Error Message"):
message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}"
warn(message)
messages.append(message)

if isinstance(data, dict):
results = data.get("historical", [])
if not response:
message = f"No data found for {symbol}."
warn(message)
messages.append(message)

if "," in query.symbol:
for d in results:
d["symbol"] = symbol
return results
if isinstance(response, list) and len(response) > 0:
data = response
if len(symbols) > 1:
for d in data:
d["symbol"] = symbol

urls = [get_url_params(symbol) for symbol in query.symbol.split(",")]
if isinstance(response, dict) and response.get("historical"):
data = response["historical"]
if len(symbols) > 1:
for d in data:
d["symbol"] = symbol

return await amake_requests(urls, callback, **kwargs)
if data:
results.extend(data)

tasks = [get_one(symbol) for symbol in symbols]

await asyncio.gather(*tasks)

if not results:
raise EmptyDataError(
f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}"
)

return results

@staticmethod
def transform_data(
Expand All @@ -138,7 +163,15 @@ def transform_data(
to_pop = ["label", "changePercent", "unadjustedVolume"]
results: List[FMPCryptoHistoricalData] = []

for d in sorted(data, key=lambda x: x["date"], reverse=False):
for d in sorted(
data,
key=lambda x: (
(x["date"], x["symbol"])
if len(query.symbol.split(",")) > 1
else x["date"]
),
reverse=False,
):
_ = [d.pop(pop) for pop in to_pop if pop in d]
if d.get("unadjusted_volume") == d.get("volume"):
_ = d.pop("unadjusted_volume")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@

# pylint: disable=unused-argument

import warnings
import asyncio
from datetime import datetime
from typing import Any, Dict, List, Literal, Optional
from warnings import warn

from dateutil.relativedelta import relativedelta
from openbb_core.provider.abstract.fetcher import Fetcher
Expand All @@ -16,16 +17,14 @@
DATA_DESCRIPTIONS,
QUERY_DESCRIPTIONS,
)
from openbb_core.provider.utils.errors import EmptyDataError
from openbb_core.provider.utils.helpers import (
ClientResponse,
amake_requests,
amake_request,
get_querystring,
)
from openbb_fmp.utils.helpers import get_interval
from pydantic import Field

_warn = warnings.warn


class FMPCurrencyHistoricalQueryParams(CurrencyHistoricalQueryParams):
"""FMP Currency Historical Price Query.
Expand Down Expand Up @@ -102,29 +101,55 @@ def get_url_params(symbol: str) -> str:
url = f"{base_url}/historical-price-full/forex/{url_params}"
return url

# if there are more than 20 symbols, we need to increase the timeout
if len(query.symbol.split(",")) > 20:
kwargs.update({"preferences": {"request_timeout": 30}})
symbols = query.symbol.split(",")

results = []
messages = []

async def get_one(symbol):
"""Get data for one symbol."""

url = get_url_params(symbol)

data = []

response = await amake_request(url, **kwargs)

async def callback(response: ClientResponse, _: Any) -> List[Dict]:
data = await response.json()
symbol = response.url.parts[-1]
results = []
if not data:
_warn(f"No data found the the symbol: {symbol}")
return results
if isinstance(response, dict) and response.get("Error Message"):
message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}"
warn(message)
messages.append(message)

if isinstance(data, dict):
results = data.get("historical", [])
if not response:
message = f"No data found for {symbol}."
warn(message)
messages.append(message)

if "," in query.symbol:
for d in results:
d["symbol"] = symbol
return results
if isinstance(response, list) and len(response) > 0:
data = response
if len(symbols) > 1:
for d in data:
d["symbol"] = symbol

urls = [get_url_params(symbol) for symbol in query.symbol.split(",")]
if isinstance(response, dict) and response.get("historical"):
data = response["historical"]
if len(symbols) > 1:
for d in data:
d["symbol"] = symbol

return await amake_requests(urls, callback, **kwargs)
if data:
results.extend(data)

tasks = [get_one(symbol) for symbol in symbols]

await asyncio.gather(*tasks)

if not results:
raise EmptyDataError(
f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}"
)

return results

@staticmethod
def transform_data(
Expand All @@ -136,7 +161,15 @@ def transform_data(
to_pop = ["label", "changePercent", "unadjustedVolume"]
results: List[FMPCurrencyHistoricalData] = []

for d in sorted(data, key=lambda x: x["date"], reverse=False):
for d in sorted(
data,
key=lambda x: (
(x["date"], x["symbol"])
if len(query.symbol.split(",")) > 1
else x["date"]
),
reverse=False,
):
_ = [d.pop(pop) for pop in to_pop if pop in d]
if d.get("unadjusted_volume") == d.get("volume"):
_ = d.pop("unadjusted_volume")
Expand Down
Loading
Loading