From c86460f4725f789a01563db8661d92f52a321e11 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 4 Nov 2023 22:24:18 -0700 Subject: [PATCH 1/4] add multi-ticker support to etf_countries --- .../standard_models/etf_countries.py | 3 - .../fmp/openbb_fmp/models/etf_countries.py | 53 +++-- .../test_fmp_etf_countries_fetcher.yaml | 182 +++++++++++++++++- .../providers/fmp/tests/test_fmp_fetchers.py | 2 +- 4 files changed, 215 insertions(+), 25 deletions(-) diff --git a/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py b/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py index a2df5c732f35..99daf311c239 100644 --- a/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py +++ b/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py @@ -27,6 +27,3 @@ class EtfCountriesData(Data): """ETF Countries Data.""" country: str = Field(description="The country of the exposure.") - weight: float = Field( - description="Exposure of the ETF to the country in normalized percentage points." - ) diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py b/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py index 160a0f207977..31970c63d104 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional +import pandas as pd from openbb_fmp.utils.helpers import create_url, get_data_many from openbb_provider.abstract.fetcher import Fetcher from openbb_provider.standard_models.etf_countries import ( @@ -17,8 +18,6 @@ class FMPEtfCountriesQueryParams(EtfCountriesQueryParams): class FMPEtfCountriesData(EtfCountriesData): """FMP ETF Country Weighting Data.""" - __alias_dict__ = {"weight": "weightPercentage"} - class FMPEtfCountriesFetcher( Fetcher[ @@ -40,24 +39,54 @@ def extract_data( **kwargs: Any, ) -> List[Dict]: """Return the raw data from the FMP endpoint.""" + api_key = credentials.get("fmp_api_key") if credentials else "" + symbols = ( + query.symbol.split(",") if "," in query.symbol else [query.symbol.upper()] + ) + results = {} - url = create_url( - version=3, - endpoint=f"etf-country-weightings/{query.symbol.upper()}", - api_key=api_key, + for symbol in symbols: + data = {} + url = create_url( + version=3, + endpoint=f"etf-country-weightings/{symbol}", + api_key=api_key, + ) + result = get_data_many(url, **kwargs) + df = pd.DataFrame(result).set_index("country") + if len(df) > 0: + for i in df.index: + data.update( + {i: float(df.loc[i]["weightPercentage"].replace("%", ""))} + ) + results.update({symbol: data}) + + output = ( + pd.DataFrame(results) + .transpose() + .reset_index() + .fillna(value=0) + .replace(0, None) + .rename(columns={"index": "symbol"}) + ).transpose() + output.columns = output.loc["symbol"].to_list() + output = output.drop("symbol", axis=0).sort_values( + by=output.columns[0], ascending=False ) - return get_data_many(url, **kwargs) + return ( + output.reset_index().rename(columns={"index": "country"}).to_dict("records") + ) @staticmethod def transform_data( query: FMPEtfCountriesQueryParams, data: List[Dict], **kwargs: Any ) -> List[FMPEtfCountriesData]: """Return the transformed data.""" - for d in data: - if d["weightPercentage"] is not None and d["weightPercentage"].endswith( - "%" - ): - d["weightPercentage"] = float(d["weightPercentage"][:-1]) / 100 + # for d in data: + # if d["weightPercentage"] is not None and d["weightPercentage"].endswith( + # "%" + # ): + # d["weightPercentage"] = float(d["weightPercentage"][:-1]) / 100 return [FMPEtfCountriesData.model_validate(d) for d in data] diff --git a/openbb_platform/providers/fmp/tests/record/http/test_fmp_fetchers/test_fmp_etf_countries_fetcher.yaml b/openbb_platform/providers/fmp/tests/record/http/test_fmp_fetchers/test_fmp_etf_countries_fetcher.yaml index 9b2bc7f8b230..ac658155a354 100644 --- a/openbb_platform/providers/fmp/tests/record/http/test_fmp_fetchers/test_fmp_etf_countries_fetcher.yaml +++ b/openbb_platform/providers/fmp/tests/record/http/test_fmp_fetchers/test_fmp_etf_countries_fetcher.yaml @@ -5,16 +5,18 @@ interactions: Accept: - '*/*' Accept-Encoding: - - gzip, deflate + - gzip, deflate, br Connection: - keep-alive method: GET - uri: https://financialmodelingprep.com/api/v3/etf-country-weightings/MISL?apikey=MOCK_API_KEY + uri: https://financialmodelingprep.com/api/v3/etf-country-weightings/VTI?apikey=MOCK_API_KEY response: body: - string: "[\n {\n \"country\": \"United States\",\n \"weightPercentage\": - \"98.35%\"\n },\n {\n \"country\": \"Other\",\n \"weightPercentage\": - \"1.65%\"\n }\n]" + string: !!binary | + H4sIAAAAAAAAA6XSTQuCQBAG4Hu/YlnoJqKG9nGrTkZUEJ2iw6SDCrnCOItZ9N9TusYYdNjTvPPw + srvnkVLP7iilk8oaplYvlD6ZgjFVRwbGWjufeYNFlvMBKUHDkGEfnEfuzB/rLvByvkp7zpEkwXOn + kQTEhDcwqUT4bhhKxLEp+IE0xHjuRGTWYCAFWQgCSVjamgluxQDiTSVkhVTa/3qsoS3BqLjur0R8 + 366MKG2QamxFQVrfYf89hltIxtbesbxWlrIfiNHlDcqTD93xAgAA headers: Access-Control-Allow-Credentials: - 'true' @@ -29,16 +31,178 @@ interactions: - '3600' Connection: - keep-alive - Content-Length: - - '141' + Content-Encoding: + - gzip Content-Type: - application/json; charset=utf-8 Date: - - Wed, 01 Nov 2023 13:27:52 GMT + - Sun, 05 Nov 2023 05:15:21 GMT ETag: - - W/"8d-2VmJo306+5EGW4NWPOj2rTFnMzA" + - W/"2f1-z5/4g/ulC3y6Aw5o83OT94EFKKU" Server: - nginx/1.18.0 (Ubuntu) + Transfer-Encoding: + - chunked + Vary: + - Accept-Encoding + X-Frame-Options: + - SAMEORIGIN + X-Powered-By: + - Express + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Connection: + - keep-alive + method: GET + uri: https://financialmodelingprep.com/api/v3/etf-country-weightings/QQQ?apikey=MOCK_API_KEY + response: + body: + string: !!binary | + H4sIAAAAAAAAA4XPwQrCMAyA4fueIhS8jTI3nc6beFdBPImHsIa2MCp0KTLEd3djV8kOOSV8/Hlk + AJ9xAFT7SoHjoA6g7sEzGbgxMvUqn/dv8tbxlWJLgdHSdNjUulmpcf/N/0Mx2YSDRBR6KxIn5wPK + QC0CZ2JHscNgxFcKvd+JHRjQLIRsSkm4TB0yUK0l4Jh6jtj5hYqympHs+QNActKX3gEAAA== + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-Requested-With, content-type, auth-token, Authorization, stripe-signature, + APPS + Access-Control-Allow-Methods: + - GET, POST, OPTIONS + Access-Control-Allow-Origin: + - '*' + Access-Control-Max-Age: + - '3600' + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 05 Nov 2023 05:15:21 GMT + ETag: + - W/"1de-u/Rt0fxDvduVLpiU+/1mhnLcZeE" + Server: + - nginx/1.18.0 (Ubuntu) + Transfer-Encoding: + - chunked + Vary: + - Accept-Encoding + X-Frame-Options: + - SAMEORIGIN + X-Powered-By: + - Express + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Connection: + - keep-alive + method: GET + uri: https://financialmodelingprep.com/api/v3/etf-country-weightings/VOO?apikey=MOCK_API_KEY + response: + body: + string: !!binary | + H4sIAAAAAAAAA4XQPQvCQAwG4L2/Ihy4leOqpX6Mbi4qFCdxOHqhPagnpClFxf/uHV0lDpne5CHJ + NQN4xwJQzWMMTE+1A3UJntFBzZZxUPmcT+jbjs9IDQa2LabG7XqhYvrJfzIHwt4GJwGFrjYSceIO + SQKMrqT5evL8Qvq3htHlSmL2SPfRWZkoROKI6ZS0iPjRyJTiTwey2MuCWc5CdvsCtZKIl98BAAA= + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-Requested-With, content-type, auth-token, Authorization, stripe-signature, + APPS + Access-Control-Allow-Methods: + - GET, POST, OPTIONS + Access-Control-Allow-Origin: + - '*' + Access-Control-Max-Age: + - '3600' + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 05 Nov 2023 05:15:22 GMT + ETag: + - W/"1df-OEmfqekeHOVvwG/zgq/j7mJKmsQ" + Server: + - nginx/1.18.0 (Ubuntu) + Transfer-Encoding: + - chunked + Vary: + - Accept-Encoding + X-Frame-Options: + - SAMEORIGIN + X-Powered-By: + - Express + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Connection: + - keep-alive + method: GET + uri: https://financialmodelingprep.com/api/v3/etf-country-weightings/IWM?apikey=MOCK_API_KEY + response: + body: + string: !!binary | + H4sIAAAAAAAAA6XTTUvDQBAG4Ht/xbLQWwlJa9R6jNAS8aNQPImHaTIkgWQik100iv/dBPEm7woe + 9rTLw7vz8bQw5mM6xtii9+J0tFfG5sotSWlX3zev3FS1O7AWLI4qnp/EUbpZ2un+c/Ur8SiN49Ic + HTkeELRNo80aSdc0diQmH+ZIkJoynSEpY+18SYhIovMEEQ+uZsUZtrAud71Q0WMhuUTCsZGKXnrl + AAJLsVOSIiQgIB+UuP1PhEzpvQkI8QWeDCHczjhap0i49W/cnXqvVSAHVDKqqaPAaMawGIfpK13g + KzEcrL1nlYHHgAFX7YY1LMCx2DcnpdYR3hEk3PO8Yn/Y9Z+eLJ6/ANaC2CnFBAAA + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-Requested-With, content-type, auth-token, Authorization, stripe-signature, + APPS + Access-Control-Allow-Methods: + - GET, POST, OPTIONS + Access-Control-Allow-Origin: + - '*' + Access-Control-Max-Age: + - '3600' + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 05 Nov 2023 05:15:22 GMT + ETag: + - W/"4c5-b5x+LUJL2Tk17h5pypIQmPJe3ag" + Server: + - nginx/1.18.0 (Ubuntu) + Transfer-Encoding: + - chunked + Vary: + - Accept-Encoding X-Frame-Options: - SAMEORIGIN X-Powered-By: diff --git a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py index 52b83e84be06..ea1af095292c 100644 --- a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py +++ b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py @@ -541,7 +541,7 @@ def test_fmp_price_performance_fetcher(credentials=test_credentials): @pytest.mark.record_http def test_fmp_etf_countries_fetcher(credentials=test_credentials): - params = {"symbol": "MISL"} + params = {"symbol": "VTI,QQQ,VOO,IWM"} fetcher = FMPEtfCountriesFetcher() result = fetcher.test(params, credentials) From 6cff19aeda5abcde2f429b18b1371fd1e19454c5 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 4 Nov 2023 23:14:40 -0700 Subject: [PATCH 2/4] remove commented-out lines --- .../providers/fmp/openbb_fmp/models/etf_countries.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py b/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py index 31970c63d104..676b5f92232b 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py @@ -84,9 +84,4 @@ def transform_data( query: FMPEtfCountriesQueryParams, data: List[Dict], **kwargs: Any ) -> List[FMPEtfCountriesData]: """Return the transformed data.""" - # for d in data: - # if d["weightPercentage"] is not None and d["weightPercentage"].endswith( - # "%" - # ): - # d["weightPercentage"] = float(d["weightPercentage"][:-1]) / 100 return [FMPEtfCountriesData.model_validate(d) for d in data] From 83bb559fd158dcebf6182af4b418308b2e6556d4 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:05:39 -0800 Subject: [PATCH 3/4] normalized percent points --- .../openbb_provider/standard_models/etf_countries.py | 4 +++- .../providers/fmp/openbb_fmp/models/etf_countries.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py b/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py index 99daf311c239..d1e25e858f86 100644 --- a/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py +++ b/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py @@ -26,4 +26,6 @@ def upper_symbol(cls, v: Union[str, List[str], Set[str]]): class EtfCountriesData(Data): """ETF Countries Data.""" - country: str = Field(description="The country of the exposure.") + country: str = Field( + description="The country of the exposure. Corresponding values are normalized percent points." + ) diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py b/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py index 676b5f92232b..54add8f0a29b 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/etf_countries.py @@ -58,7 +58,10 @@ def extract_data( if len(df) > 0: for i in df.index: data.update( - {i: float(df.loc[i]["weightPercentage"].replace("%", ""))} + { + i: float(df.loc[i]["weightPercentage"].replace("%", "")) + * 0.01 + } ) results.update({symbol: data}) From 1cc2082f8aa24c088038fe03edb3dd9b3d13b84c Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Nov 2023 23:26:34 -0800 Subject: [PATCH 4/4] percent -> percentage --- .../provider/openbb_provider/standard_models/etf_countries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py b/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py index d1e25e858f86..1168bd4196f3 100644 --- a/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py +++ b/openbb_platform/platform/provider/openbb_provider/standard_models/etf_countries.py @@ -27,5 +27,5 @@ class EtfCountriesData(Data): """ETF Countries Data.""" country: str = Field( - description="The country of the exposure. Corresponding values are normalized percent points." + description="The country of the exposure. Corresponding values are normalized percentage points." )