fix yahoo api

pdet · Sep 14, 2024 · 6639e35 · 6639e35
1 parent 1024914
commit 6639e35
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 313 deletions.
diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml
@@ -16,7 +16,7 @@ jobs:
     name: Build extension binaries
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.0.0
     with:
-      duckdb_version: v1.0.0
+      duckdb_version: v1.1.0
       extension_name: scrooge
       exclude_archs: "windows_amd64_rtools"
 
@@ -26,7 +26,7 @@ jobs:
     uses: ./.github/workflows/_extension_deploy.yml
     secrets: inherit
     with:
-      duckdb_version: v1.0.0
+      duckdb_version: v1.1.0
       extension_name: scrooge
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
       exclude_archs: "windows_amd64_rtools"
diff --git a/duckdb b/duckdb
diff --git a/extension_config.cmake b/extension_config.cmake
@@ -8,6 +8,4 @@ duckdb_extension_load(scrooge
 
 # Build the httpfs extension to test with s3/http
 duckdb_extension_load(httpfs)
-
-# Any extra extensions that should be built
-# e.g.: duckdb_extension_load(json)
+duckdb_extension_load(json)
diff --git a/src/scanner/yahoo_finance.cpp b/src/scanner/yahoo_finance.cpp
@@ -84,12 +84,20 @@ shared_ptr<Relation> GeneratePlan(YahooFunctionData &bind_data) {
   bind_data.from_epoch += bind_data.increment_epoch;
   bind_data.cur_to_epoch += bind_data.increment_epoch;
 
-  string url = "https://query1.finance.yahoo.com/v7/finance/download/" +
+  string url = "https://query2.finance.yahoo.com/v8/finance/chart/" +
                bind_data.symbol + "?period1=" + from + "&period2=" + to +
                "&interval=" + bind_data.interval + "&events=history";
-  string query = "SELECT '" + bind_data.symbol + "' as symbol, * " +
-                 "FROM read_csv('" + url + "');";
-
+  string query =
+      "SELECT '" + bind_data.symbol +
+      "'as symbol, list_transform(chart.result[1].timestamp, x -> "
+      "make_timestamp(x*1000000)::date) as date, "
+      "chart.result[1].indicators.quote[1].open as open, "
+      "chart.result[1].indicators.quote[1].high as high, "
+      "chart.result[1].indicators.quote[1].low as low, "
+      "chart.result[1].indicators.quote[1].close as close, "
+      "chart.result[1].indicators.adjclose[1].adjclose as adj_close, "
+      "chart.result[1].indicators.quote[1].volume as volume " +
+      "FROM read_json('" + url + "');";
   return bind_data.conn->RelationFromQuery(query);
 }
 

diff --git a/src/scrooge_extension.cpp b/src/scrooge_extension.cpp
@@ -35,13 +35,14 @@ void ScroogeExtension::Load(DuckDB &db) {
   catalog.CreateTableFunction(*con.context, &yahoo_scanner_info);
 
   // Create Portfolio Frontier Function
-  TableFunction portfolio_frontier(
-      "portfolio_frontier",
-      {duckdb::LogicalType::LIST(duckdb::LogicalType::VARCHAR),
-       LogicalType::ANY, LogicalType::ANY, LogicalType::INTEGER},
-      scrooge::PortfolioFrontier::Scan, scrooge::PortfolioFrontier::Bind);
-  CreateTableFunctionInfo portfolio_frontier_info(portfolio_frontier);
-  catalog.CreateTableFunction(*con.context, &portfolio_frontier_info);
+  // FIXME: this should not be dependent of the yahoo scanner
+  // TableFunction portfolio_frontier(
+  //     "portfolio_frontier",
+  //     {duckdb::LogicalType::LIST(duckdb::LogicalType::VARCHAR),
+  //      LogicalType::ANY, LogicalType::ANY, LogicalType::INTEGER},
+  //     scrooge::PortfolioFrontier::Scan, scrooge::PortfolioFrontier::Bind);
+  // CreateTableFunctionInfo portfolio_frontier_info(portfolio_frontier);
+  // catalog.CreateTableFunction(*con.context, &portfolio_frontier_info);
 
   // Create Ethereum Scanner Function
   TableFunction ethereum_rpc_scanner(

diff --git a/test/sql/scrooge/test_eth_scan.test b/test/sql/scrooge/test_eth_scan.test
@@ -68,15 +68,13 @@ FROM read_eth(
 )
 ----
 
-statement error
+statement ok
 FROM read_eth(
     'USDT',
     'Transfer',
     99999999999,
     999999999999
 )
-----
-range is too large
 
 statement error
 FROM read_eth(

diff --git a/test/sql/scrooge/test_portfolio_frontier.test b/test/sql/scrooge/test_portfolio_frontier.test
diff --git a/test/sql/scrooge/test_yahoo.test b/test/sql/scrooge/test_yahoo.test