test

miraisolutions · Oct 28, 2024 · f512e1b · f512e1b
1 parent cd8bc09
commit f512e1b
Show file tree

Hide file tree

Showing 3 changed files with 329 additions and 2 deletions.
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -13,3 +13,4 @@
 cran-comments.md
 ^\.github$
 ^adc.json$
+^spark_versions.json$
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -25,11 +25,10 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          # Spark 3.5 seems currently unavailable so we use 3.4 for now
           - os-name: ubuntu
             os-version: latest
             java: 17
-            spark: "3.4"
+            spark: "3.5"
             r-version: release
           # - os-name: ubuntu
           #   os-version: latest
@@ -67,6 +66,7 @@ jobs:
       BIGQUERY_MATERIALIZATION_DATASET: test
       BIGQUERY_APPLICATION_CREDENTIALS: ${{ github.workspace }}/adc.json
       R_DEFAULT_INTERNET_TIMEOUT: 1800
+      R_SPARKINSTALL_INSTALL_INFO_PATH: ${{ github.workspace }}/spark_versions.json
 
     steps:
       - uses: actions/checkout@v4

diff --git a/spark_versions.json b/spark_versions.json
@@ -0,0 +1,326 @@
+[
+  {
+    "spark": "2.4.0",
+    "hadoop": "2.6",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.0",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.1",
+    "hadoop": "2.6",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.1",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.2",
+    "hadoop": "2.6",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.2",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.3",
+    "hadoop": "2.6",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.3",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.4",
+    "hadoop": "2.6",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.4/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.4",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.4/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.5",
+    "hadoop": "2.6",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.5/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.5",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.5/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.6",
+    "hadoop": "2.6",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.6/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.6",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.6/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.7",
+    "hadoop": "2.6",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.7/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "2.4.7",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-2.4.7/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.0.0",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.0.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.0.0",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.0.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.0.1",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.0.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.0.1",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.0.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.0.2",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.0.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.0.2",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.0.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.0.3",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.0.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.0.3",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.0.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.1.1",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.1.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.1.1",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.1.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.1.2",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.1.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.1.2",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.1.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.1.3",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.1.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.1.3",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.1.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.0",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.0",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.1",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.1",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.2",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.2",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.3",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.3",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.4",
+    "hadoop": "2.7",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.4/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.2.4",
+    "hadoop": "3.2",
+    "base": "https://archive.apache.org/dist/spark/spark-3.2.4/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.3.0",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.3.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.3.1",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.3.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.3.2",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.3.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.3.3",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.3.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.3.4",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.3.4/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.4.0",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.4.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.4.1",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.4.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.4.2",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.4.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.4.3",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.4.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.4.4",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.4.4/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.5.0",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.5.0/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.5.1",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.5.1/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.5.2",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.5.2/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  },
+  {
+    "spark": "3.5.3",
+    "hadoop": "3",
+    "base": "https://archive.apache.org/dist/spark/spark-3.5.3/",
+    "pattern": "spark-%s-bin-hadoop%s.tgz"
+  }
+]