Initial s3 commit

.github/workflows/checks.yml

@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Checks
+on: [push, pull_request]
+
+jobs:
+  fmt:
+    name: Format
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64]
+        rust: [stable]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+          rustup component add rustfmt
+      - name: Run
+        run: cargo fmt --all -- --check
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64]
+        rust: [stable]
+    container:
+      image: ${{ matrix.arch }}/rust
+      env:
+        # Disable full debug symbol generation to speed up CI build and keep memory down
+        # "1" means line tables only, which is useful for panic tracebacks.
+        RUSTFLAGS: "-C debuginfo=1"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Cache Cargo
+        uses: actions/cache@v2
+        with:
+          path: /home/runner/.cargo
+          key: cargo-S3-cache-
+      - name: Cache Rust dependencies
+        uses: actions/cache@v2
+        with:
+          path: /home/runner/target
+          key: target-S3-cache-
+      - name: Setup Rust toolchain
+        run: |
+          rustup toolchain install ${{ matrix.rust }}
+          rustup default ${{ matrix.rust }}
+          rustup component add rustfmt clippy
+      - name: Run clippy
+        run: |
+          cargo clippy --all-features --workspace -- -D warnings
+        env:
+          CARGO_HOME: "/github/home/.cargo"
+          CARGO_TARGET_DIR: "/github/home/target"

.github/workflows/rust.yml → .github/workflows/test.yml

@@ -20,7 +20,7 @@ on: [push, pull_request]
 
 jobs:
   test:
-    name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
+    name: Core
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -49,9 +49,9 @@ jobs:
       - name: Run tests (ex-FlightSQL)
         run: |
           # All features except FlightSQL which requires being run on a single thread for determinism
-          cargo test --features=deltalake,s3,functions-json
+          cargo test --features=deltalake
   test-flightsql:
-    name: Test FlightSQL on AMD64 Rust ${{ matrix.rust }}
+    name: FlightSQL
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -81,33 +81,39 @@ jobs:
           # Single thread needed because we spin up a server that listens on port and we need each
           # test to only be run against the server spun up in that test.  With parallelism tests 
           # can connec to server in different test which breaks determinism.
-          cargo test --features=flightsql -- --test-threads=1
-
-  fmt:
-    name: Rust formatting
+          cargo test --features=flightsql extension_cases::flightsql -- --test-threads=1
+  test-s3:
+    name: S3
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64]
+        rust: [stable]
     steps:
-      - uses: actions/checkout@v2
-      - name: Setup toolchain
+      - uses: actions/checkout@v3
+
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@v0.2.3
+        with:
+          image-tag: 'latest'
+          install-awslocal: 'true'
+          configuration: DEBUG=1
+
+      - name: Run Tests against LocalStack
         run: |
-          rustup toolchain install stable
-          rustup default stable
-          rustup component add rustfmt
-      - name: Run
-        run: cargo fmt --all -- --check
-  clippy:
-    name: Clippy
+          awslocal s3 mb s3://test
+          awslocal s3 mv data/aggregate_test_100.csv s3://test/
+          echo "Test Execution complete!"
+      - name: Run S3 tests
+        run: |
+          cargo test --features=s3 extension_cases::s3
+  test-functions-json:
+    name: Functions-JSON
     runs-on: ubuntu-latest
     strategy:
       matrix:
         arch: [amd64]
         rust: [stable]
-    container:
-      image: ${{ matrix.arch }}/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v2
         with:
@@ -116,20 +122,17 @@ jobs:
         uses: actions/cache@v2
         with:
           path: /home/runner/.cargo
-          key: cargo-S3-cache-
+          key: cargo-dft-cache-
       - name: Cache Rust dependencies
         uses: actions/cache@v2
         with:
           path: /home/runner/target
-          key: target-S3-cache-
+          key: target-dft-cache-
       - name: Setup Rust toolchain
         run: |
           rustup toolchain install ${{ matrix.rust }}
           rustup default ${{ matrix.rust }}
-          rustup component add rustfmt clippy
-      - name: Run clippy
+          rustup component add rustfmt
+      - name: Run FlightSQL tests
         run: |
-          cargo clippy --all-targets --workspace -- -D warnings
-        env:
-          CARGO_HOME: "/github/home/.cargo"
-          CARGO_TARGET_DIR: "/github/home/target"
+          cargo test --features=functions-json extension_cases::functions_json

src/extensions/s3.rs

@@ -51,7 +51,7 @@ impl Extension for AwsS3Extension {
         for s3_config in s3_configs {
             match s3_config.to_object_store() {
                 Ok(object_store) => {
-                    info!("Created object store");
+                    info!("Created object store: {}", object_store);
                     if let Some(object_store_url) = s3_config.object_store_url() {
                         info!("Endpoint exists");
                         if let Ok(parsed_endpoint) = Url::parse(object_store_url) {

tests/cli_cases/mod.rs

@@ -17,8 +17,6 @@
 
 mod basic;
 mod config;
-#[cfg(feature = "flightsql")]
-mod flightsql;
 
 use assert_cmd::Command;
 use predicates::str::ContainsPredicate;

tests/config.rs

@@ -19,6 +19,7 @@ use std::{io::Write, path::PathBuf};
 
 use tempfile::{tempdir, TempDir};
 
+#[derive(Debug)]
 pub struct TestConfig {
     #[allow(dead_code)]
     dir: TempDir,
@@ -42,8 +43,35 @@ impl TestConfigBuilder {
 
     pub fn with_ddl_path(&mut self, ddl_path: PathBuf) -> &mut Self {
         self.config_text.push_str("[execution]\n");
-        let param = format!("ddl_path = '{}'", ddl_path.display());
+        let param = format!("ddl_path = '{}'\n", ddl_path.display());
         self.config_text.push_str(&param);
         self
     }
+    #[allow(clippy::too_many_arguments)]
+    pub fn with_s3_object_store(
+        &mut self,
+        store: &str,
+        bucket_name: &str,
+        object_store_url: &str,
+        endpoint: &str,
+        access_key: &str,
+        secret_key: &str,
+        allow_http: bool,
+    ) -> &mut Self {
+        self.config_text
+            .push_str(&format!("[[execution.object_store.{}]]\n", store));
+        self.config_text
+            .push_str(&format!("bucket_name = '{}'\n", bucket_name));
+        self.config_text
+            .push_str(&format!("object_store_url = '{}'\n", object_store_url));
+        self.config_text
+            .push_str(&format!("aws_endpoint = '{}'\n", endpoint));
+        self.config_text
+            .push_str(&format!("aws_access_key_id = '{}'\n", access_key));
+        self.config_text
+            .push_str(&format!("aws_secret_access_key = '{}'\n", secret_key));
+        self.config_text
+            .push_str(&format!("aws_allow_http = {}\n", allow_http));
+        self
+    }
 }

tests/extension_cases/mod.rs

@@ -17,8 +17,12 @@
 
 //! Tests for extensions (stored in the `extension_cases` directory)
 
+#[cfg(feature = "flightsql")]
+mod flightsql;
 #[cfg(feature = "functions-json")]
 mod functions_json;
+#[cfg(feature = "s3")]
+mod s3;
 
 use datafusion::arrow::array::RecordBatch;
 use datafusion::arrow::util::pretty::pretty_format_batches;

tests/extension_cases/s3.rs

@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io::Write;
+
+use assert_cmd::Command;
+
+use crate::{cli_cases::contains_str, config::TestConfigBuilder};
+
+#[test]
+fn test_s3_basic() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let ddl_path = tempdir.path().join("my_ddl.sql");
+    let mut file = std::fs::File::create(ddl_path.clone()).unwrap();
+    let ddl = "CREATE EXTERNAL TABLE a STORED AS CSV LOCATION 's3://test/aggregate_test_100.csv';";
+    file.write_all(ddl.as_bytes()).unwrap();
+    file.flush().unwrap();
+    let mut config_builder = TestConfigBuilder::default();
+    config_builder.with_ddl_path(ddl_path);
+    config_builder.with_s3_object_store(
+        "s3",
+        "test",
+        "s3://test",
+        "http://localhost:4566",
+        "LSIAQAAAAAAVNCBMPNSG",
+        "5555555555555555555555555555555555555555",
+        true,
+    );
+    let config = config_builder.build("my_config.toml");
+
+    let assert = Command::cargo_bin("dft")
+        .unwrap()
+        .arg("--config")
+        .arg(config.path)
+        .arg("--run-ddl")
+        .arg("-c")
+        .arg("SELECT column_1 FROM a LIMIT 1")
+        .assert()
+        .success();
+
+    let expected = r#"
++----------+
+| column_1 |
++----------+
+| c1       |
++----------+
+"#;
+
+    assert.stdout(contains_str(expected));
+}