feat(oay): Add basic s3 list_objects_v2 with start_after support (apa…

…che#2219)

* feat(oay): Add list objects v2 support

Signed-off-by: Xuanwo <github@xuanwo.io>

* Update

Signed-off-by: Xuanwo <github@xuanwo.io>

* Return error if list with start_after is not supported

Signed-off-by: Xuanwo <github@xuanwo.io>

---------

Signed-off-by: Xuanwo <github@xuanwo.io>

bin/oay/.gitignore

@@ -0,0 +1 @@
+oay.toml

bin/oay/Cargo.toml

@@ -31,12 +31,13 @@ version.workspace = true
 
 [dependencies]
 anyhow = "1"
+axum = "0.6"
+chrono = "0.4.24"
 clap = { version = "4", features = ["cargo", "string"] }
 dirs = "5.0.0"
-env_logger = "0.10"
 futures = "0.3"
-log = "0.4"
 opendal.workspace = true
+quick-xml = { version = "0.27", features = ["serialize", "overlapped-lists"] }
 serde = { version = "1", features = ["derive"] }
 tokio = { version = "1.27", features = [
   "fs",
@@ -45,4 +46,9 @@ tokio = { version = "1.27", features = [
   "io-std",
 ] }
 toml = "0.7.3"
+tower = "0.4"
+tower-http = { version = "0.4", features = ["trace"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 url = "2.3.1"
+uuid = { version = "1", features = ["v4", "fast-rng"] }

bin/oay/README.md

@@ -11,3 +11,7 @@ Allow users to access different storage backend through their preferred APIs.
 ## Status
 
 Our first milestone is to provide S3 APIs.
+
+### S3 API
+
+Only `list_object_v2` with `start_after` is supported.

bin/oay/oay.toml.example

@@ -0,0 +1,9 @@
+[backend]
+type = "s3"
+
+access_key_id = "access_key_id"
+secret_access_key = "secret_access_key"
+
+[frontends.s3]
+enable = true
+addr = "127.0.0.1:2000"

bin/oay/src/bin/oay.rs

@@ -16,10 +16,37 @@
 // under the License.
 
 use anyhow::Result;
+use oay::services::S3Service;
+use oay::Config;
+use opendal::services::Memory;
+use opendal::Operator;
+use std::sync::Arc;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    println!("Hello, world!");
+    tracing_subscriber::registry()
+        .with(fmt::layer().pretty())
+        .with(EnvFilter::from_default_env())
+        .init();
+
+    let cfg: Config = Config {
+        backend: oay::BackendConfig {
+            typ: "memory".to_string(),
+        },
+        frontends: oay::FrontendsConfig {
+            s3: oay::S3Config {
+                enable: true,
+                addr: "127.0.0.1:3000".to_string(),
+            },
+        },
+    };
+
+    let op = Operator::new(Memory::default())?.finish();
+
+    let s3 = S3Service::new(Arc::new(cfg), op);
+
+    s3.serve().await?;
 
     Ok(())
 }

bin/oay/src/config.rs

@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Serialize, Deserialize)]
+pub struct Config {
+    pub backend: BackendConfig,
+    pub frontends: FrontendsConfig,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BackendConfig {
+    #[serde(rename = "type")]
+    pub typ: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct FrontendsConfig {
+    pub s3: S3Config,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct S3Config {
+    pub enable: bool,
+    pub addr: String,
+}

bin/oay/lib.rs → bin/oay/src/lib.rs

@@ -14,3 +14,8 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+pub mod services;
+
+mod config;
+pub use config::*;

bin/oay/src/services/mod.rs

@@ -0,0 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod s3;
+pub use s3::S3Service;

bin/oay/src/services/s3/mod.rs

@@ -0,0 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod service;
+pub use service::*;