From 22d2cef0134deb51d1c3d5b6d4bca2e44878eb9e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 25 Dec 2024 17:02:02 +0800 Subject: [PATCH] feat(dev): Add config parse and generate support (#5454) --- dev/Cargo.lock | 342 +++++++++++++++++++ dev/Cargo.toml | 10 + dev/README.md | 19 ++ dev/src/generate/binding_python.rs | 25 ++ dev/src/generate/mod.rs | 34 ++ dev/src/generate/parser.rs | 520 +++++++++++++++++++++++++++++ dev/src/main.rs | 35 +- 7 files changed, 983 insertions(+), 2 deletions(-) create mode 100644 dev/README.md create mode 100644 dev/src/generate/binding_python.rs create mode 100644 dev/src/generate/mod.rs create mode 100644 dev/src/generate/parser.rs diff --git a/dev/Cargo.lock b/dev/Cargo.lock index 8c87366945dd..bbc6ca74ea1e 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -2,6 +2,348 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + +[[package]] +name = "clap" +version = "4.5.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + [[package]] name = "odev" version = "0.0.1" +dependencies = [ + "anyhow", + "clap", + "env_logger", + "log", + "pretty_assertions", + "syn", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index c7eda12b0194..7674646a94a8 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -26,3 +26,13 @@ license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.75" version = "0.0.1" + +[dependencies] +anyhow = "1.0.95" +clap = { version = "4.5.23", features = ["derive"] } +env_logger = "0.11.6" +log = "0.4.22" +syn = { version = "2.0.91", features = ["visit","full","extra-traits"] } + +[dev-dependencies] +pretty_assertions = "1.4.1" diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 000000000000..bbf68c5d25c2 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,19 @@ +# Apache OpenDAL™ Dev + +This project is designed to provide dev tools for the entire opendal project. + +## Usage + +```bash +cargo o help +``` + +## Features + +### Generate + +Generate code for OpenDAL services. + +```bash +cargo o generate -l python +``` diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs new file mode 100644 index 000000000000..7f3e3bdef83f --- /dev/null +++ b/dev/src/generate/binding_python.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::generate::parser::Services; +use anyhow::Result; + +pub fn generate(services: &Services) -> Result<()> { + println!("{:?}", services); + + Ok(()) +} diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs new file mode 100644 index 000000000000..743aa3b1c483 --- /dev/null +++ b/dev/src/generate/mod.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod parser; + +mod binding_python; + +use anyhow::Result; +use std::path::PathBuf; + +pub fn run(language: &str) -> Result<()> { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let services_path = manifest_dir.join("../core/src/services").canonicalize()?; + let services = parser::parse(&services_path.to_string_lossy())?; + + match language { + "python" | "py" => binding_python::generate(&services), + _ => Err(anyhow::anyhow!("Unsupported language: {}", language)), + } +} diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs new file mode 100644 index 000000000000..e8c3b6f03551 --- /dev/null +++ b/dev/src/generate/parser.rs @@ -0,0 +1,520 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; +use anyhow::{anyhow, Context}; +use log::debug; +use std::collections::hash_map; +use std::collections::HashMap; +use std::fs; +use std::fs::read_dir; +use std::str::FromStr; +use syn::{Field, GenericArgument, Item, ItemStruct, PathArguments, Type, TypePath}; + +#[derive(Debug, Clone)] +pub struct Services(HashMap); + +impl IntoIterator for Services { + type Item = (String, Service); + type IntoIter = hash_map::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +/// Service represents a service supported by opendal core, like `s3` and `fs` +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Service { + /// All configurations for this service. + pub config: Vec, +} + +/// Config represents a configuration item for a service. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Config { + /// The name of this config, for example, `access_key_id` and `secret_access_key` + pub name: String, + /// The value type this config. + pub value: ConfigType, + /// If given config is optional or not. + pub optional: bool, + /// The comments for this config. + /// + /// All white spaces and extra new lines will be trimmed. + pub comments: String, +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum ConfigType { + /// Mapping to rust's `bool` + Bool, + /// Mapping to rust's `String` + String, + /// Mapping to rust's `Duration` + Duration, + + /// Mapping to rust's `usize` + Usize, + /// Mapping to rust's `u64` + U64, + /// Mapping to rust's `i64` + I64, + /// Mapping to rust's `u32` + U32, + /// Mapping to rust's `u16` + U16, + + /// Mapping to rust's `Vec` + /// + /// Please note, all vec in config are `,` separated string. + Vec, +} + +impl FromStr for ConfigType { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let ct = match s { + "bool" => ConfigType::Bool, + "String" => ConfigType::String, + "Duration" => ConfigType::Duration, + + "usize" => ConfigType::Usize, + "u64" => ConfigType::U64, + "i64" => ConfigType::I64, + "u32" => ConfigType::U32, + "u16" => ConfigType::U16, + + "Vec" => ConfigType::Vec, + v => return Err(anyhow!("unsupported config type {v:?}")), + }; + + Ok(ct) + } +} + +/// List and parse given path to a `Services` struct. +pub fn parse(path: &str) -> Result { + let mut map = HashMap::default(); + + for dir in read_dir(path)? { + let dir = dir?; + if dir.file_type()?.is_file() { + continue; + } + let path = dir.path().join("config.rs"); + let content = fs::read_to_string(&path)?; + let parser = ServiceParser { + service: dir.file_name().to_string_lossy().to_string(), + path: path.to_string_lossy().to_string(), + content, + }; + let service = parser.parse().context(format!("path: {path:?}"))?; + map.insert(parser.service, service); + } + + Ok(Services(map)) +} + +/// ServiceParser is used to parse a service config file. +pub struct ServiceParser { + service: String, + path: String, + content: String, +} + +/// A typical service config will look like this: +/// +/// ``` +/// #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +/// #[serde(default)] +/// #[non_exhaustive] +/// pub struct S3Config { +/// /// root of this backend. +/// /// +/// /// All operations will happen under this root. +/// /// +/// /// default to `/` if not set. +/// pub root: Option, +/// /// bucket name of this backend. +/// /// +/// /// required. +/// pub bucket: String, +/// /// is bucket versioning enabled for this bucket +/// pub enable_versioning: bool, +/// } +/// ``` +impl ServiceParser { + /// Parse the content of this service. + fn parse(&self) -> Result { + debug!("service {} parse started", self.service); + + let ast = syn::parse_file(&self.content)?; + + let config_struct = ast + .items + .iter() + .find_map(|v| { + if let Item::Struct(v) = v { + if v.ident.to_string().contains("Config") { + return Some(v.clone()); + } + } + None + }) + .ok_or_else(|| anyhow!("there is no Config in {}", &self.path))?; + + let mut config = Vec::with_capacity(config_struct.fields.len()); + for field in config_struct.fields { + let field = Self::parse_field(field)?; + config.push(field); + } + + debug!("service {} parse finished", self.service); + Ok(Service { config }) + } + + /// TODO: Add comment parse support. + fn parse_field(field: Field) -> Result { + let name = field + .ident + .clone() + .ok_or_else(|| anyhow!("field name is missing for {:?}", &field))?; + + let (cfg_type, optional) = match &field.ty { + Type::Path(TypePath { path, .. }) => { + let segment = path + .segments + .last() + .ok_or_else(|| anyhow!("config type must be provided for {field:?}"))?; + + let optional = segment.ident == "Option"; + + let type_name = if optional { + if let PathArguments::AngleBracketed(args) = &segment.arguments { + if let Some(GenericArgument::Type(Type::Path(inner_path))) = + args.args.first() + { + if let Some(inner_segment) = inner_path.path.segments.last() { + inner_segment.ident.to_string() + } else { + unreachable!("Option must have segment") + } + } else { + unreachable!("Option must have GenericArgument") + } + } else { + unreachable!("Option must have angle bracketed arguments") + } + } else { + segment.ident.to_string() + }; + + let typ = type_name.as_str().parse()?; + + (typ, optional) + } + v => return Err(anyhow!("unsupported config type {v:?}")), + }; + + Ok(Config { + name: name.to_string(), + value: cfg_type, + optional, + comments: "".to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use std::path::PathBuf; + + #[test] + fn test_parse_field() { + let cases = vec![ + ( + "pub root: Option", + Config { + name: "root".to_string(), + value: ConfigType::String, + optional: true, + comments: "".to_string(), + }, + ), + ( + "root: String", + Config { + name: "root".to_string(), + value: ConfigType::String, + optional: false, + comments: "".to_string(), + }, + ), + ]; + + for (input, expected) in cases { + let input = format!("struct Test {{ {input} }}"); + let x: ItemStruct = syn::parse_str(&input).unwrap(); + let actual = + ServiceParser::parse_field(x.fields.iter().next().unwrap().clone()).unwrap(); + assert_eq!(actual, expected); + } + } + + #[test] + fn test_parse_service() { + let content = r#" +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use serde::Deserialize; +use serde::Serialize; + +/// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct S3Config { + /// root of this backend. + /// + /// All operations will happen under this root. + /// + /// default to `/` if not set. + pub root: Option, + /// bucket name of this backend. + /// + /// required. + pub bucket: String, + /// is bucket versioning enabled for this bucket + pub enable_versioning: bool, + /// endpoint of this backend. + /// + /// Endpoint must be full uri, e.g. + /// + /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com` + /// - Cloudflare R2: `https://.r2.cloudflarestorage.com` + /// - Aliyun OSS: `https://{region}.aliyuncs.com` + /// - Tencent COS: `https://cos.{region}.myqcloud.com` + /// - Minio: `http://127.0.0.1:9000` + /// + /// If user inputs endpoint without scheme like "s3.amazonaws.com", we + /// will prepend "https://" before it. + /// + /// - If endpoint is set, we will take user's input first. + /// - If not, we will try to load it from environment. + /// - If still not set, default to `https://s3.amazonaws.com`. + pub endpoint: Option, + /// Region represent the signing region of this endpoint. This is required + /// if you are using the default AWS S3 endpoint. + /// + /// If using a custom endpoint, + /// - If region is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub region: Option, + + /// access_key_id of this backend. + /// + /// - If access_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub access_key_id: Option, + /// secret_access_key of this backend. + /// + /// - If secret_access_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub secret_access_key: Option, + /// session_token (aka, security token) of this backend. + /// + /// This token will expire after sometime, it's recommended to set session_token + /// by hand. + pub session_token: Option, + /// role_arn for this backend. + /// + /// If `role_arn` is set, we will use already known config as source + /// credential to assume role with `role_arn`. + pub role_arn: Option, + /// external_id for this backend. + pub external_id: Option, + /// role_session_name for this backend. + pub role_session_name: Option, + /// Disable config load so that opendal will not load config from + /// environment. + /// + /// For examples: + /// + /// - envs like `AWS_ACCESS_KEY_ID` + /// - files like `~/.aws/config` + pub disable_config_load: bool, + /// Disable load credential from ec2 metadata. + /// + /// This option is used to disable the default behavior of opendal + /// to load credential from ec2 metadata, a.k.a, IMDSv2 + pub disable_ec2_metadata: bool, + /// Allow anonymous will allow opendal to send request without signing + /// when credential is not loaded. + pub allow_anonymous: bool, + /// server_side_encryption for this backend. + /// + /// Available values: `AES256`, `aws:kms`. + pub server_side_encryption: Option, + /// server_side_encryption_aws_kms_key_id for this backend + /// + /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` + /// is not set, S3 will use aws managed kms key to encrypt data. + /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` + /// is a valid kms key id, S3 will use the provided kms key to encrypt data. + /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be + /// returned. + /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id` + /// is a noop. + pub server_side_encryption_aws_kms_key_id: Option, + /// server_side_encryption_customer_algorithm for this backend. + /// + /// Available values: `AES256`. + pub server_side_encryption_customer_algorithm: Option, + /// server_side_encryption_customer_key for this backend. + /// + /// # Value + /// + /// base64 encoded key that matches algorithm specified in + /// `server_side_encryption_customer_algorithm`. + pub server_side_encryption_customer_key: Option, + /// Set server_side_encryption_customer_key_md5 for this backend. + /// + /// # Value + /// + /// MD5 digest of key specified in `server_side_encryption_customer_key`. + pub server_side_encryption_customer_key_md5: Option, + /// default storage_class for this backend. + /// + /// Available values: + /// - `DEEP_ARCHIVE` + /// - `GLACIER` + /// - `GLACIER_IR` + /// - `INTELLIGENT_TIERING` + /// - `ONEZONE_IA` + /// - `OUTPOSTS` + /// - `REDUCED_REDUNDANCY` + /// - `STANDARD` + /// - `STANDARD_IA` + /// + /// S3 compatible services don't support all of them + pub default_storage_class: Option, + /// Enable virtual host style so that opendal will send API requests + /// in virtual host style instead of path style. + /// + /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name` + /// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` + pub enable_virtual_host_style: bool, + /// Set maximum batch operations of this backend. + /// + /// Some compatible services have a limit on the number of operations in a batch request. + /// For example, R2 could return `Internal Error` while batch delete 1000 files. + /// + /// Please tune this value based on services' document. + #[deprecated( + since = "0.52.0", + note = "Please use `delete_max_size` instead of `batch_max_operations`" + )] + pub batch_max_operations: Option, + /// Set the maximum delete size of this backend. + /// + /// Some compatible services have a limit on the number of operations in a batch request. + /// For example, R2 could return `Internal Error` while batch delete 1000 files. + /// + /// Please tune this value based on services' document. + pub delete_max_size: Option, + /// Disable stat with override so that opendal will not send stat request with override queries. + /// + /// For example, R2 doesn't support stat with `response_content_type` query. + pub disable_stat_with_override: bool, + /// Checksum Algorithm to use when sending checksums in HTTP headers. + /// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example. + /// + /// Available options: + /// - "crc32c" + pub checksum_algorithm: Option, + /// Disable write with if match so that opendal will not send write request with if match headers. + /// + /// For example, Ceph RADOS S3 doesn't support write with if match. + pub disable_write_with_if_match: bool, +} + +impl Debug for S3Config { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("S3Config"); + + d.field("root", &self.root) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("region", &self.region); + + d.finish_non_exhaustive() + } +} +"#; + let parser = ServiceParser { + service: "s3".to_string(), + path: "test".to_string(), + content: content.to_string(), + }; + + let service = parser.parse().unwrap(); + assert_eq!(service.config.len(), 26); + assert_eq!( + service.config[25], + Config { + name: "disable_write_with_if_match".to_string(), + value: ConfigType::Bool, + optional: false, + comments: "".to_string(), + }, + ); + } + + #[test] + fn test_parse() { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let path = manifest_dir + .join("../core/src/services") + .canonicalize() + .unwrap(); + + // Parse should just pass. + let _ = parse(&path.to_string_lossy()).unwrap(); + } +} diff --git a/dev/src/main.rs b/dev/src/main.rs index 2f4454c78b56..a56430abf8a4 100644 --- a/dev/src/main.rs +++ b/dev/src/main.rs @@ -15,6 +15,37 @@ // specific language governing permissions and limitations // under the License. -fn main() { - println!("hello, world!") +mod generate; + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command(version, about, long_about = None)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Generate all services configs for opendal. + Generate { + #[arg(short, long)] + language: String, + }, +} + +fn main() -> Result<()> { + env_logger::init(); + + let cli = Cli::parse(); + + match cli.command { + Commands::Generate { language } => { + generate::run(&language)?; + } + } + + Ok(()) }