From 6becac4eaa5a23c7542ba2e279ac60d958071028 Mon Sep 17 00:00:00 2001 From: suyanhanx Date: Mon, 22 Jan 2024 23:08:54 +0800 Subject: [PATCH 1/8] feat(services/koofr): set test for koofr Signed-off-by: suyanhanx --- .env.example | 2 +- .github/services/koofr/koofr/action.yml | 38 +++++++++++++++++++++++++ core/src/services/koofr/backend.rs | 27 ++++++++++++++---- core/tests/behavior/async_copy.rs | 5 ++++ 4 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 .github/services/koofr/koofr/action.yml diff --git a/.env.example b/.env.example index ec9a1292b242..d554ceaa0532 100644 --- a/.env.example +++ b/.env.example @@ -208,7 +208,7 @@ OPENDAL_YANDEX_DISK_ACCESS_TOKEN= OPENDAL_KOOFR_DISK_ROOT=/path/to/dir OPENDAL_KOOFR_ENDPOINT= OPENDAL_KOOFR_EMAIL= -OPENDAL_KOOFR_PASSWORD=> +OPENDAL_KOOFR_PASSWORD= # icloud OPENDAL_ICLOUD_ROOT=/ OPENDAL_ICLOUD_APPLE_ID= diff --git a/.github/services/koofr/koofr/action.yml b/.github/services/koofr/koofr/action.yml new file mode 100644 index 000000000000..5c1b51c0580c --- /dev/null +++ b/.github/services/koofr/koofr/action.yml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: koofr +description: 'Behavior test for Koofr. This service is sponsored by @suyanhanx.' + +runs: + using: "composite" + steps: + - name: Setup + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OPENDAL_KOOFR_ROOT: op://services/koofr/root + OPENDAL_KOOFR_ENDPOINT: "https://app.koofr.net" + OPENDAL_KOOFR_EMAIL: op://services/koofr/email + OPENDAL_KOOFR_PASSWORD: op://services/koofr/password + - name: Setup + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + RUST_TEST_THREADS=1 + EOF diff --git a/core/src/services/koofr/backend.rs b/core/src/services/koofr/backend.rs index 6e19d9a7e78e..f0f7fdfe814d 100644 --- a/core/src/services/koofr/backend.rs +++ b/core/src/services/koofr/backend.rs @@ -50,7 +50,7 @@ pub struct KoofrConfig { pub endpoint: String, /// Koofr email. pub email: String, - /// password of this backend. + /// password of this backend. (Must be the application password) pub password: Option, } @@ -115,10 +115,16 @@ impl KoofrBuilder { self } - /// Koofr app password. + /// Koofr application password. /// /// Go to https://app.koofr.net/app/admin/preferences/password. - /// Click "Generate Password" button to generate a new password. + /// Click "Generate Password" button to generate a new application password. + /// + /// # Notes + /// + /// This is not user's Koofr account password. + /// Please use the application password instead. + /// Please also remind users of this. pub fn password(&mut self, password: &str) -> &mut Self { self.config.password = if password.is_empty() { None @@ -172,8 +178,8 @@ impl Builder for KoofrBuilder { let root = normalize_root(&self.config.root.clone().unwrap_or_default()); debug!("backend use root {}", &root); + println!("backend use root {}", &root); - // Handle endpoint. if self.config.endpoint.is_empty() { return Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_operation("Builder::build") @@ -182,7 +188,6 @@ impl Builder for KoofrBuilder { debug!("backend use endpoint {}", &self.config.endpoint); - // Handle email. if self.config.email.is_empty() { return Err(Error::new(ErrorKind::ConfigInvalid, "email is empty") .with_operation("Builder::build") @@ -253,7 +258,9 @@ impl Accessor for KoofrBackend { write_can_empty: true, delete: true, + rename: true, + copy: true, list: true, @@ -348,6 +355,11 @@ impl Accessor for KoofrBackend { async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result { self.core.ensure_dir_exists(to).await?; + + if from == to { + return Ok(RpCopy::default()); + } + let resp = self.core.remove(to).await?; let status = resp.status(); @@ -372,6 +384,11 @@ impl Accessor for KoofrBackend { async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result { self.core.ensure_dir_exists(to).await?; + + if from == to { + return Ok(RpRename::default()); + } + let resp = self.core.remove(to).await?; let status = resp.status(); diff --git a/core/tests/behavior/async_copy.rs b/core/tests/behavior/async_copy.rs index cee9494ec8a4..08ac6a642bf7 100644 --- a/core/tests/behavior/async_copy.rs +++ b/core/tests/behavior/async_copy.rs @@ -63,6 +63,11 @@ pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> { /// Copy a file with non ascii name and test contents. pub async fn test_copy_file_with_non_ascii_name(op: Operator) -> Result<()> { + // Koofr does not support non-ascii name.(2024-01-22) + if op.info().scheme() == Scheme::Koofr { + return Ok(()); + } + let source_path = "🐂🍺中文.docx"; let target_path = "😈🐅Français.docx"; let (source_content, _) = gen_bytes(op.info().full_capability()); From c6a4aed99878cdd703ce5528ccb124b96a25550f Mon Sep 17 00:00:00 2001 From: suyanhanx Date: Mon, 22 Jan 2024 23:10:35 +0800 Subject: [PATCH 2/8] fix env var name Signed-off-by: suyanhanx --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index d554ceaa0532..bcb6952d97de 100644 --- a/.env.example +++ b/.env.example @@ -205,7 +205,7 @@ OPENDAL_PCLOUD_PASSWORD= OPENDAL_YANDEX_DISK_ROOT=/path/to/dir OPENDAL_YANDEX_DISK_ACCESS_TOKEN= # koofr -OPENDAL_KOOFR_DISK_ROOT=/path/to/dir +OPENDAL_KOOFR_ROOT=/path/to/dir OPENDAL_KOOFR_ENDPOINT= OPENDAL_KOOFR_EMAIL= OPENDAL_KOOFR_PASSWORD= From 12bd406fba4dbfff83f77ea469651a3682af825e Mon Sep 17 00:00:00 2001 From: suyanhanx Date: Mon, 22 Jan 2024 23:32:44 +0800 Subject: [PATCH 3/8] fix feature flag in bindings Signed-off-by: suyanhanx --- bindings/java/Cargo.toml | 2 ++ bindings/nodejs/Cargo.toml | 18 ++++++++++-------- bindings/python/Cargo.toml | 4 +++- core/Cargo.toml | 2 +- core/src/services/koofr/backend.rs | 1 - core/tests/behavior/async_copy.rs | 2 +- 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 9787a7325f9b..12bff79f6435 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -72,6 +72,7 @@ services-all = [ "services-onedrive", "services-persy", "services-postgresql", + "services-koofr", "services-mysql", "services-redb", "services-redis", @@ -125,6 +126,7 @@ services-gridfs = ["opendal/services-gridfs"] services-hdfs = ["opendal/services-hdfs"] services-huggingface = ["opendal/services-huggingface"] services-ipfs = ["opendal/services-ipfs"] +services-koofr = ["opendal/services-koofr"] services-libsql = ["opendal/services-libsql"] services-memcached = ["opendal/services-memcached"] services-mini-moka = ["opendal/services-mini-moka"] diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml index 51da25bd668a..937a227ce07e 100644 --- a/bindings/nodejs/Cargo.toml +++ b/bindings/nodejs/Cargo.toml @@ -49,7 +49,9 @@ default = [ services-all = [ "default", + "services-alluxio", "services-azfile", + "services-b2", "services-cacache", "services-dashmap", "services-dropbox", @@ -59,13 +61,17 @@ services-all = [ # FIXME waiting for this issue close https://github.com/async-rs/async-tls/issues/55 # "services-ftp", "services-gdrive", + "services-gridfs", # FIXME how to support HDFS services in bindings? # "services-hdfs", "services-huggingface", "services-ipfs", + "services-koofr", + "services-libsql", "services-memcached", "services-mini-moka", "services-moka", + "services-mongodb", "services-onedrive", "services-persy", "services-postgresql", @@ -73,20 +79,15 @@ services-all = [ "services-redb", "services-redis", "services-rocksdb", + "services-seafile", "services-sled", + "services-sqlite", "services-supabase", "services-swift", "services-tikv", + "services-upyun", "services-vercel-artifacts", "services-wasabi", - "services-mongodb", - "services-gridfs", - "services-sqlite", - "services-libsql", - "services-alluxio", - "services-b2", - "services-seafile", - "services-upyun", ] # Default services provided by opendal. @@ -120,6 +121,7 @@ services-gridfs = ["opendal/services-gridfs"] services-hdfs = ["opendal/services-hdfs"] services-huggingface = ["opendal/services-huggingface"] services-ipfs = ["opendal/services-ipfs"] +services-koofr = ["opendal/services-koofr"] services-libsql = ["opendal/services-libsql"] services-memcached = ["opendal/services-memcached"] services-mini-moka = ["opendal/services-mini-moka"] diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index fe00e54af13a..d065e1b39b44 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -89,6 +89,7 @@ services-all = [ "services-b2", "services-seafile", "services-upyun", + "services-koofr", ] # Default services provided by opendal. @@ -122,6 +123,7 @@ services-gridfs = ["opendal/services-gridfs"] services-hdfs = ["opendal/services-hdfs"] services-huggingface = ["opendal/services-huggingface"] services-ipfs = ["opendal/services-ipfs"] +services-koofr = ["opendal/services-koofr"] services-libsql = ["opendal/services-libsql"] services-memcached = ["opendal/services-memcached"] services-mini-moka = ["opendal/services-mini-moka"] @@ -161,4 +163,4 @@ features = [ # Depend on "openssh" which depends on "tokio-pipe" that is unavailable on Windows. "services-sftp", ] -workspace = true \ No newline at end of file +workspace = true diff --git a/core/Cargo.toml b/core/Cargo.toml index 276aa7f2f179..eeec171d907f 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -197,7 +197,7 @@ services-upyun = ["dep:hmac", "dep:sha1"] services-vercel-artifacts = [] # Deprecated # wasabi services support has been removed. -# We will remove this feature in next version. +# We will remove this feature in the next version. services-wasabi = [] services-webdav = [] services-webhdfs = [] diff --git a/core/src/services/koofr/backend.rs b/core/src/services/koofr/backend.rs index f0f7fdfe814d..a7f8fc35901c 100644 --- a/core/src/services/koofr/backend.rs +++ b/core/src/services/koofr/backend.rs @@ -178,7 +178,6 @@ impl Builder for KoofrBuilder { let root = normalize_root(&self.config.root.clone().unwrap_or_default()); debug!("backend use root {}", &root); - println!("backend use root {}", &root); if self.config.endpoint.is_empty() { return Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") diff --git a/core/tests/behavior/async_copy.rs b/core/tests/behavior/async_copy.rs index 08ac6a642bf7..82dc4637ed8e 100644 --- a/core/tests/behavior/async_copy.rs +++ b/core/tests/behavior/async_copy.rs @@ -63,7 +63,7 @@ pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> { /// Copy a file with non ascii name and test contents. pub async fn test_copy_file_with_non_ascii_name(op: Operator) -> Result<()> { - // Koofr does not support non-ascii name.(2024-01-22) + // Koofr does not support non-ascii name.(https://github.com/apache/opendal/issues/4051) if op.info().scheme() == Scheme::Koofr { return Ok(()); } From 77dba32e90d034ce7523ba3ced0591fdff6de8c2 Mon Sep 17 00:00:00 2001 From: suyanhanx Date: Mon, 22 Jan 2024 23:43:29 +0800 Subject: [PATCH 4/8] fix setup config Signed-off-by: suyanhanx --- .github/services/koofr/koofr/action.yml | 2 +- .../java/org/apache/opendal/test/behavior/AsyncCopyTest.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/services/koofr/koofr/action.yml b/.github/services/koofr/koofr/action.yml index 5c1b51c0580c..204d6097f2c8 100644 --- a/.github/services/koofr/koofr/action.yml +++ b/.github/services/koofr/koofr/action.yml @@ -27,7 +27,7 @@ runs: export-env: true env: OPENDAL_KOOFR_ROOT: op://services/koofr/root - OPENDAL_KOOFR_ENDPOINT: "https://app.koofr.net" + OPENDAL_KOOFR_ENDPOINT: op://services/koofr/endpoint OPENDAL_KOOFR_EMAIL: op://services/koofr/email OPENDAL_KOOFR_PASSWORD: op://services/koofr/password - name: Setup diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java index b292e30c42ff..065af82c8ec2 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java @@ -64,6 +64,11 @@ public void testCopyFileWithAsciiName() { */ @Test public void testCopyFileWithNonAsciiName() { + // Services-koofr doesn't support non-ascii name (https://github.com/apache/opendal/issues/4051) + if (op().info.scheme == "koofr") { + return; + } + final String sourcePath = "🐂🍺中文.docx"; final String targetPath = "😈🐅Français.docx"; final byte[] content = generateBytes(); From 986897d2142bb693201bb926466a3a9f360e916c Mon Sep 17 00:00:00 2001 From: suyanhanx Date: Tue, 23 Jan 2024 23:14:53 +0800 Subject: [PATCH 5/8] fix skip condition Signed-off-by: suyanhanx --- .../java/org/apache/opendal/test/behavior/AsyncCopyTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java index 065af82c8ec2..477d5d45c0b8 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java @@ -65,9 +65,7 @@ public void testCopyFileWithAsciiName() { @Test public void testCopyFileWithNonAsciiName() { // Services-koofr doesn't support non-ascii name (https://github.com/apache/opendal/issues/4051) - if (op().info.scheme == "koofr") { - return; - } + assumeTrue(op().info.scheme != "koofr", "Services-koofr doesn't support non-ascii name"); final String sourcePath = "🐂🍺中文.docx"; final String targetPath = "😈🐅Français.docx"; From 4ba6e7b2d9ae3a6f6e34533fdbbc9d0383fcd227 Mon Sep 17 00:00:00 2001 From: suyanhanx Date: Tue, 23 Jan 2024 23:42:58 +0800 Subject: [PATCH 6/8] fix skip condition Signed-off-by: suyanhanx --- .../java/org/apache/opendal/test/behavior/AsyncCopyTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java index 477d5d45c0b8..5c6203fde62a 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java @@ -22,6 +22,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.util.Objects; import java.util.UUID; import org.apache.opendal.Capability; import org.apache.opendal.OpenDALException; @@ -65,7 +67,7 @@ public void testCopyFileWithAsciiName() { @Test public void testCopyFileWithNonAsciiName() { // Services-koofr doesn't support non-ascii name (https://github.com/apache/opendal/issues/4051) - assumeTrue(op().info.scheme != "koofr", "Services-koofr doesn't support non-ascii name"); + assumeTrue(!Objects.equals(op().info.scheme, "koofr"), "Services-koofr doesn't support non-ascii name"); final String sourcePath = "🐂🍺中文.docx"; final String targetPath = "😈🐅Français.docx"; From c407eb6595e9771e31a6e4ba472c481108d0e942 Mon Sep 17 00:00:00 2001 From: suyanhanx Date: Tue, 23 Jan 2024 23:50:55 +0800 Subject: [PATCH 7/8] fix format Signed-off-by: suyanhanx --- .../java/org/apache/opendal/test/behavior/AsyncCopyTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java index 5c6203fde62a..5944c02ea94b 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java @@ -22,7 +22,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; - import java.util.Objects; import java.util.UUID; import org.apache.opendal.Capability; From b00bae552169ea0b692a2d21dd1b78f6168fc765 Mon Sep 17 00:00:00 2001 From: suyanhanx Date: Wed, 24 Jan 2024 13:07:09 +0800 Subject: [PATCH 8/8] remove test-threads limit Signed-off-by: suyanhanx --- .github/services/koofr/koofr/action.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/services/koofr/koofr/action.yml b/.github/services/koofr/koofr/action.yml index 204d6097f2c8..20dd0bcd91b6 100644 --- a/.github/services/koofr/koofr/action.yml +++ b/.github/services/koofr/koofr/action.yml @@ -30,9 +30,3 @@ runs: OPENDAL_KOOFR_ENDPOINT: op://services/koofr/endpoint OPENDAL_KOOFR_EMAIL: op://services/koofr/email OPENDAL_KOOFR_PASSWORD: op://services/koofr/password - - name: Setup - shell: bash - run: | - cat << EOF >> $GITHUB_ENV - RUST_TEST_THREADS=1 - EOF