From 359771a664e8ba9620d45af753f3b7cc1a25c94b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 28 Sep 2023 13:14:04 +0800 Subject: [PATCH] feat: Add write_can_empty in Capability and related tests (#3200) * feat: Add write_can_empty in Capability and related tests Signed-off-by: Xuanwo * Fix gcs Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- core/src/raw/adapters/kv/backend.rs | 1 + core/src/raw/adapters/typed_kv/backend.rs | 1 + .../raw/oio/write/multipart_upload_write.rs | 8 +++++++- core/src/raw/oio/write/range_write.rs | 8 +++++++- core/src/services/azblob/backend.rs | 1 + core/src/services/cos/backend.rs | 1 + core/src/services/fs/backend.rs | 1 + core/src/services/gcs/backend.rs | 1 + core/src/services/obs/backend.rs | 1 + core/src/services/oss/backend.rs | 1 + core/src/services/s3/backend.rs | 1 + core/src/services/webdav/backend.rs | 1 + core/src/types/capability.rs | 2 ++ core/tests/behavior/write.rs | 18 ++++++++++++++++++ 14 files changed, 44 insertions(+), 2 deletions(-) diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs index 211e9c9ef4ae..94db4de8dc5e 100644 --- a/core/src/raw/adapters/kv/backend.rs +++ b/core/src/raw/adapters/kv/backend.rs @@ -84,6 +84,7 @@ impl Accessor for Backend { } if cap.write { + cap.write_can_empty = true; cap.create_dir = true; cap.delete = true; } diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs index 6d515b01a9e1..d5313b8e0097 100644 --- a/core/src/raw/adapters/typed_kv/backend.rs +++ b/core/src/raw/adapters/typed_kv/backend.rs @@ -84,6 +84,7 @@ impl Accessor for Backend { if kv_cap.set { cap.write = true; + cap.write_can_empty = true; cap.create_dir = true; } diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs index 4cf597e599a4..f95cf64f1ed3 100644 --- a/core/src/raw/oio/write/multipart_upload_write.rs +++ b/core/src/raw/oio/write/multipart_upload_write.rs @@ -258,7 +258,13 @@ where (w, res) })); } - None => return Poll::Ready(Ok(())), + None => { + // Call write_once if there is no data in cache and no upload_id. + self.state = State::Close(Box::pin(async move { + let res = w.write_once(0, AsyncBody::Empty).await; + (w, res) + })); + } }, } } diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs index 6dd9c23b2050..3872d1cd591a 100644 --- a/core/src/raw/oio/write/range_write.rs +++ b/core/src/raw/oio/write/range_write.rs @@ -223,7 +223,13 @@ impl oio::Write for RangeWriter { (w, res) })); } - None => return Poll::Ready(Ok(())), + None => { + // Call write_once if there is no data in buffer and no location. + self.state = State::Complete(Box::pin(async move { + let res = w.write_once(0, AsyncBody::Empty).await; + (w, res) + })); + } }, } } diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index 627d6fb02c49..de6d5e3e943a 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -529,6 +529,7 @@ impl Accessor for AzblobBackend { read_with_override_content_disposition: true, write: true, + write_can_empty: true, write_can_append: true, write_with_cache_control: true, write_with_content_type: true, diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index be19e08717d8..d376abcb7ab6 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -265,6 +265,7 @@ impl Accessor for CosBackend { read_with_if_none_match: true, write: true, + write_can_empty: true, write_can_append: true, write_can_multi: true, write_with_content_type: true, diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index a225958d2cea..40ada10f2194 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -267,6 +267,7 @@ impl Accessor for FsBackend { read_with_range: true, write: true, + write_can_empty: true, write_can_append: true, write_can_multi: true, create_dir: true, diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index 57ce7cfbd515..e65fb155de91 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -334,6 +334,7 @@ impl Accessor for GcsBackend { read_with_if_none_match: true, write: true, + write_can_empty: true, write_can_multi: true, write_with_content_type: true, // The buffer size should be a multiple of 256 KiB (256 x 1024 bytes), unless it's the last chunk that completes the upload. diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index cff7c254f4e1..aa5a86864c63 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -272,6 +272,7 @@ impl Accessor for ObsBackend { read_with_if_none_match: true, write: true, + write_can_empty: true, write_can_append: true, write_can_multi: true, write_with_content_type: true, diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index cc4dddbaefd1..4f90ea9579a2 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -398,6 +398,7 @@ impl Accessor for OssBackend { read_with_if_none_match: true, write: true, + write_can_empty: true, write_can_append: true, write_can_multi: true, write_with_cache_control: true, diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 82e420421df8..06010685100d 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -913,6 +913,7 @@ impl Accessor for S3Backend { read_with_override_content_type: true, write: true, + write_can_empty: true, write_can_multi: true, write_with_cache_control: true, write_with_content_type: true, diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index 549a6b34ff34..4c4cf6b834a2 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -238,6 +238,7 @@ impl Accessor for WebdavBackend { read_with_range: true, write: true, + write_can_empty: true, create_dir: true, delete: true, diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs index abe7eb9c44cd..2767c29e49dd 100644 --- a/core/src/types/capability.rs +++ b/core/src/types/capability.rs @@ -76,6 +76,8 @@ pub struct Capability { pub write: bool, /// If operator supports write can be called in multi times. pub write_can_multi: bool, + /// If operator supports write with empty content. + pub write_can_empty: bool, /// If operator supports write by append. pub write_can_append: bool, /// If operator supports write with content type. diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs index 442aff1a53f8..675d0c7a37e9 100644 --- a/core/tests/behavior/write.rs +++ b/core/tests/behavior/write.rs @@ -48,6 +48,7 @@ pub fn behavior_write_tests(op: &Operator) -> Vec { test_create_dir, test_create_dir_existing, test_write_only, + test_write_with_empty_content, test_write_with_dir_path, test_write_with_special_chars, test_write_with_cache_control, @@ -136,6 +137,23 @@ pub async fn test_write_only(op: Operator) -> Result<()> { Ok(()) } +/// Write a file with empty content. +pub async fn test_write_with_empty_content(op: Operator) -> Result<()> { + if !op.info().full_capability().write_can_empty { + return Ok(()); + } + + let path = uuid::Uuid::new_v4().to_string(); + + op.write(&path, vec![]).await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), 0); + + op.delete(&path).await.expect("delete must succeed"); + Ok(()) +} + /// Write file with dir path should return an error pub async fn test_write_with_dir_path(op: Operator) -> Result<()> { let path = format!("{}/", uuid::Uuid::new_v4());