From 3d0bc6b3c7cbe377a39894355f5be4950e42c9d0 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 19 May 2020 16:03:54 +0800 Subject: [PATCH 1/2] tests: Add integration test for uss Signed-off-by: Xuanwo --- .drone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index d6d48283d..3b4d3a8b1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,6 +24,8 @@ steps: from_secret: STORAGE_TEST_SERVICE_QINGSTOR STORAGE_TEST_SERVICE_S3: from_secret: STORAGE_TEST_SERVICE_S3 + STORAGE_TEST_SERVICE_USS: + from_secret: STORAGE_TEST_SERVICE_USS commands: - cd tests && go test -race -v ./... @@ -33,6 +35,6 @@ volumes: path: /var/lib/cache/drone/storage/go --- kind: signature -hmac: b3de0e11b75bf4c4be478525915f2ac8c9cf942bbfddc23165bf7c49a9cba1b8 +hmac: 4e8cfadbaebc10dee4638873bb1045687019da9ed1b3f8cb155831b68e6310da ... From fd39aac5d5b4f1341de631ad02cecd9fd0b30c00 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 19 May 2020 16:52:05 +0800 Subject: [PATCH 2/2] services/uss: Disable async delete for unexpected behavior Signed-off-by: Xuanwo --- .drone.yml | 4 +--- services/uss/storager.go | 36 +++++++++++++++++------------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/.drone.yml b/.drone.yml index 3b4d3a8b1..d6d48283d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,8 +24,6 @@ steps: from_secret: STORAGE_TEST_SERVICE_QINGSTOR STORAGE_TEST_SERVICE_S3: from_secret: STORAGE_TEST_SERVICE_S3 - STORAGE_TEST_SERVICE_USS: - from_secret: STORAGE_TEST_SERVICE_USS commands: - cd tests && go test -race -v ./... @@ -35,6 +33,6 @@ volumes: path: /var/lib/cache/drone/storage/go --- kind: signature -hmac: 4e8cfadbaebc10dee4638873bb1045687019da9ed1b3f8cb155831b68e6310da +hmac: b3de0e11b75bf4c4be478525915f2ac8c9cf942bbfddc23165bf7c49a9cba1b8 ... diff --git a/services/uss/storager.go b/services/uss/storager.go index 77abc35d4..206bde6b2 100644 --- a/services/uss/storager.go +++ b/services/uss/storager.go @@ -55,22 +55,21 @@ func (s *Storage) ListDir(path string, pairs ...*types.Pair) (err error) { return err } - maxListLevel := 1 - rp := s.getAbsPath(path) // USS SDK will close this channel in List ch := make(chan *upyun.FileInfo, 200) go func() { - errlock.Lock() - defer errlock.Unlock() - - err = s.bucket.List(&upyun.GetObjectsConfig{ + xerr := s.bucket.List(&upyun.GetObjectsConfig{ Path: rp, ObjectsChan: ch, - MaxListLevel: maxListLevel, + MaxListLevel: 1, }) + + errlock.Lock() + defer errlock.Unlock() + err = xerr }() for v := range ch { @@ -117,22 +116,21 @@ func (s *Storage) ListPrefix(prefix string, pairs ...*types.Pair) (err error) { return err } - maxListLevel := -1 - rp := s.getAbsPath(prefix) // USS SDK will close this channel in List ch := make(chan *upyun.FileInfo, 200) go func() { - errlock.Lock() - defer errlock.Unlock() - - err = s.bucket.List(&upyun.GetObjectsConfig{ + xerr := s.bucket.List(&upyun.GetObjectsConfig{ Path: rp, ObjectsChan: ch, - MaxListLevel: maxListLevel, + MaxListLevel: -1, }) + + errlock.Lock() + defer errlock.Unlock() + err = xerr }() for v := range ch { @@ -245,6 +243,11 @@ func (s *Storage) Stat(path string, pairs ...*types.Pair) (o *types.Object, err } // Delete implements Storager.Delete +// +// USS requires a short time between PUT and DELETE, or we will get this error: +// DELETE 429 {"msg":"concurrent put or delete","code":42900007,"id":"xxx"} +// +// Due to this problem, uss can't pass the storager integration tests. func (s *Storage) Delete(path string, pairs ...*types.Pair) (err error) { defer func() { err = s.formatError(services.OpDelete, err, path) @@ -254,11 +257,6 @@ func (s *Storage) Delete(path string, pairs ...*types.Pair) (err error) { err = s.bucket.Delete(&upyun.DeleteObjectConfig{ Path: rp, - // USS requires a short time between PUT and DELETE, or we will get this error: - // DELETE 429 {"msg":"concurrent put or delete","code":42900007,"id":"xxx"} - // - // In order to pass the integration tests, use async delete instead - Async: true, }) if err != nil { return err