feat: Implement CreateLink and setup linker test in go-service-qingst…

…or (#81)

* feat: Implement CreateLink and setup linker test in go-service-qingstor

* fix: Resolve null pointer problem when undefined

* Change the return value when virtual link is not enabled and set the custom metadata name to a constant

* refactor: Refactored createLink

* Delete go.mod replace

* make build

* Modified custom metadata name

* fix: Fixed append test failures.

go.mod

@@ -4,12 +4,12 @@ go 1.14
 
 require (
 	bou.ke/monkey v1.0.2
-	github.com/beyondstorage/go-endpoint v1.0.1
-	github.com/beyondstorage/go-integration-test/v4 v4.1.1
-	github.com/beyondstorage/go-storage/v4 v4.4.0
+	github.com/beyondstorage/go-endpoint v1.1.0
+	github.com/beyondstorage/go-integration-test/v4 v4.3.0
+	github.com/beyondstorage/go-storage/v4 v4.4.1-0.20210817085851-348ecf741fc1
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14
-	github.com/qingstor/qingstor-sdk-go/v4 v4.3.0
+	github.com/qingstor/qingstor-sdk-go/v4 v4.3.1-0.20210819104240-5afc7bc0d697
 	github.com/stretchr/testify v1.7.0
 )

go.sum

@@ -4,14 +4,13 @@ github.com/Xuanwo/go-bufferpool v0.0.0-20200622083641-bc954721ce54 h1:wA7f87ODtF
 github.com/Xuanwo/go-bufferpool v0.0.0-20200622083641-bc954721ce54/go.mod h1:Mle++9GGouhOwGj52i9PJLNAPmW2nb8PWBP7JJzNCzk=
 github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY=
 github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8=
-github.com/beyondstorage/go-endpoint v1.0.1 h1:F8x2dGLMu9je6g7zPbKoxCXDlug97K26SeCx7KEHgyg=
-github.com/beyondstorage/go-endpoint v1.0.1/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM=
-github.com/beyondstorage/go-integration-test/v4 v4.1.1 h1:9bSXKbr6hLb4+ZsmAhWE32fvqhyrpub4U4qgBGeth4A=
-github.com/beyondstorage/go-integration-test/v4 v4.1.1/go.mod h1:ihtCaOJvaHGE0v+IhY6ZUF5NU1IND6xmdrJI9Lq/jhc=
-github.com/beyondstorage/go-storage/v4 v4.2.0/go.mod h1:rUNzOXcikYk5w0ewvNsKbztg7ndQDyDvjDuP0bznSLU=
-github.com/beyondstorage/go-storage/v4 v4.4.0 h1:sWURraKFjNR4qpwthr45cAGOIx6EOLrrJcz6su4Je30=
+github.com/beyondstorage/go-endpoint v1.1.0 h1:cpjmQdrAMyaLoT161NIFU/eXcsuMI3xViycid5/mBZg=
+github.com/beyondstorage/go-endpoint v1.1.0/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM=
+github.com/beyondstorage/go-integration-test/v4 v4.3.0 h1:WZ95f78RKlHpvft8zHcMaoa2aaTF/jzlzINhMD0EMHY=
+github.com/beyondstorage/go-integration-test/v4 v4.3.0/go.mod h1:HKgzemQZpxoHBL49JYEUnLTb5eteUhzcvmmPL7EDT/Y=
 github.com/beyondstorage/go-storage/v4 v4.4.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw=
-github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w=
+github.com/beyondstorage/go-storage/v4 v4.4.1-0.20210817085851-348ecf741fc1 h1:JCnxrnTXfhejRexyAQ0cd3TYrjsx1ogA/T/RC8iPYz4=
+github.com/beyondstorage/go-storage/v4 v4.4.1-0.20210817085851-348ecf741fc1/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw=
 github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY=
 github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=
 github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ=
@@ -21,11 +20,9 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -48,8 +45,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/qingstor/log v0.0.0-20200804082313-615256cccabc h1:633tgwvkk8O26HZD/upVtDzWcrMoCiffZAVDtEtFW9E=
 github.com/qingstor/log v0.0.0-20200804082313-615256cccabc/go.mod h1:XfNhERXtkg/S9MWaMOk/m7mrmj4vcWDvhw17hmH4JGQ=
-github.com/qingstor/qingstor-sdk-go/v4 v4.3.0 h1:+qcVqBiuGUsTPJZCHc5QK81NgfIMLjoIHXovdIRHKvk=
-github.com/qingstor/qingstor-sdk-go/v4 v4.3.0/go.mod h1:GT0QY+LwqnOQhsRkbxUds8cTYR4inZpgOQEkCKYRiAs=
+github.com/qingstor/qingstor-sdk-go/v4 v4.3.1-0.20210819104240-5afc7bc0d697 h1:6AsznCSi0t+H5+JfCQl7IubJuWuPZvpVGjoPaBbc/Hk=
+github.com/qingstor/qingstor-sdk-go/v4 v4.3.1-0.20210819104240-5afc7bc0d697/go.mod h1:GT0QY+LwqnOQhsRkbxUds8cTYR4inZpgOQEkCKYRiAs=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
@@ -69,7 +66,6 @@ golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgm
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

service.toml

@@ -17,8 +17,8 @@ optional = ["location"]
 optional = ["location"]
 
 [namespace.storage]
-features = ["virtual_dir"]
-implement = ["appender", "copier", "direr", "fetcher", "mover", "multiparter", "reacher"]
+features = ["virtual_dir", "virtual_link"]
+implement = ["appender", "copier", "direr", "fetcher", "mover", "multiparter", "reacher", "linker"]
 
 [namespace.storage.new]
 required = ["name"]

storage.go

@@ -183,6 +183,42 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre
 	return
 }
 
+// metadataLinkTargetHeader is the name of the user-defined metadata name used to store the target.
+const metadataLinkTargetHeader = "x-qs-meta-bs-link-target"
+
+func (s *Storage) createLink(ctx context.Context, path string, target string, opt pairStorageCreateLink) (o *Object, err error) {
+	rt := s.getAbsPath(target)
+	rp := s.getAbsPath(path)
+
+	input := &service.PutObjectInput{
+		// As qingstor does not support symlink, we can only use user-defined metadata to simulate it.
+		// ref: https://github.com/beyondstorage/go-service-qingstor/blob/master/rfcs/79-add-virtual-link-support.md
+		XQSMetaData: &map[string]string{
+			metadataLinkTargetHeader: rt,
+		},
+	}
+
+	_, err = s.bucket.PutObjectWithContext(ctx, rp, input)
+	if err != nil {
+		return nil, err
+	}
+
+	o = s.newObject(true)
+	o.ID = rp
+	o.Path = path
+
+	if !s.features.VirtualLink {
+		// The virtual link is not enabled, so we set the object mode to `ModeRead`.
+		o.Mode |= ModeRead
+	} else {
+		// qingstor does not have an absolute path, so when we call `getAbsPath`, it will remove the prefix `/`.
+		// To ensure that the path matches the one the user gets, we should re-add `/` here.
+		o.SetLinkTarget("/" + rt)
+		o.Mode |= ModeLink
+	}
+	return
+}
+
 func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStorageCreateMultipart) (o *Object, err error) {
 	input := &service.InitiateMultipartUploadInput{}
 	if opt.HasEncryptionCustomerAlgorithm {
@@ -570,10 +606,30 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
 	o = s.newObject(true)
 	o.ID = rp
 	o.Path = path
-	if opt.HasObjectMode && opt.ObjectMode.IsDir() {
-		o.Mode |= ModeDir
-	} else {
-		o.Mode |= ModeRead
+
+	if output.XQSMetaData != nil {
+		metadata := *output.XQSMetaData
+		// By calling `HeadObject`, the first letter of the `key` of the object metadata will be capitalized.
+		if v, ok := metadata[metadataLinkTargetHeader]; ok {
+			// The path is a symlink object.
+			if !s.features.VirtualLink {
+				// The virtual link is not enabled, so we set the object mode to `ModeRead`.
+				o.Mode |= ModeRead
+			} else {
+				// qingstor does not have an absolute path, so when we call `getAbsPath`, it will remove the prefix `/`.
+				// To ensure that the path matches the one the user gets, we should re-add `/` here.
+				o.SetLinkTarget("/" + v)
+				o.Mode |= ModeLink
+			}
+		}
+	}
+
+	if o.Mode&ModeLink == 0 && o.Mode&ModeRead == 0 {
+		if opt.HasObjectMode && opt.ObjectMode.IsDir() {
+			o.Mode |= ModeDir
+		} else {
+			o.Mode |= ModeRead
+		}
 	}
 
 	o.SetContentLength(service.Int64Value(output.ContentLength))

tests/storage_test.go

@@ -34,3 +34,10 @@ func TestDirer(t *testing.T) {
 	}
 	tests.TestDirer(t, setupTest(t))
 }
+
+func TestLinker(t *testing.T) {
+	if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" {
+		t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped")
+	}
+	tests.TestLinker(t, setupTest(t))
+}

tests/utils_test.go

@@ -20,7 +20,8 @@ func setupTest(t *testing.T) types.Storager {
 		ps.WithName(os.Getenv("STORAGE_QINGSTOR_NAME")),
 		ps.WithWorkDir("/"+uuid.New().String()+"/"),
 		qingstor.WithStorageFeatures(qingstor.StorageFeatures{
-			VirtualDir: true,
+			VirtualDir:  true,
+			VirtualLink: true,
 		}),
 	)
 	if err != nil {

utils.go

@@ -67,6 +67,7 @@ type Storage struct {
 	typ.UnimplementedReacher
 	typ.UnimplementedAppender
 	typ.UnimplementedDirer
+	typ.UnimplementedLinker
 }
 
 // String implements Storager.String
@@ -394,6 +395,8 @@ func (s *Storage) formatFileObject(v *service.KeyType) (o *typ.Object, err error
 	o = s.newObject(false)
 	o.ID = *v.Key
 	o.Path = s.getRelPath(*v.Key)
+	// If you have enabled virtual link, you will not get the accurate object type.
+	// If you want to get the exact object mode, please use `stat`
 	o.Mode |= typ.ModeRead
 
 	o.SetContentLength(service.Int64Value(v.Size))