From bccd18f2c667770c28349bea66ecce163d850cf1 Mon Sep 17 00:00:00 2001 From: eryajf Date: Mon, 11 Mar 2024 19:45:08 +0800 Subject: [PATCH 1/9] feat: add doge driver --- .air.toml | 44 ++++++ .gitignore | 1 + drivers/all.go | 1 + drivers/doge/driver.go | 156 +++++++++++++++++++++ drivers/doge/meta.go | 36 +++++ drivers/doge/types.go | 1 + drivers/doge/util.go | 309 +++++++++++++++++++++++++++++++++++++++++ go.sum | 16 --- 8 files changed, 548 insertions(+), 16 deletions(-) create mode 100644 .air.toml create mode 100644 drivers/doge/driver.go create mode 100644 drivers/doge/meta.go create mode 100644 drivers/doge/types.go create mode 100644 drivers/doge/util.go diff --git a/.air.toml b/.air.toml new file mode 100644 index 00000000000..52d17fb43da --- /dev/null +++ b/.air.toml @@ -0,0 +1,44 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] +args_bin = ["server"] +bin = "./tmp/main" +cmd = "go build -o ./tmp/main ." +delay = 0 +exclude_dir = ["assets", "tmp", "vendor", "testdata"] +exclude_file = [] +exclude_regex = ["_test.go"] +exclude_unchanged = false +follow_symlink = false +full_bin = "" +include_dir = [] +include_ext = ["go", "tpl", "tmpl", "html"] +include_file = [] +kill_delay = "0s" +log = "build-errors.log" +poll = false +poll_interval = 0 +rerun = false +rerun_delay = 500 +send_interrupt = false +stop_on_error = false + +[color] +app = "" +build = "yellow" +main = "magenta" +runner = "green" +watcher = "cyan" + +[log] +main_only = false +time = false + +[misc] +clean_on_exit = false + +[screen] +clear_on_rebuild = false +keep_scroll = true diff --git a/.gitignore b/.gitignore index 9f5ade897ca..1d71f0d608c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ output/ *.json /build /data/ +/tmp/ /log/ /lang/ /daemon/ diff --git a/drivers/all.go b/drivers/all.go index 08d8f1cbd42..f49c992e69b 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -21,6 +21,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/crypt" + _ "github.com/alist-org/alist/v3/drivers/doge" _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/google_drive" diff --git a/drivers/doge/driver.go b/drivers/doge/driver.go new file mode 100644 index 00000000000..70c1cdc4b0f --- /dev/null +++ b/drivers/doge/driver.go @@ -0,0 +1,156 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "net/url" + stdpath "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/stream" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + log "github.com/sirupsen/logrus" +) + +type S3 struct { + model.Storage + Addition + Session *session.Session + client *s3.S3 + linkClient *s3.S3 +} + +func (d *S3) Config() driver.Config { + return config +} + +func (d *S3) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *S3) Init(ctx context.Context) error { + if d.Region == "" { + d.Region = "automatic" + } + err := d.initSession() + if err != nil { + return err + } + d.client = d.getClient(false) + d.linkClient = d.getClient(true) + return nil +} + +func (d *S3) Drop(ctx context.Context) error { + return nil +} + +func (d *S3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if d.ListObjectVersion == "v2" { + return d.listV2(dir.GetPath(), args) + } + return d.listV1(dir.GetPath(), args) +} + +func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + path := getKey(file.GetPath(), false) + filename := stdpath.Base(path) + disposition := fmt.Sprintf(`attachment; filename*=UTF-8''%s`, url.PathEscape(filename)) + if d.AddFilenameToDisposition { + disposition = fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, filename, url.PathEscape(filename)) + } + input := &s3.GetObjectInput{ + Bucket: &d.Bucket, + Key: &path, + //ResponseContentDisposition: &disposition, + } + if d.CustomHost == "" { + input.ResponseContentDisposition = &disposition + } + req, _ := d.linkClient.GetObjectRequest(input) + var link string + var err error + if d.CustomHost != "" { + err = req.Build() + link = req.HTTPRequest.URL.String() + if d.RemoveBucket { + link = strings.Replace(link, "/"+d.Bucket, "", 1) + } + } else { + link, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + } + if err != nil { + return nil, err + } + return &model.Link{ + URL: link, + }, nil +} + +func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.Put(ctx, &model.Object{ + Path: stdpath.Join(parentDir.GetPath(), dirName), + }, &stream.FileStream{ + Obj: &model.Object{ + Name: getPlaceholderName(d.Placeholder), + Modified: time.Now(), + }, + Reader: io.NopCloser(bytes.NewReader([]byte{})), + Mimetype: "application/octet-stream", + }, func(float64) {}) +} + +func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + err := d.Copy(ctx, srcObj, dstDir) + if err != nil { + return err + } + return d.Remove(ctx, srcObj) +} + +func (d *S3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.IsDir()) + if err != nil { + return err + } + return d.Remove(ctx, srcObj) +} + +func (d *S3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()) +} + +func (d *S3) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + return d.removeDir(ctx, obj.GetPath()) + } + return d.removeFile(obj.GetPath()) +} + +func (d *S3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + uploader := s3manager.NewUploader(d.Session) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } + key := getKey(stdpath.Join(dstDir.GetPath(), stream.GetName()), false) + contentType := stream.GetMimetype() + log.Debugln("key:", key) + input := &s3manager.UploadInput{ + Bucket: &d.Bucket, + Key: &key, + Body: stream, + ContentType: &contentType, + } + _, err := uploader.UploadWithContext(ctx, input) + return err +} + +var _ driver.Driver = (*S3)(nil) diff --git a/drivers/doge/meta.go b/drivers/doge/meta.go new file mode 100644 index 00000000000..453f4db72e8 --- /dev/null +++ b/drivers/doge/meta.go @@ -0,0 +1,36 @@ +package s3 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Bucket string `json:"bucket" required:"true"` + Endpoint string `json:"endpoint" required:"true"` + Region string `json:"region"` + AccessKeyID string `json:"access_key_id" required:"true"` + SecretAccessKey string `json:"secret_access_key" required:"true"` + SessionToken string `json:"session_token"` + CustomHost string `json:"custom_host"` + SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` + Placeholder string `json:"placeholder"` + ForcePathStyle bool `json:"force_path_style"` + ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` + RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` + AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` +} + +var config = driver.Config{ + Name: "S3", + DefaultRoot: "/", + LocalSort: true, + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &S3{} + }) +} diff --git a/drivers/doge/types.go b/drivers/doge/types.go new file mode 100644 index 00000000000..3ed7f97237d --- /dev/null +++ b/drivers/doge/types.go @@ -0,0 +1 @@ +package s3 diff --git a/drivers/doge/util.go b/drivers/doge/util.go new file mode 100644 index 00000000000..0f1d53a67c0 --- /dev/null +++ b/drivers/doge/util.go @@ -0,0 +1,309 @@ +package s3 + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + log "github.com/sirupsen/logrus" +) + +// do others that not defined in Driver interface + +func (d *S3) initSession() error { + var err error + credentialsTmp, err := getCredentials(d.AccessKeyID, d.SecretAccessKey) + if err != nil { + return err + } + cfg := &aws.Config{ + Credentials: credentials.NewStaticCredentials(credentialsTmp.AccessKeyId, credentialsTmp.SecretAccessKey, credentialsTmp.SessionToken), + Region: &d.Region, + Endpoint: &d.Endpoint, + S3ForcePathStyle: aws.Bool(d.ForcePathStyle), + } + d.Session, err = session.NewSession(cfg) + return err +} + +func (d *S3) getClient(link bool) *s3.S3 { + client := s3.New(d.Session) + if link && d.CustomHost != "" { + client.Handlers.Build.PushBack(func(r *request.Request) { + if r.HTTPRequest.Method != http.MethodGet { + return + } + //判断CustomHost是否以http://或https://开头 + split := strings.SplitN(d.CustomHost, "://", 2) + if utils.SliceContains([]string{"http", "https"}, split[0]) { + r.HTTPRequest.URL.Scheme = split[0] + r.HTTPRequest.URL.Host = split[1] + } else { + r.HTTPRequest.URL.Host = d.CustomHost + } + }) + } + return client +} + +type TmpTokenResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data TmpTokenResponseData `json:"data,omitempty"` +} +type TmpTokenResponseData struct { + Credentials Credentials `json:"Credentials"` +} +type Credentials struct { + AccessKeyId string `json:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty"` + SessionToken string `json:"sessionToken,omitempty"` +} + +func getCredentials(AccessKey, SecretKey string) (rst Credentials, err error) { + apiPath := "/auth/tmp_token.json" + reqBody, err := json.Marshal(map[string]interface{}{"channel": "OSS_FULL", "scopes": []string{"*"}}) + if err != nil { + return rst, err + } + + signStr := apiPath + "\n" + string(reqBody) + hmacObj := hmac.New(sha1.New, []byte(SecretKey)) + hmacObj.Write([]byte(signStr)) + sign := hex.EncodeToString(hmacObj.Sum(nil)) + Authorization := "TOKEN " + AccessKey + ":" + sign + + req, err := http.NewRequest("POST", "https://api.dogecloud.com"+apiPath, strings.NewReader(string(reqBody))) + if err != nil { + return rst, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", Authorization) + client := http.Client{} + resp, err := client.Do(req) + if err != nil { + return rst, err + } + defer resp.Body.Close() + ret, err := io.ReadAll(resp.Body) + if err != nil { + return rst, err + } + var tmpTokenResp TmpTokenResponse + err = json.Unmarshal(ret, &tmpTokenResp) + if err != nil { + return rst, err + } + return tmpTokenResp.Data.Credentials, nil +} + +func getKey(path string, dir bool) string { + path = strings.TrimPrefix(path, "/") + if path != "" && dir { + path += "/" + } + return path +} + +var defaultPlaceholderName = ".alist" + +func getPlaceholderName(placeholder string) string { + if placeholder == "" { + return defaultPlaceholderName + } + return placeholder +} + +func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { + prefix = getKey(prefix, true) + log.Debugf("list: %s", prefix) + files := make([]model.Obj, 0) + marker := "" + for { + input := &s3.ListObjectsInput{ + Bucket: &d.Bucket, + Marker: &marker, + Prefix: &prefix, + Delimiter: aws.String("/"), + } + listObjectsResult, err := d.client.ListObjects(input) + if err != nil { + return nil, err + } + for _, object := range listObjectsResult.CommonPrefixes { + name := path.Base(strings.Trim(*object.Prefix, "/")) + file := model.Object{ + //Id: *object.Key, + Name: name, + Modified: d.Modified, + IsFolder: true, + } + files = append(files, &file) + } + for _, object := range listObjectsResult.Contents { + name := path.Base(*object.Key) + if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { + continue + } + file := model.Object{ + //Id: *object.Key, + Name: name, + Size: *object.Size, + Modified: *object.LastModified, + } + files = append(files, &file) + } + if listObjectsResult.IsTruncated == nil { + return nil, errors.New("IsTruncated nil") + } + if *listObjectsResult.IsTruncated { + marker = *listObjectsResult.NextMarker + } else { + break + } + } + return files, nil +} + +func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { + prefix = getKey(prefix, true) + files := make([]model.Obj, 0) + var continuationToken, startAfter *string + for { + input := &s3.ListObjectsV2Input{ + Bucket: &d.Bucket, + ContinuationToken: continuationToken, + Prefix: &prefix, + Delimiter: aws.String("/"), + StartAfter: startAfter, + } + listObjectsResult, err := d.client.ListObjectsV2(input) + if err != nil { + return nil, err + } + log.Debugf("resp: %+v", listObjectsResult) + for _, object := range listObjectsResult.CommonPrefixes { + name := path.Base(strings.Trim(*object.Prefix, "/")) + file := model.Object{ + //Id: *object.Key, + Name: name, + Modified: d.Modified, + IsFolder: true, + } + files = append(files, &file) + } + for _, object := range listObjectsResult.Contents { + if strings.HasSuffix(*object.Key, "/") { + continue + } + name := path.Base(*object.Key) + if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { + continue + } + file := model.Object{ + //Id: *object.Key, + Name: name, + Size: *object.Size, + Modified: *object.LastModified, + } + files = append(files, &file) + } + if !aws.BoolValue(listObjectsResult.IsTruncated) { + break + } + if listObjectsResult.NextContinuationToken != nil { + continuationToken = listObjectsResult.NextContinuationToken + continue + } + if len(listObjectsResult.Contents) == 0 { + break + } + startAfter = listObjectsResult.Contents[len(listObjectsResult.Contents)-1].Key + } + return files, nil +} + +func (d *S3) copy(ctx context.Context, src string, dst string, isDir bool) error { + if isDir { + return d.copyDir(ctx, src, dst) + } + return d.copyFile(ctx, src, dst) +} + +func (d *S3) copyFile(ctx context.Context, src string, dst string) error { + srcKey := getKey(src, false) + dstKey := getKey(dst, false) + input := &s3.CopyObjectInput{ + Bucket: &d.Bucket, + CopySource: aws.String("/" + d.Bucket + "/" + srcKey), + Key: &dstKey, + } + _, err := d.client.CopyObject(input) + return err +} + +func (d *S3) copyDir(ctx context.Context, src string, dst string) error { + objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) + if err != nil { + return err + } + for _, obj := range objs { + cSrc := path.Join(src, obj.GetName()) + cDst := path.Join(dst, obj.GetName()) + if obj.IsDir() { + err = d.copyDir(ctx, cSrc, cDst) + } else { + err = d.copyFile(ctx, cSrc, cDst) + } + if err != nil { + return err + } + } + return nil +} + +func (d *S3) removeDir(ctx context.Context, src string) error { + objs, err := op.List(ctx, d, src, model.ListArgs{}) + if err != nil { + return err + } + for _, obj := range objs { + cSrc := path.Join(src, obj.GetName()) + if obj.IsDir() { + err = d.removeDir(ctx, cSrc) + } else { + err = d.removeFile(cSrc) + } + if err != nil { + return err + } + } + _ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder))) + _ = d.removeFile(path.Join(src, d.Placeholder)) + return nil +} + +func (d *S3) removeFile(src string) error { + key := getKey(src, false) + input := &s3.DeleteObjectInput{ + Bucket: &d.Bucket, + Key: &key, + } + _, err := d.client.DeleteObject(input) + return err +} diff --git a/go.sum b/go.sum index a46f0d55cce..3ea10e49e0b 100644 --- a/go.sum +++ b/go.sum @@ -447,8 +447,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4UvMT7OmKhH9sOIvS8iXkxfPdnn9wQA= -github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= @@ -507,12 +505,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= -golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -532,8 +526,6 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= @@ -542,8 +534,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -573,8 +563,6 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -583,8 +571,6 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -610,8 +596,6 @@ golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 95ea4455708baaa84f1f790066a5d1e93744d662 Mon Sep 17 00:00:00 2001 From: eryajf Date: Tue, 19 Mar 2024 00:04:36 +0800 Subject: [PATCH 2/9] =?UTF-8?q?doc:=20=E8=A1=A5=E5=85=85readme=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + README_cn.md | 1 + README_ja.md | 1 + 3 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 431b32128a5..0935b9cc1f1 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) - [x] [FeijiPan](https://www.feijipan.com/) + - [x] [dogecloud](https://www.dogecloud.com/product/oss) - [x] Easy to deploy and out-of-the-box - [x] File preview (PDF, markdown, code, plain text, ...) - [x] Image preview in gallery mode diff --git a/README_cn.md b/README_cn.md index db8455e1eb8..e6feab09936 100644 --- a/README_cn.md +++ b/README_cn.md @@ -75,6 +75,7 @@ - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) - [x] [飞机盘](https://www.feijipan.com/) + - [x] [多吉云](https://www.dogecloud.com/product/oss) - [x] 部署方便，开箱即用 - [x] 文件预览（PDF、markdown、代码、纯文本……） - [x] 画廊模式下的图像预览 diff --git a/README_ja.md b/README_ja.md index 67b2840a586..2ad984365d7 100644 --- a/README_ja.md +++ b/README_ja.md @@ -76,6 +76,7 @@ - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) - [x] [FeijiPan](https://www.feijipan.com/) + - [x] [dogecloud](https://www.dogecloud.com/product/oss) - [x] デプロイが簡単で、すぐに使える - [x] ファイルプレビュー (PDF, マークダウン, コード, プレーンテキスト, ...) - [x] ギャラリーモードでの画像プレビュー From d64bf888cebfcb07198fa368b2c70721efb9d931 Mon Sep 17 00:00:00 2001 From: eryajf Date: Tue, 19 Mar 2024 00:15:59 +0800 Subject: [PATCH 3/9] =?UTF-8?q?fix:=20=E5=AF=B9=E9=BD=90meta=E4=BF=A1?= =?UTF-8?q?=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/doge/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/doge/meta.go b/drivers/doge/meta.go index 453f4db72e8..0a320263f92 100644 --- a/drivers/doge/meta.go +++ b/drivers/doge/meta.go @@ -23,7 +23,7 @@ type Addition struct { } var config = driver.Config{ - Name: "S3", + Name: "Doge", DefaultRoot: "/", LocalSort: true, CheckStatus: true, From 54d2a60223da5e7189724bdaddcaac04dee8196f Mon Sep 17 00:00:00 2001 From: eryajf Date: Sun, 24 Mar 2024 11:19:19 +0800 Subject: [PATCH 4/9] =?UTF-8?q?fix:=20=E8=B0=83=E6=95=B4=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E4=BD=93=E5=90=8D=E5=AD=97,=E4=B8=8Edriver=E4=BF=9D=E6=8C=81?= =?UTF-8?q?=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/doge/driver.go | 28 ++++++++++++++-------------- drivers/doge/meta.go | 2 +- drivers/doge/util.go | 18 +++++++++--------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/drivers/doge/driver.go b/drivers/doge/driver.go index 70c1cdc4b0f..49c5f0c3cb7 100644 --- a/drivers/doge/driver.go +++ b/drivers/doge/driver.go @@ -20,7 +20,7 @@ import ( log "github.com/sirupsen/logrus" ) -type S3 struct { +type Doge struct { model.Storage Addition Session *session.Session @@ -28,15 +28,15 @@ type S3 struct { linkClient *s3.S3 } -func (d *S3) Config() driver.Config { +func (d *Doge) Config() driver.Config { return config } -func (d *S3) GetAddition() driver.Additional { +func (d *Doge) GetAddition() driver.Additional { return &d.Addition } -func (d *S3) Init(ctx context.Context) error { +func (d *Doge) Init(ctx context.Context) error { if d.Region == "" { d.Region = "automatic" } @@ -49,18 +49,18 @@ func (d *S3) Init(ctx context.Context) error { return nil } -func (d *S3) Drop(ctx context.Context) error { +func (d *Doge) Drop(ctx context.Context) error { return nil } -func (d *S3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { +func (d *Doge) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if d.ListObjectVersion == "v2" { return d.listV2(dir.GetPath(), args) } return d.listV1(dir.GetPath(), args) } -func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { +func (d *Doge) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { path := getKey(file.GetPath(), false) filename := stdpath.Base(path) disposition := fmt.Sprintf(`attachment; filename*=UTF-8''%s`, url.PathEscape(filename)) @@ -95,7 +95,7 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo }, nil } -func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { +func (d *Doge) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return d.Put(ctx, &model.Object{ Path: stdpath.Join(parentDir.GetPath(), dirName), }, &stream.FileStream{ @@ -108,7 +108,7 @@ func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) e }, func(float64) {}) } -func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *Doge) Move(ctx context.Context, srcObj, dstDir model.Obj) error { err := d.Copy(ctx, srcObj, dstDir) if err != nil { return err @@ -116,7 +116,7 @@ func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return d.Remove(ctx, srcObj) } -func (d *S3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { +func (d *Doge) Rename(ctx context.Context, srcObj model.Obj, newName string) error { err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.IsDir()) if err != nil { return err @@ -124,18 +124,18 @@ func (d *S3) Rename(ctx context.Context, srcObj model.Obj, newName string) error return d.Remove(ctx, srcObj) } -func (d *S3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *Doge) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()) } -func (d *S3) Remove(ctx context.Context, obj model.Obj) error { +func (d *Doge) Remove(ctx context.Context, obj model.Obj) error { if obj.IsDir() { return d.removeDir(ctx, obj.GetPath()) } return d.removeFile(obj.GetPath()) } -func (d *S3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *Doge) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { uploader := s3manager.NewUploader(d.Session) if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) @@ -153,4 +153,4 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreame return err } -var _ driver.Driver = (*S3)(nil) +var _ driver.Driver = (*Doge)(nil) diff --git a/drivers/doge/meta.go b/drivers/doge/meta.go index 0a320263f92..f87a93f3be7 100644 --- a/drivers/doge/meta.go +++ b/drivers/doge/meta.go @@ -31,6 +31,6 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { - return &S3{} + return &Doge{} }) } diff --git a/drivers/doge/util.go b/drivers/doge/util.go index 0f1d53a67c0..816e2fe2dde 100644 --- a/drivers/doge/util.go +++ b/drivers/doge/util.go @@ -25,7 +25,7 @@ import ( // do others that not defined in Driver interface -func (d *S3) initSession() error { +func (d *Doge) initSession() error { var err error credentialsTmp, err := getCredentials(d.AccessKeyID, d.SecretAccessKey) if err != nil { @@ -41,7 +41,7 @@ func (d *S3) initSession() error { return err } -func (d *S3) getClient(link bool) *s3.S3 { +func (d *Doge) getClient(link bool) *s3.S3 { client := s3.New(d.Session) if link && d.CustomHost != "" { client.Handlers.Build.PushBack(func(r *request.Request) { @@ -129,7 +129,7 @@ func getPlaceholderName(placeholder string) string { return placeholder } -func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { +func (d *Doge) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { prefix = getKey(prefix, true) log.Debugf("list: %s", prefix) files := make([]model.Obj, 0) @@ -180,7 +180,7 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { return files, nil } -func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { +func (d *Doge) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { prefix = getKey(prefix, true) files := make([]model.Obj, 0) var continuationToken, startAfter *string @@ -238,14 +238,14 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { return files, nil } -func (d *S3) copy(ctx context.Context, src string, dst string, isDir bool) error { +func (d *Doge) copy(ctx context.Context, src string, dst string, isDir bool) error { if isDir { return d.copyDir(ctx, src, dst) } return d.copyFile(ctx, src, dst) } -func (d *S3) copyFile(ctx context.Context, src string, dst string) error { +func (d *Doge) copyFile(ctx context.Context, src string, dst string) error { srcKey := getKey(src, false) dstKey := getKey(dst, false) input := &s3.CopyObjectInput{ @@ -257,7 +257,7 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { return err } -func (d *S3) copyDir(ctx context.Context, src string, dst string) error { +func (d *Doge) copyDir(ctx context.Context, src string, dst string) error { objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) if err != nil { return err @@ -277,7 +277,7 @@ func (d *S3) copyDir(ctx context.Context, src string, dst string) error { return nil } -func (d *S3) removeDir(ctx context.Context, src string) error { +func (d *Doge) removeDir(ctx context.Context, src string) error { objs, err := op.List(ctx, d, src, model.ListArgs{}) if err != nil { return err @@ -298,7 +298,7 @@ func (d *S3) removeDir(ctx context.Context, src string) error { return nil } -func (d *S3) removeFile(src string) error { +func (d *Doge) removeFile(src string) error { key := getKey(src, false) input := &s3.DeleteObjectInput{ Bucket: &d.Bucket, From 05622793d4e7e01b6914d3d43ed14625ebbefea7 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 24 Mar 2024 15:31:12 +0800 Subject: [PATCH 5/9] perf: merge to s3 --- drivers/all.go | 1 - drivers/doge/driver.go | 156 --------------------- drivers/doge/meta.go | 36 ----- drivers/doge/types.go | 1 - drivers/doge/util.go | 309 ----------------------------------------- drivers/s3/driver.go | 4 +- drivers/s3/goge.go | 62 +++++++++ drivers/s3/meta.go | 26 ++-- drivers/s3/util.go | 12 +- 9 files changed, 93 insertions(+), 514 deletions(-) delete mode 100644 drivers/doge/driver.go delete mode 100644 drivers/doge/meta.go delete mode 100644 drivers/doge/types.go delete mode 100644 drivers/doge/util.go create mode 100644 drivers/s3/goge.go diff --git a/drivers/all.go b/drivers/all.go index f49c992e69b..08d8f1cbd42 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -21,7 +21,6 @@ import ( _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/crypt" - _ "github.com/alist-org/alist/v3/drivers/doge" _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/google_drive" diff --git a/drivers/doge/driver.go b/drivers/doge/driver.go deleted file mode 100644 index 49c5f0c3cb7..00000000000 --- a/drivers/doge/driver.go +++ /dev/null @@ -1,156 +0,0 @@ -package s3 - -import ( - "bytes" - "context" - "fmt" - "io" - "net/url" - stdpath "path" - "strings" - "time" - - "github.com/alist-org/alist/v3/internal/stream" - - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/model" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - log "github.com/sirupsen/logrus" -) - -type Doge struct { - model.Storage - Addition - Session *session.Session - client *s3.S3 - linkClient *s3.S3 -} - -func (d *Doge) Config() driver.Config { - return config -} - -func (d *Doge) GetAddition() driver.Additional { - return &d.Addition -} - -func (d *Doge) Init(ctx context.Context) error { - if d.Region == "" { - d.Region = "automatic" - } - err := d.initSession() - if err != nil { - return err - } - d.client = d.getClient(false) - d.linkClient = d.getClient(true) - return nil -} - -func (d *Doge) Drop(ctx context.Context) error { - return nil -} - -func (d *Doge) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if d.ListObjectVersion == "v2" { - return d.listV2(dir.GetPath(), args) - } - return d.listV1(dir.GetPath(), args) -} - -func (d *Doge) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - path := getKey(file.GetPath(), false) - filename := stdpath.Base(path) - disposition := fmt.Sprintf(`attachment; filename*=UTF-8''%s`, url.PathEscape(filename)) - if d.AddFilenameToDisposition { - disposition = fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, filename, url.PathEscape(filename)) - } - input := &s3.GetObjectInput{ - Bucket: &d.Bucket, - Key: &path, - //ResponseContentDisposition: &disposition, - } - if d.CustomHost == "" { - input.ResponseContentDisposition = &disposition - } - req, _ := d.linkClient.GetObjectRequest(input) - var link string - var err error - if d.CustomHost != "" { - err = req.Build() - link = req.HTTPRequest.URL.String() - if d.RemoveBucket { - link = strings.Replace(link, "/"+d.Bucket, "", 1) - } - } else { - link, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) - } - if err != nil { - return nil, err - } - return &model.Link{ - URL: link, - }, nil -} - -func (d *Doge) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - return d.Put(ctx, &model.Object{ - Path: stdpath.Join(parentDir.GetPath(), dirName), - }, &stream.FileStream{ - Obj: &model.Object{ - Name: getPlaceholderName(d.Placeholder), - Modified: time.Now(), - }, - Reader: io.NopCloser(bytes.NewReader([]byte{})), - Mimetype: "application/octet-stream", - }, func(float64) {}) -} - -func (d *Doge) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - err := d.Copy(ctx, srcObj, dstDir) - if err != nil { - return err - } - return d.Remove(ctx, srcObj) -} - -func (d *Doge) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.IsDir()) - if err != nil { - return err - } - return d.Remove(ctx, srcObj) -} - -func (d *Doge) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()) -} - -func (d *Doge) Remove(ctx context.Context, obj model.Obj) error { - if obj.IsDir() { - return d.removeDir(ctx, obj.GetPath()) - } - return d.removeFile(obj.GetPath()) -} - -func (d *Doge) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - uploader := s3manager.NewUploader(d.Session) - if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) - } - key := getKey(stdpath.Join(dstDir.GetPath(), stream.GetName()), false) - contentType := stream.GetMimetype() - log.Debugln("key:", key) - input := &s3manager.UploadInput{ - Bucket: &d.Bucket, - Key: &key, - Body: stream, - ContentType: &contentType, - } - _, err := uploader.UploadWithContext(ctx, input) - return err -} - -var _ driver.Driver = (*Doge)(nil) diff --git a/drivers/doge/meta.go b/drivers/doge/meta.go deleted file mode 100644 index f87a93f3be7..00000000000 --- a/drivers/doge/meta.go +++ /dev/null @@ -1,36 +0,0 @@ -package s3 - -import ( - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/op" -) - -type Addition struct { - driver.RootPath - Bucket string `json:"bucket" required:"true"` - Endpoint string `json:"endpoint" required:"true"` - Region string `json:"region"` - AccessKeyID string `json:"access_key_id" required:"true"` - SecretAccessKey string `json:"secret_access_key" required:"true"` - SessionToken string `json:"session_token"` - CustomHost string `json:"custom_host"` - SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` - Placeholder string `json:"placeholder"` - ForcePathStyle bool `json:"force_path_style"` - ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` - RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` - AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` -} - -var config = driver.Config{ - Name: "Doge", - DefaultRoot: "/", - LocalSort: true, - CheckStatus: true, -} - -func init() { - op.RegisterDriver(func() driver.Driver { - return &Doge{} - }) -} diff --git a/drivers/doge/types.go b/drivers/doge/types.go deleted file mode 100644 index 3ed7f97237d..00000000000 --- a/drivers/doge/types.go +++ /dev/null @@ -1 +0,0 @@ -package s3 diff --git a/drivers/doge/util.go b/drivers/doge/util.go deleted file mode 100644 index 816e2fe2dde..00000000000 --- a/drivers/doge/util.go +++ /dev/null @@ -1,309 +0,0 @@ -package s3 - -import ( - "context" - "crypto/hmac" - "crypto/sha1" - "encoding/hex" - "encoding/json" - "errors" - "io" - "net/http" - "path" - "strings" - - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - log "github.com/sirupsen/logrus" -) - -// do others that not defined in Driver interface - -func (d *Doge) initSession() error { - var err error - credentialsTmp, err := getCredentials(d.AccessKeyID, d.SecretAccessKey) - if err != nil { - return err - } - cfg := &aws.Config{ - Credentials: credentials.NewStaticCredentials(credentialsTmp.AccessKeyId, credentialsTmp.SecretAccessKey, credentialsTmp.SessionToken), - Region: &d.Region, - Endpoint: &d.Endpoint, - S3ForcePathStyle: aws.Bool(d.ForcePathStyle), - } - d.Session, err = session.NewSession(cfg) - return err -} - -func (d *Doge) getClient(link bool) *s3.S3 { - client := s3.New(d.Session) - if link && d.CustomHost != "" { - client.Handlers.Build.PushBack(func(r *request.Request) { - if r.HTTPRequest.Method != http.MethodGet { - return - } - //判断CustomHost是否以http://或https://开头 - split := strings.SplitN(d.CustomHost, "://", 2) - if utils.SliceContains([]string{"http", "https"}, split[0]) { - r.HTTPRequest.URL.Scheme = split[0] - r.HTTPRequest.URL.Host = split[1] - } else { - r.HTTPRequest.URL.Host = d.CustomHost - } - }) - } - return client -} - -type TmpTokenResponse struct { - Code int `json:"code"` - Msg string `json:"msg"` - Data TmpTokenResponseData `json:"data,omitempty"` -} -type TmpTokenResponseData struct { - Credentials Credentials `json:"Credentials"` -} -type Credentials struct { - AccessKeyId string `json:"accessKeyId,omitempty"` - SecretAccessKey string `json:"secretAccessKey,omitempty"` - SessionToken string `json:"sessionToken,omitempty"` -} - -func getCredentials(AccessKey, SecretKey string) (rst Credentials, err error) { - apiPath := "/auth/tmp_token.json" - reqBody, err := json.Marshal(map[string]interface{}{"channel": "OSS_FULL", "scopes": []string{"*"}}) - if err != nil { - return rst, err - } - - signStr := apiPath + "\n" + string(reqBody) - hmacObj := hmac.New(sha1.New, []byte(SecretKey)) - hmacObj.Write([]byte(signStr)) - sign := hex.EncodeToString(hmacObj.Sum(nil)) - Authorization := "TOKEN " + AccessKey + ":" + sign - - req, err := http.NewRequest("POST", "https://api.dogecloud.com"+apiPath, strings.NewReader(string(reqBody))) - if err != nil { - return rst, err - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", Authorization) - client := http.Client{} - resp, err := client.Do(req) - if err != nil { - return rst, err - } - defer resp.Body.Close() - ret, err := io.ReadAll(resp.Body) - if err != nil { - return rst, err - } - var tmpTokenResp TmpTokenResponse - err = json.Unmarshal(ret, &tmpTokenResp) - if err != nil { - return rst, err - } - return tmpTokenResp.Data.Credentials, nil -} - -func getKey(path string, dir bool) string { - path = strings.TrimPrefix(path, "/") - if path != "" && dir { - path += "/" - } - return path -} - -var defaultPlaceholderName = ".alist" - -func getPlaceholderName(placeholder string) string { - if placeholder == "" { - return defaultPlaceholderName - } - return placeholder -} - -func (d *Doge) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { - prefix = getKey(prefix, true) - log.Debugf("list: %s", prefix) - files := make([]model.Obj, 0) - marker := "" - for { - input := &s3.ListObjectsInput{ - Bucket: &d.Bucket, - Marker: &marker, - Prefix: &prefix, - Delimiter: aws.String("/"), - } - listObjectsResult, err := d.client.ListObjects(input) - if err != nil { - return nil, err - } - for _, object := range listObjectsResult.CommonPrefixes { - name := path.Base(strings.Trim(*object.Prefix, "/")) - file := model.Object{ - //Id: *object.Key, - Name: name, - Modified: d.Modified, - IsFolder: true, - } - files = append(files, &file) - } - for _, object := range listObjectsResult.Contents { - name := path.Base(*object.Key) - if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { - continue - } - file := model.Object{ - //Id: *object.Key, - Name: name, - Size: *object.Size, - Modified: *object.LastModified, - } - files = append(files, &file) - } - if listObjectsResult.IsTruncated == nil { - return nil, errors.New("IsTruncated nil") - } - if *listObjectsResult.IsTruncated { - marker = *listObjectsResult.NextMarker - } else { - break - } - } - return files, nil -} - -func (d *Doge) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { - prefix = getKey(prefix, true) - files := make([]model.Obj, 0) - var continuationToken, startAfter *string - for { - input := &s3.ListObjectsV2Input{ - Bucket: &d.Bucket, - ContinuationToken: continuationToken, - Prefix: &prefix, - Delimiter: aws.String("/"), - StartAfter: startAfter, - } - listObjectsResult, err := d.client.ListObjectsV2(input) - if err != nil { - return nil, err - } - log.Debugf("resp: %+v", listObjectsResult) - for _, object := range listObjectsResult.CommonPrefixes { - name := path.Base(strings.Trim(*object.Prefix, "/")) - file := model.Object{ - //Id: *object.Key, - Name: name, - Modified: d.Modified, - IsFolder: true, - } - files = append(files, &file) - } - for _, object := range listObjectsResult.Contents { - if strings.HasSuffix(*object.Key, "/") { - continue - } - name := path.Base(*object.Key) - if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { - continue - } - file := model.Object{ - //Id: *object.Key, - Name: name, - Size: *object.Size, - Modified: *object.LastModified, - } - files = append(files, &file) - } - if !aws.BoolValue(listObjectsResult.IsTruncated) { - break - } - if listObjectsResult.NextContinuationToken != nil { - continuationToken = listObjectsResult.NextContinuationToken - continue - } - if len(listObjectsResult.Contents) == 0 { - break - } - startAfter = listObjectsResult.Contents[len(listObjectsResult.Contents)-1].Key - } - return files, nil -} - -func (d *Doge) copy(ctx context.Context, src string, dst string, isDir bool) error { - if isDir { - return d.copyDir(ctx, src, dst) - } - return d.copyFile(ctx, src, dst) -} - -func (d *Doge) copyFile(ctx context.Context, src string, dst string) error { - srcKey := getKey(src, false) - dstKey := getKey(dst, false) - input := &s3.CopyObjectInput{ - Bucket: &d.Bucket, - CopySource: aws.String("/" + d.Bucket + "/" + srcKey), - Key: &dstKey, - } - _, err := d.client.CopyObject(input) - return err -} - -func (d *Doge) copyDir(ctx context.Context, src string, dst string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) - if err != nil { - return err - } - for _, obj := range objs { - cSrc := path.Join(src, obj.GetName()) - cDst := path.Join(dst, obj.GetName()) - if obj.IsDir() { - err = d.copyDir(ctx, cSrc, cDst) - } else { - err = d.copyFile(ctx, cSrc, cDst) - } - if err != nil { - return err - } - } - return nil -} - -func (d *Doge) removeDir(ctx context.Context, src string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{}) - if err != nil { - return err - } - for _, obj := range objs { - cSrc := path.Join(src, obj.GetName()) - if obj.IsDir() { - err = d.removeDir(ctx, cSrc) - } else { - err = d.removeFile(cSrc) - } - if err != nil { - return err - } - } - _ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder))) - _ = d.removeFile(path.Join(src, d.Placeholder)) - return nil -} - -func (d *Doge) removeFile(src string) error { - key := getKey(src, false) - input := &s3.DeleteObjectInput{ - Bucket: &d.Bucket, - Key: &key, - } - _, err := d.client.DeleteObject(input) - return err -} diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index c8099ee43dd..3209a476c33 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -26,10 +26,12 @@ type S3 struct { Session *session.Session client *s3.S3 linkClient *s3.S3 + + config driver.Config } func (d *S3) Config() driver.Config { - return config + return d.config } func (d *S3) GetAddition() driver.Additional { diff --git a/drivers/s3/goge.go b/drivers/s3/goge.go new file mode 100644 index 00000000000..8b516d97e23 --- /dev/null +++ b/drivers/s3/goge.go @@ -0,0 +1,62 @@ +package s3 + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "strings" +) + +type TmpTokenResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data TmpTokenResponseData `json:"data,omitempty"` +} +type TmpTokenResponseData struct { + Credentials Credentials `json:"Credentials"` +} +type Credentials struct { + AccessKeyId string `json:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty"` + SessionToken string `json:"sessionToken,omitempty"` +} + +func getCredentials(AccessKey, SecretKey string) (rst Credentials, err error) { + apiPath := "/auth/tmp_token.json" + reqBody, err := json.Marshal(map[string]interface{}{"channel": "OSS_FULL", "scopes": []string{"*"}}) + if err != nil { + return rst, err + } + + signStr := apiPath + "\n" + string(reqBody) + hmacObj := hmac.New(sha1.New, []byte(SecretKey)) + hmacObj.Write([]byte(signStr)) + sign := hex.EncodeToString(hmacObj.Sum(nil)) + Authorization := "TOKEN " + AccessKey + ":" + sign + + req, err := http.NewRequest("POST", "https://api.dogecloud.com"+apiPath, strings.NewReader(string(reqBody))) + if err != nil { + return rst, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", Authorization) + client := http.Client{} + resp, err := client.Do(req) + if err != nil { + return rst, err + } + defer resp.Body.Close() + ret, err := io.ReadAll(resp.Body) + if err != nil { + return rst, err + } + var tmpTokenResp TmpTokenResponse + err = json.Unmarshal(ret, &tmpTokenResp) + if err != nil { + return rst, err + } + return tmpTokenResp.Data.Credentials, nil +} diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 453f4db72e8..4436c61508e 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -22,15 +22,25 @@ type Addition struct { AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` } -var config = driver.Config{ - Name: "S3", - DefaultRoot: "/", - LocalSort: true, - CheckStatus: true, -} - func init() { op.RegisterDriver(func() driver.Driver { - return &S3{} + return &S3{ + config: driver.Config{ + Name: "S3", + DefaultRoot: "/", + LocalSort: true, + CheckStatus: true, + }, + } + }) + op.RegisterDriver(func() driver.Driver { + return &S3{ + config: driver.Config{ + Name: "Doge", + DefaultRoot: "/", + LocalSort: true, + CheckStatus: true, + }, + } }) } diff --git a/drivers/s3/util.go b/drivers/s3/util.go index 5578176a7a3..31e658bdcab 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -21,13 +21,21 @@ import ( // do others that not defined in Driver interface func (d *S3) initSession() error { + var err error + accessKeyID, secretAccessKey, sessionToken := d.AccessKeyID, d.SecretAccessKey, d.SessionToken + if d.config.Name == "Doge" { + credentialsTmp, err := getCredentials(d.AccessKeyID, d.SecretAccessKey) + if err != nil { + return err + } + accessKeyID, secretAccessKey, sessionToken = credentialsTmp.AccessKeyId, credentialsTmp.SecretAccessKey, credentialsTmp.SessionToken + } cfg := &aws.Config{ - Credentials: credentials.NewStaticCredentials(d.AccessKeyID, d.SecretAccessKey, d.SessionToken), + Credentials: credentials.NewStaticCredentials(accessKeyID, secretAccessKey, sessionToken), Region: &d.Region, Endpoint: &d.Endpoint, S3ForcePathStyle: aws.Bool(d.ForcePathStyle), } - var err error d.Session, err = session.NewSession(cfg) return err } From 8c7d54140447f6423bdcfa25af4553bd32a252ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=8C=E4=B8=AB=E8=AE=B2=E6=A2=B5?= Date: Sun, 24 Mar 2024 18:35:02 +0800 Subject: [PATCH 6/9] Rename goge.go to doge.go --- drivers/s3/{goge.go => doge.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename drivers/s3/{goge.go => doge.go} (100%) diff --git a/drivers/s3/goge.go b/drivers/s3/doge.go similarity index 100% rename from drivers/s3/goge.go rename to drivers/s3/doge.go From a59a08b6771b383a1a830670dc06941005451e64 Mon Sep 17 00:00:00 2001 From: eryajf Date: Tue, 26 Mar 2024 23:19:44 +0800 Subject: [PATCH 7/9] =?UTF-8?q?fix:=20=E8=A7=A3=E5=86=B3=E5=A4=9A=E5=90=89?= =?UTF-8?q?=E4=BA=91=E4=B8=B4=E6=97=B6=E7=A7=98=E9=92=A5=E4=B8=A4=E4=B8=AA?= =?UTF-8?q?=E5=B0=8F=E6=97=B6=E8=BF=87=E6=9C=9F=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/s3/doge.go | 1 + drivers/s3/driver.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/s3/doge.go b/drivers/s3/doge.go index 8b516d97e23..12a584ca4f2 100644 --- a/drivers/s3/doge.go +++ b/drivers/s3/doge.go @@ -17,6 +17,7 @@ type TmpTokenResponse struct { } type TmpTokenResponseData struct { Credentials Credentials `json:"Credentials"` + ExpiredAt int `json:"ExpiredAt"` } type Credentials struct { AccessKeyId string `json:"accessKeyId,omitempty"` diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 3209a476c33..375cdee836e 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -11,6 +11,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" @@ -28,6 +29,7 @@ type S3 struct { linkClient *s3.S3 config driver.Config + cron *cron.Cron } func (d *S3) Config() driver.Config { @@ -42,6 +44,15 @@ func (d *S3) Init(ctx context.Context) error { if d.Region == "" { d.Region = "alist" } + if d.config.Name == "Doge" { + d.cron = cron.NewCron(time.Minute * 118) + d.cron.Do(func() { + err := d.initSession() + if err != nil { + log.Errorln("Doge init session error:", err) + } + }) + } err := d.initSession() if err != nil { return err From d90a1678f7441645a36f2279839cb611f37b17c1 Mon Sep 17 00:00:00 2001 From: eryajf Date: Wed, 27 Mar 2024 14:06:38 +0800 Subject: [PATCH 8/9] =?UTF-8?q?fix:=20=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=9C=A8Drop=E4=B8=ADStop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/s3/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 375cdee836e..bc9b42f37f1 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -63,6 +63,9 @@ func (d *S3) Init(ctx context.Context) error { } func (d *S3) Drop(ctx context.Context) error { + if d.cron != nil { + d.cron.Stop() + } return nil } From e7192cd3e7ea9bf198e78fe338e7d3850a374e81 Mon Sep 17 00:00:00 2001 From: eryajf Date: Thu, 28 Mar 2024 18:33:44 +0800 Subject: [PATCH 9/9] =?UTF-8?q?fix:=20Doge=20=E9=87=8D=E7=BD=AE=E7=A7=98?= =?UTF-8?q?=E9=92=A5=E4=B9=8B=E5=90=8E=E6=9C=AA=E9=87=8D=E6=96=B0=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=AE=A2=E6=88=B7=E7=AB=AF=E5=AF=BC=E8=87=B4403?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/s3/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index bc9b42f37f1..728c642038c 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -45,12 +45,15 @@ func (d *S3) Init(ctx context.Context) error { d.Region = "alist" } if d.config.Name == "Doge" { + // 多吉云每次临时生成的秘钥有效期为 2h，所以这里设置为 118 分钟重新生成一次 d.cron = cron.NewCron(time.Minute * 118) d.cron.Do(func() { err := d.initSession() if err != nil { log.Errorln("Doge init session error:", err) } + d.client = d.getClient(false) + d.linkClient = d.getClient(true) }) } err := d.initSession()