Skip to content

Commit

Permalink
feat: refactor offline download (#5408 close #4108)
Browse files Browse the repository at this point in the history
* wip: refactor offline download (#5331)

* base tool

* working: aria2

* refactor: change type of percentage to float64

* wip: adapt aria2

* wip: use items in offline_download

* wip: use tool manager

* wip: adapt qBittorrent

* chore: fix typo

* Squashed commit of the following:

commit 4fc0a77
Author: Andy Hsu <i@nn.ci>
Date:   Fri Oct 20 21:06:25 2023 +0800

    fix(baidu_netdisk): upload file > 4GB (close #5392)

commit aaffaee
Author: gmugu <94156510@qq.com>
Date:   Thu Oct 19 19:17:53 2023 +0800

    perf(webdav): support request with cookies (#5391)

commit 8ef8023
Author: NewbieOrange <NewbieOrange@users.noreply.github.com>
Date:   Thu Oct 19 19:17:09 2023 +0800

    fix(aliyundrive_open): upload progress for normal upload (#5398)

commit cdfbe6d
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Wed Oct 18 16:27:07 2023 +0800

    fix: hash gcid empty file (#5394)

commit 94d0287
Author: Andy Hsu <i@nn.ci>
Date:   Sat Oct 14 13:17:51 2023 +0800

    ci: remove `pr-welcome` label when close issue [skip ci]

commit 7f73354
Author: itsHenry <2671230065@qq.com>
Date:   Sat Oct 14 13:12:46 2023 +0800

    feat(cloudreve): support thumbnail (#5373 close #5348)

    * feat(cloudreve): support thumbnail

    * chore: remove unnecessary code

commit b9e192b
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Thu Oct 12 20:57:12 2023 +0800

    fix(115): limit request rate (#5367 close #5275)

    * fix(115):limit request rate

    * chore(115): fix unit of `limit_rate`

    ---------

    Co-authored-by: Andy Hsu <i@nn.ci>

commit 69a98ea
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Wed Oct 11 22:01:55 2023 +0800

    fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.9+incompatible (#5141)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit 1ebc96a
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:32:00 2023 +0800

    fix(wopan): fatal error concurrent map writes (close #5352)

commit 66e2324
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:23:11 2023 +0800

    chore(deps): upgrade dependencies

commit 7600dc2
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:13:58 2023 +0800

    fix(aliyundrive_open): change default api to raw server (close #5358)

commit 8ef89ad
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Tue Oct 10 18:08:27 2023 +0800

    fix(baidu_netdisk): hash and `error 2` (#5356)

    * fix(baidu):hash and error:2

    * fix:invalid memory address

commit 35d6722
Author: jeffmingup <1960588251@qq.com>
Date:   Sun Oct 8 19:29:45 2023 +0800

    fix(onedrive_app): incorrect api on `_accessToken` (#5346)

commit 1a283bb
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Fri Oct 6 16:04:39 2023 +0800

    feat(google_drive): add `hash_info`, `ctime`, `thumbnail` (#5334)

commit a008f54
Author: nkh0472 <67589323+nkh0472@users.noreply.github.com>
Date:   Thu Oct 5 13:10:51 2023 +0800

    docs: minor language improvements (#5329) [skip ci]

* fix: adapt update progress type

* Squashed commit of the following:

commit 65c5ec0
Author: itsHenry <2671230065@qq.com>
Date:   Sat Nov 4 13:35:09 2023 +0800

    feat(cloudreve): folder size count and switch (#5457 close #5395)

commit a632596
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Mon Oct 30 15:11:20 2023 +0800

    fix(deps): update module github.com/charmbracelet/lipgloss to v0.9.1 (#5234)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit 4dff494
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Mon Oct 30 15:10:36 2023 +0800

    fix(deps): update golang.org/x/exp digest to 7918f67 (#5366)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit cc86d6f
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Sun Oct 29 14:45:55 2023 +0800

    fix(deps): update module golang.org/x/net to v0.17.0 [security] (#5370)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit c0f9c8e
Author: Andy Hsu <i@nn.ci>
Date:   Thu Oct 26 19:21:09 2023 +0800

    feat: add ignore direct link params (close #5434)
  • Loading branch information
xhofe authored Nov 6, 2023
1 parent 3bbdd4f commit 769281b
Show file tree
Hide file tree
Showing 48 changed files with 1,238 additions and 258 deletions.
2 changes: 2 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"os"

"github.com/alist-org/alist/v3/cmd/flags"
_ "github.com/alist-org/alist/v3/drivers"
_ "github.com/alist-org/alist/v3/internal/offline_download"
"github.com/spf13/cobra"
)

Expand Down
4 changes: 1 addition & 3 deletions cmd/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"time"

"github.com/alist-org/alist/v3/cmd/flags"
_ "github.com/alist-org/alist/v3/drivers"
"github.com/alist-org/alist/v3/internal/bootstrap"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/pkg/utils"
Expand All @@ -35,8 +34,7 @@ the address is defined in config file`,
utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart)
time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second)
}
bootstrap.InitAria2()
bootstrap.InitQbittorrent()
bootstrap.InitOfflineDownloadTools()
bootstrap.LoadStorages()
if !flags.Debug && !flags.Dev {
gin.SetMode(gin.ReleaseMode)
Expand Down
2 changes: 1 addition & 1 deletion drivers/123/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi
if err != nil {
return err
}
up(j * 100 / chunkCount)
up(float64(j) * 100 / float64(chunkCount))
}
}
// complete s3 upload
Expand Down
2 changes: 1 addition & 1 deletion drivers/189/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F
if err != nil {
return err
}
up(int(i * 100 / count))
up(float64(i) * 100 / float64(count))
}
fileMd5 := hex.EncodeToString(md5Sum.Sum(nil))
sliceMd5 := fileMd5
Expand Down
6 changes: 3 additions & 3 deletions drivers/189pc/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo
if err != nil {
return err
}
up(int(threadG.Success()) * 100 / count)
up(float64(threadG.Success()) * 100 / float64(count))
return nil
})
}
Expand Down Expand Up @@ -676,7 +676,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode
return err
}

up(int(threadG.Success()) * 100 / len(uploadUrls))
up(float64(threadG.Success()) * 100 / float64(len(uploadUrls)))
uploadProgress.UploadParts[i] = ""
return nil
})
Expand Down Expand Up @@ -812,7 +812,7 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model
if _, err := tempFile.Seek(status.GetSize(), io.SeekStart); err != nil {
return nil, err
}
up(int(status.GetSize()/file.GetSize()) * 100)
up(float64(status.GetSize()) / float64(file.GetSize()) * 100)
}

return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId)
Expand Down
5 changes: 3 additions & 2 deletions drivers/aliyundrive/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@ import (
"encoding/base64"
"encoding/hex"
"fmt"
"github.com/alist-org/alist/v3/internal/stream"
"io"
"math"
"math/big"
"net/http"
"os"
"time"

"github.com/alist-org/alist/v3/internal/stream"

"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/driver"
Expand Down Expand Up @@ -304,7 +305,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil
}
res.Body.Close()
if count > 0 {
up(i * 100 / count)
up(float64(i) * 100 / float64(count))
}
}
var resp2 base.Json
Expand Down
2 changes: 1 addition & 1 deletion drivers/aliyundrive_open/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m
return nil, err
}
offset += partSize
up(i * 100 / count)
up(float64(i*100) / float64(count))
}
} else {
log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId)
Expand Down
2 changes: 1 addition & 1 deletion drivers/baidu_netdisk/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
if err != nil {
return err
}
up(int(threadG.Success()) * 100 / len(precreateResp.BlockList))
up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList)))
precreateResp.BlockList[i] = -1
return nil
})
Expand Down
2 changes: 1 addition & 1 deletion drivers/baidu_photo/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
if err != nil {
return err
}
up(int(threadG.Success()) * 100 / len(precreateResp.BlockList))
up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList)))
precreateResp.BlockList[i] = -1
return nil
})
Expand Down
2 changes: 1 addition & 1 deletion drivers/dropbox/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt
_ = res.Body.Close()

if count > 0 {
up((i + 1) * 100 / count)
up(float64(i+1) * 100 / float64(count))
}

offset += byteSize
Expand Down
7 changes: 4 additions & 3 deletions drivers/mega/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ import (
"context"
"errors"
"fmt"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/rclone/rclone/lib/readers"
"io"
"time"

"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/rclone/rclone/lib/readers"

"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
Expand Down Expand Up @@ -169,7 +170,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea
if err != nil {
return err
}
up(id * 100 / u.Chunks())
up(float64(id) * 100 / float64(u.Chunks()))
}

_, err = u.Finish()
Expand Down
2 changes: 1 addition & 1 deletion drivers/mopan/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("upload err,code=%d", resp.StatusCode)
}
up(100 * int(threadG.Success()) / len(parts))
up(100 * float64(threadG.Success()) / float64(len(parts)))
initUpdload.PartInfos[i] = ""
return nil
})
Expand Down
2 changes: 1 addition & 1 deletion drivers/onedrive/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil
return errors.New(string(data))
}
res.Body.Close()
up(int(finish * 100 / stream.GetSize()))
up(float64(finish) * 100 / float64(stream.GetSize()))
}
return nil
}
2 changes: 1 addition & 1 deletion drivers/onedrive_app/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model.
return errors.New(string(data))
}
res.Body.Close()
up(int(finish * 100 / stream.GetSize()))
up(float64(finish) * 100 / float64(stream.GetSize()))
}
return nil
}
2 changes: 1 addition & 1 deletion drivers/quark_uc/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File
}
md5s = append(md5s, m)
partNumber++
up(int(100 * (total - left) / total))
up(100 * float64(total-left) / float64(total))
}
err = d.upCommit(pre, md5s)
if err != nil {
Expand Down
5 changes: 3 additions & 2 deletions drivers/s3/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ import (
"bytes"
"context"
"fmt"
"github.com/alist-org/alist/v3/internal/stream"
"io"
"net/url"
stdpath "path"
"strings"
"time"

"github.com/alist-org/alist/v3/internal/stream"

"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/aws/aws-sdk-go/aws/session"
Expand Down Expand Up @@ -104,7 +105,7 @@ func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) e
},
Reader: io.NopCloser(bytes.NewReader([]byte{})),
Mimetype: "application/octet-stream",
}, func(int) {})
}, func(float64) {})
}

func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
Expand Down
2 changes: 1 addition & 1 deletion drivers/teambition/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ func (d *Teambition) chunkUpload(ctx context.Context, file model.FileStreamer, t
if err != nil {
return nil, err
}
up(i * 100 / newChunk.Chunks)
up(float64(i) * 100 / float64(newChunk.Chunks))
}
_, err = base.RestyClient.R().SetHeader("Authorization", token).Post(
fmt.Sprintf("https://%s.teambition.net/upload/chunk/%s",
Expand Down
2 changes: 1 addition & 1 deletion drivers/terabox/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt
}
log.Debugln(res.String())
if len(precreateResp.BlockList) > 0 {
up(i * 100 / len(precreateResp.BlockList))
up(float64(i) * 100 / float64(len(precreateResp.BlockList)))
}
}
_, err = d.create(rawPath, stream.GetSize(), 0, precreateResp.Uploadid, block_list_str)
Expand Down
3 changes: 1 addition & 2 deletions drivers/trainbit/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"io"
"math"
"net/http"
"net/url"
"strings"
Expand Down Expand Up @@ -128,7 +127,7 @@ func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, stream model.FileS
stream,
func(byteNum int) {
total += int64(byteNum)
up(int(math.Round(float64(total) / float64(stream.GetSize()) * 100)))
up(float64(total) / float64(stream.GetSize()) * 100)
},
}
req, err := http.NewRequest(http.MethodPost, endpoint.String(), progressReader)
Expand Down
2 changes: 1 addition & 1 deletion drivers/wopan/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre
ContentType: stream.GetMimetype(),
}, dstDir.GetID(), d.FamilyID, wopan.Upload2COption{
OnProgress: func(current, total int64) {
up(int(100 * current / total))
up(100 * float64(current) / float64(total))
},
})
return err
Expand Down
5 changes: 3 additions & 2 deletions internal/aria2/monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package aria2

import (
"fmt"
"github.com/alist-org/alist/v3/internal/stream"
"os"
"path"
"path/filepath"
Expand All @@ -11,6 +10,8 @@ import (
"sync/atomic"
"time"

"github.com/alist-org/alist/v3/internal/stream"

"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/task"
Expand Down Expand Up @@ -100,7 +101,7 @@ func (m *Monitor) Update() (bool, error) {
downloaded = 0
}
progress := float64(downloaded) / float64(total) * 100
m.tsk.SetProgress(int(progress))
m.tsk.SetProgress(progress)
switch info.Status {
case "complete":
err := m.Complete()
Expand Down
16 changes: 0 additions & 16 deletions internal/bootstrap/aria2.go

This file was deleted.

10 changes: 2 additions & 8 deletions internal/bootstrap/data/setting.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"github.com/alist-org/alist/v3/cmd/flags"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/offline_download/tool"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/pkg/utils/random"
Expand Down Expand Up @@ -142,10 +143,6 @@ func InitialSettings() []model.SettingItem {
{Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL},
{Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},

// aria2 settings
{Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
{Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},

// single settings
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX},
Expand All @@ -168,11 +165,8 @@ func InitialSettings() []model.SettingItem {
{Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE},
{Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE},
{Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC},

// qbittorrent settings
{Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
{Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE},
}
initialSettingItems = append(initialSettingItems, tool.Tools.Items()...)
if flags.Dev {
initialSettingItems = append(initialSettingItems, []model.SettingItem{
{Key: "test_deprecated", Value: "test_value", Type: conf.TypeString, Flag: model.DEPRECATED},
Expand Down
17 changes: 17 additions & 0 deletions internal/bootstrap/offline_download.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package bootstrap

import (
"github.com/alist-org/alist/v3/internal/offline_download/tool"
"github.com/alist-org/alist/v3/pkg/utils"
)

func InitOfflineDownloadTools() {
for k, v := range tool.Tools {
res, err := v.Init()
if err != nil {
utils.Log.Warnf("init tool %s failed: %s", k, err)
} else {
utils.Log.Infof("init tool %s success: %s", k, res)
}
}
}
15 changes: 0 additions & 15 deletions internal/bootstrap/qbittorrent.go

This file was deleted.

4 changes: 2 additions & 2 deletions internal/driver/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ type PutResult interface {
Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error)
}

type UpdateProgress func(percentage int)
type UpdateProgress func(percentage float64)

type Progress struct {
Total int64
Expand All @@ -120,7 +120,7 @@ type Progress struct {
func (p *Progress) Write(b []byte) (n int, err error) {
n = len(b)
p.Done += int64(n)
p.up(int(float64(p.Done) / float64(p.Total) * 100))
p.up(float64(p.Done) / float64(p.Total) * 100)
return
}

Expand Down
2 changes: 1 addition & 1 deletion internal/model/setting.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const (
STYLE
PREVIEW
GLOBAL
ARIA2
OFFLINE_DOWNLOAD
INDEX
SSO
)
Expand Down
Loading

0 comments on commit 769281b

Please sign in to comment.