From 36582e77bd9abf0310861a308f0bb9699af446bc Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Thu, 18 Feb 2021 04:24:09 +0500 Subject: [PATCH 01/13] Add LFS data mirroring --- modules/forms/repo_form.go | 1 + modules/migrations/base/options.go | 1 + modules/migrations/gitea_uploader.go | 1 + modules/repository/repo.go | 71 ++++++++++++++++++++++++++++ modules/structs/repo.go | 1 + options/locale/locale_en-US.ini | 4 +- routers/api/v1/repo/migrate.go | 1 + routers/repo/migrate.go | 2 + templates/repo/migrate/git.tmpl | 15 ++++-- templates/repo/migrate/gitea.tmpl | 13 +++-- templates/repo/migrate/github.tmpl | 15 ++++-- templates/repo/migrate/gitlab.tmpl | 15 ++++-- templates/repo/migrate/gogs.tmpl | 13 +++-- templates/swagger/v1_json.tmpl | 8 ++++ 14 files changed, 136 insertions(+), 25 deletions(-) diff --git a/modules/forms/repo_form.go b/modules/forms/repo_form.go index 48af3450f371d..d99bb835c2a1a 100644 --- a/modules/forms/repo_form.go +++ b/modules/forms/repo_form.go @@ -73,6 +73,7 @@ type MigrateRepoForm struct { // required: true RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` Private bool `json:"private"` Description string `json:"description" binding:"MaxSize(255)"` Wiki bool `json:"wiki"` diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index 168f9848c813d..92b23ca51b43a 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -20,6 +20,7 @@ type MigrateOptions struct { // required: true RepoName string `json:"repo_name" binding:"Required"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` Private bool `json:"private"` Description string `json:"description"` OriginalURL string diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 3be49b5c6cedb..6d2e70994297c 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -117,6 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate OriginalURL: repo.OriginalURL, GitServiceType: opts.GitServiceType, Mirror: repo.IsMirror, + LFS: opts.LFS, CloneAddr: repo.CloneURL, Private: repo.IsPrivate, Wiki: opts.Wiki, diff --git a/modules/repository/repo.go b/modules/repository/repo.go index ede714673ab16..a1f7c996bf1bf 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -7,7 +7,9 @@ package repository import ( "context" "fmt" + "os" "path" + "path/filepath" "strings" "time" @@ -70,6 +72,75 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. return repo, fmt.Errorf("Clone: %v", err) } + if opts.LFS { + _, err = git.NewCommand("lfs", "fetch", opts.CloneAddr).RunInDir(repoPath) + if err != nil { + return repo, fmt.Errorf("LFS fetch failed %s: %v", opts.CloneAddr, err) + } + + lfsSrc := path.Join(repoPath, "lfs", "objects") + lfsDst := path.Join(setting.LFS.Path) + + // move LFS files + err := filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error { + var relSrcPath = strings.Replace(path, lfsSrc, "", 1) + if relSrcPath == "" { + return nil + } + if err != nil { + return err + } + lfsSrcFull := filepath.Join(lfsSrc, relSrcPath) + lfsDstFull := filepath.Join(lfsDst, relSrcPath) + + if _, err := os.Stat(lfsDstFull); !os.IsNotExist(err) { + return nil + } + + if info.IsDir() { + return os.Mkdir(lfsDstFull, 0755) + } + + // generate and associate LFS OIDs + file, err := os.Open(lfsSrcFull) + if err != nil { + return err + } + defer file.Close() + + oid, err := models.GenerateLFSOid(file) + if err != nil { + return err + } + fileInfo, err := file.Stat() + if err != nil { + return err + } + + lfsDstFull = filepath.Join(lfsDst, oid[0:2], oid[2:4], oid[4:]) + err = os.Rename(lfsSrcFull, lfsDstFull) + if err != nil { + return err + } + + _, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: repo.ID}) + if err != nil { + log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", oid, fileInfo.Size(), u.Name, repoPath, err) + return err + } + + return nil + }) + if err != nil { + return repo, fmt.Errorf("Failed to move LFS files %s: %v", lfsSrc, err) + } + + err = os.RemoveAll(path.Join(repoPath, "lfs")) + if err != nil { + return repo, fmt.Errorf("Failed to remove LFS files %s: %v", repoPath, err) + } + } + if opts.Wiki { wikiPath := models.WikiPath(u.Name, opts.RepoName) wikiRemotePath := WikiRemoteURL(opts.CloneAddr) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index d588813b21883..5bd5314cb1a4c 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -253,6 +253,7 @@ type MigrateRepoOptions struct { AuthToken string `json:"auth_token"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` Private bool `json:"private"` Description string `json:"description" binding:"MaxSize(255)"` Wiki bool `json:"wiki"` diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 767696cfb901c..e6df0c9288648 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -754,6 +754,7 @@ need_auth = Clone Authorization migrate_options = Migration Options migrate_service = Migration Service migrate_options_mirror_helper = This repository will be a mirror +migrate_options_mirror_lfs = Mirror LFS data migrate_options_mirror_disabled = Your site administrator has disabled new mirrors. migrate_items = Migration Items migrate_items_wiki = Wiki @@ -770,7 +771,6 @@ migrate.clone_local_path = or a local server path migrate.permission_denied = You are not allowed to import local repositories. migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory." migrate.failed = Migration failed: %v -migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead. migrate.migrate_items_options = Access Token is required to migrate additional items migrated_from = Migrated from %[2]s migrated_from_fake = Migrated From %[1]s @@ -931,7 +931,7 @@ ext_issues = Ext. Issues ext_issues.desc = Link to an external issue tracker. projects = Projects -projects.desc = Manage issues and pulls in project boards. +projects.desc = Manage issues and pulls in project boards. projects.description = Description (optional) projects.description_placeholder = Description projects.create = Create Project diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index 61cd12b991cb1..f4c5b0ee1b7b8 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -134,6 +134,7 @@ func Migrate(ctx *context.APIContext) { Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, + LFS: form.LFS, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 89452de0fabfb..15df1dfdee884 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -47,6 +47,7 @@ func Migrate(ctx *context.Context) { ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors ctx.Data["mirror"] = ctx.Query("mirror") == "1" + ctx.Data["LFS"] = ctx.Query("lfs") == "1" ctx.Data["wiki"] = ctx.Query("wiki") == "1" ctx.Data["milestones"] = ctx.Query("milestones") == "1" ctx.Data["labels"] = ctx.Query("labels") == "1" @@ -172,6 +173,7 @@ func MigratePost(ctx *context.Context) { Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror && !setting.Repository.DisableMirrors, + LFS: form.LFS, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/templates/repo/migrate/git.tmpl b/templates/repo/migrate/git.tmpl index 233a019435308..26d2311471209 100644 --- a/templates/repo/migrate/git.tmpl +++ b/templates/repo/migrate/git.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
@@ -30,15 +29,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} - +
+ {{else}} +
+ - {{end}}
+
+ + +
+ {{end}}
diff --git a/templates/repo/migrate/gitea.tmpl b/templates/repo/migrate/gitea.tmpl index b21e6b18ffdfc..b984e54cd69d6 100644 --- a/templates/repo/migrate/gitea.tmpl +++ b/templates/repo/migrate/gitea.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
@@ -27,15 +26,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} +
+ {{else}} +
- {{end}}
+
+ + +
+ {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/repo/migrate/github.tmpl b/templates/repo/migrate/github.tmpl index 06f76d72980e6..f0103f74a9c62 100644 --- a/templates/repo/migrate/github.tmpl +++ b/templates/repo/migrate/github.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} @@ -27,15 +26,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} - +
+ {{else}} +
+ - {{end}}
+
+ + +
+ {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/repo/migrate/gitlab.tmpl b/templates/repo/migrate/gitlab.tmpl index 545a1ff43717f..ff8f3637370d6 100644 --- a/templates/repo/migrate/gitlab.tmpl +++ b/templates/repo/migrate/gitlab.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} @@ -27,15 +26,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} - +
+ {{else}} +
+ - {{end}}
+
+ + +
+ {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/repo/migrate/gogs.tmpl b/templates/repo/migrate/gogs.tmpl index ac81872b92270..84156310aba18 100644 --- a/templates/repo/migrate/gogs.tmpl +++ b/templates/repo/migrate/gogs.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} @@ -27,15 +26,21 @@
+ {{if .DisableMirrors}}
- {{if .DisableMirrors}} - {{else}} +
+ {{else}} +
- {{end}}
+
+ + +
+ {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 2dedb56d1ec25..21f3db90a5c15 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -14636,6 +14636,10 @@ "type": "boolean", "x-go-name": "Labels" }, + "lfs": { + "type": "boolean", + "x-go-name": "LFS" + }, "milestones": { "type": "boolean", "x-go-name": "Milestones" @@ -14715,6 +14719,10 @@ "type": "boolean", "x-go-name": "Labels" }, + "lfs": { + "type": "boolean", + "x-go-name": "LFS" + }, "milestones": { "type": "boolean", "x-go-name": "Milestones" From e7941dea81c03ba2c33a6e5b60b3830f05af32e0 Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Thu, 18 Feb 2021 05:57:32 +0500 Subject: [PATCH 02/13] amend for gogs --- options/locale/locale_en-US.ini | 1 + templates/repo/migrate/gogs.tmpl | 13 ++++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e6df0c9288648..60680ad425840 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -771,6 +771,7 @@ migrate.clone_local_path = or a local server path migrate.permission_denied = You are not allowed to import local repositories. migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory." migrate.failed = Migration failed: %v +migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead. migrate.migrate_items_options = Access Token is required to migrate additional items migrated_from = Migrated from %[2]s migrated_from_fake = Migrated From %[1]s diff --git a/templates/repo/migrate/gogs.tmpl b/templates/repo/migrate/gogs.tmpl index 84156310aba18..ac81872b92270 100644 --- a/templates/repo/migrate/gogs.tmpl +++ b/templates/repo/migrate/gogs.tmpl @@ -15,6 +15,7 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} + {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} @@ -26,21 +27,15 @@
- {{if .DisableMirrors}}
+ {{if .DisableMirrors}} -
- {{else}} -
+ {{else}} + {{end}}
-
- - -
- {{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} From a72732405585405719c95d93ce3a16759c0ef2bc Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Fri, 19 Feb 2021 09:37:41 +0500 Subject: [PATCH 03/13] add "git-lfs not installed" error --- models/error.go | 14 ++++++++++++++ models/repo.go | 9 ++++++++- options/locale/locale_en-US.ini | 1 + routers/repo/migrate.go | 5 ++++- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/models/error.go b/models/error.go index fc161ed806f33..d3270aef39c03 100644 --- a/models/error.go +++ b/models/error.go @@ -56,6 +56,20 @@ func (err ErrNamePatternNotAllowed) Error() string { return fmt.Sprintf("name pattern is not allowed [pattern: %s]", err.Pattern) } +// ErrLFSNotInstalled represents an "git-lfs not found" error. +type ErrLFSNotInstalled struct { +} + +// IsLFSNotInstalled checks if an error is an ErrLFSNotInstalled. +func IsLFSNotInstalled(err error) bool { + _, ok := err.(ErrLFSNotInstalled) + return ok +} + +func (err ErrLFSNotInstalled) Error() string { + return "git-lfs not found" +} + // ErrNameCharsNotAllowed represents a "character not allowed in name" error. type ErrNameCharsNotAllowed struct { Name string diff --git a/models/repo.go b/models/repo.go index 62d64fbee9183..08fee4b547672 100644 --- a/models/repo.go +++ b/models/repo.go @@ -18,6 +18,7 @@ import ( "net" "net/url" "os" + "os/exec" "path" "path/filepath" "sort" @@ -935,7 +936,7 @@ func (repo *Repository) CloneLink() (cl *CloneLink) { } // CheckCreateRepository check if could created a repository -func CheckCreateRepository(doer, u *User, name string, overwriteOrAdopt bool) error { +func CheckCreateRepository(doer, u *User, name string, lfs bool, overwriteOrAdopt bool) error { if !doer.CanCreateRepo() { return ErrReachLimitOfRepo{u.MaxRepoCreation} } @@ -959,6 +960,12 @@ func CheckCreateRepository(doer, u *User, name string, overwriteOrAdopt bool) er if !overwriteOrAdopt && isExist { return ErrRepoFilesAlreadyExist{u.Name, name} } + + if lfs { + if _, err := exec.LookPath("git-lfs"); err != nil { + return ErrLFSNotInstalled{} + } + } return nil } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index b07dfb08f09ea..00aa059821c38 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -392,6 +392,7 @@ user_not_exist = The user does not exist. team_not_exist = The team does not exist. last_org_owner = You cannot remove the last user from the 'owners' team. There must be at least one owner for an organization. cannot_add_org_to_team = An organization cannot be added as a team member. +lfs_not_installed = Please install git-lfs. invalid_ssh_key = Can not verify your SSH key: %s invalid_gpg_key = Can not verify your GPG key: %s diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 15df1dfdee884..06fb53e7244de 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -101,6 +101,9 @@ func handleMigrateError(ctx *context.Context, owner *models.User, err error, nam case models.IsErrNamePatternNotAllowed(err): ctx.Data["Err_RepoName"] = true ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form) + case models.IsLFSNotInstalled(err): + ctx.Data["Err_LFSNotInstalled"] = true + ctx.RenderWithErr(ctx.Tr("form.lfs_not_installed"), tpl, form) default: remoteAddr, _ := auth.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword, owner) err = util.URLSanitizedError(err, remoteAddr) @@ -194,7 +197,7 @@ func MigratePost(ctx *context.Context) { opts.Releases = false } - err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName, false) + err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName, opts.LFS, false) if err != nil { handleMigrateError(ctx, ctxUser, err, "MigratePost", tpl, form) return From 3b991479fdbaadab20b4c655182576b66d50d3fa Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Fri, 19 Feb 2021 12:10:05 +0500 Subject: [PATCH 04/13] fetch lfs to gitea setting.LFS.Path/tmp --- modules/repository/repo.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index a1f7c996bf1bf..b8d3ca0dd887e 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -73,15 +73,21 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. } if opts.LFS { + lfsFetchDir := filepath.Join(setting.LFS.Path, "tmp") + _, err = git.NewCommand("config", "lfs.storage", lfsFetchDir).RunInDir(repoPath) + if err != nil { + return repo, fmt.Errorf("Failed `git config lfs.storage lfsFetchDir`: %v", err) + } + _, err = git.NewCommand("lfs", "fetch", opts.CloneAddr).RunInDir(repoPath) if err != nil { - return repo, fmt.Errorf("LFS fetch failed %s: %v", opts.CloneAddr, err) + return repo, fmt.Errorf("Failed `lfs fetch` %s: %v", opts.CloneAddr, err) } - lfsSrc := path.Join(repoPath, "lfs", "objects") + lfsSrc := path.Join(lfsFetchDir, "objects") lfsDst := path.Join(setting.LFS.Path) - // move LFS files + // rename LFS files err := filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error { var relSrcPath = strings.Replace(path, lfsSrc, "", 1) if relSrcPath == "" { @@ -135,7 +141,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. return repo, fmt.Errorf("Failed to move LFS files %s: %v", lfsSrc, err) } - err = os.RemoveAll(path.Join(repoPath, "lfs")) + err = os.RemoveAll(lfsFetchDir) if err != nil { return repo, fmt.Errorf("Failed to remove LFS files %s: %v", repoPath, err) } From 7c744c1400f1532e673d4bedc58eb37205c7011a Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Sat, 20 Feb 2021 04:32:08 +0500 Subject: [PATCH 05/13] add size restrictions --- modules/repository/repo.go | 103 ++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index b8d3ca0dd887e..9a50fb1c4d0ea 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "time" @@ -72,6 +73,34 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. return repo, fmt.Errorf("Clone: %v", err) } + if opts.Wiki { + wikiPath := models.WikiPath(u.Name, opts.RepoName) + wikiRemotePath := WikiRemoteURL(opts.CloneAddr) + if len(wikiRemotePath) > 0 { + if err := util.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + + if err = git.CloneWithContext(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + Branch: "master", + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := util.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + } + } + } + + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + return repo, fmt.Errorf("OpenRepository: %v", err) + } + defer gitRepo.Close() + if opts.LFS { lfsFetchDir := filepath.Join(setting.LFS.Path, "tmp") _, err = git.NewCommand("config", "lfs.storage", lfsFetchDir).RunInDir(repoPath) @@ -79,7 +108,47 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. return repo, fmt.Errorf("Failed `git config lfs.storage lfsFetchDir`: %v", err) } - _, err = git.NewCommand("lfs", "fetch", opts.CloneAddr).RunInDir(repoPath) + excludedPointersPaths := []string{} + + // scan repo for pointer files, exclude those exceeding MaxFileSize from being downloaded + if setting.LFS.MaxFileSize > 0 { + headBranch, _ := gitRepo.GetHEADBranch() + headCommit, _ := gitRepo.GetCommit(headBranch.Name) + entries, err := headCommit.Tree.ListEntriesRecursive() + if err != nil { + return repo, fmt.Errorf("Failed to access git files: %v", err) + } + + for _, entry := range entries { + entryString, _ := entry.Blob().GetBlobContent() + + if !strings.HasPrefix(entryString, models.LFSMetaFileIdentifier) { + continue + } + + splitLines := strings.Split(entryString, "\n") + if len(splitLines) < 3 { + continue + } + + size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) + if err != nil { + continue + } + + if size > setting.LFS.MaxFileSize { + excludedPointersPaths = append(excludedPointersPaths, entry.Name()) + log.Info("Denied LFS[%s] download of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", entry.Name(), entry.Blob().Size(), u.Name, repo.Name, setting.LFS.MaxFileSize) + } + } + } + + // fetch LFS files + cmd := git.NewCommand("lfs", "fetch", opts.CloneAddr) + if len(excludedPointersPaths) > 0 { + cmd.AddArguments("--exclude", strings.Join(excludedPointersPaths, ",")) + } + _, err = cmd.RunInDir(repoPath) if err != nil { return repo, fmt.Errorf("Failed `lfs fetch` %s: %v", opts.CloneAddr, err) } @@ -88,7 +157,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. lfsDst := path.Join(setting.LFS.Path) // rename LFS files - err := filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error { var relSrcPath = strings.Replace(path, lfsSrc, "", 1) if relSrcPath == "" { return nil @@ -143,38 +212,10 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. err = os.RemoveAll(lfsFetchDir) if err != nil { - return repo, fmt.Errorf("Failed to remove LFS files %s: %v", repoPath, err) - } - } - - if opts.Wiki { - wikiPath := models.WikiPath(u.Name, opts.RepoName) - wikiRemotePath := WikiRemoteURL(opts.CloneAddr) - if len(wikiRemotePath) > 0 { - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) - } - - if err = git.CloneWithContext(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - Branch: "master", - }); err != nil { - log.Warn("Clone wiki: %v", err) - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) - } - } + return repo, fmt.Errorf("Failed to delete temporary LFS directories %s: %v", repoPath, err) } } - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - return repo, fmt.Errorf("OpenRepository: %v", err) - } - defer gitRepo.Close() - repo.IsEmpty, err = gitRepo.IsEmpty() if err != nil { return repo, fmt.Errorf("git.IsEmpty: %v", err) From cd1dfbb23cc5c59b980b832b12a65b78dd9bf9ef Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Sun, 28 Feb 2021 10:42:20 +0500 Subject: [PATCH 06/13] Rewritten on LFS API --- models/lfs.go | 6 + modules/forms/repo_form.go | 2 + modules/lfs/client.go | 143 ++++++++++++++++++++ modules/migrations/base/options.go | 2 + modules/migrations/gitea_uploader.go | 2 + modules/repository/repo.go | 188 +++++++++++---------------- modules/structs/repo.go | 2 + options/locale/locale_en-US.ini | 2 + routers/api/v1/repo/migrate.go | 2 + routers/repo/migrate.go | 3 + templates/repo/migrate/github.tmpl | 15 +++ web_src/js/features/migration.js | 24 +++- 12 files changed, 276 insertions(+), 115 deletions(-) create mode 100644 modules/lfs/client.go diff --git a/models/lfs.go b/models/lfs.go index 274b32a736758..9e7fcb07e84fa 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -27,6 +27,12 @@ type LFSMetaObject struct { CreatedUnix timeutil.TimeStamp `xorm:"created"` } +// LFSMetaObjectBasic represents basic LFS metadata. +type LFSMetaObjectBasic struct { + Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 `xorm:"NOT NULL"` +} + // RelativePath returns the relative path of the lfs object func (m *LFSMetaObject) RelativePath() string { if len(m.Oid) < 5 { diff --git a/modules/forms/repo_form.go b/modules/forms/repo_form.go index d99bb835c2a1a..cb0110370708b 100644 --- a/modules/forms/repo_form.go +++ b/modules/forms/repo_form.go @@ -74,6 +74,8 @@ type MigrateRepoForm struct { RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"` Mirror bool `json:"mirror"` LFS bool `json:"lfs"` + LFSServer string `json:"lfs_server"` + LFSFetchOlder bool `json:"lfs_fetch_older"` Private bool `json:"private"` Description string `json:"description" binding:"MaxSize(255)"` Wiki bool `json:"wiki"` diff --git a/modules/lfs/client.go b/modules/lfs/client.go new file mode 100644 index 0000000000000..34c82a4799725 --- /dev/null +++ b/modules/lfs/client.go @@ -0,0 +1,143 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "encoding/json" + "bytes" + "fmt" + "io" + "net/http" + "context" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" +) + +type BatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers,omitempty"` + Ref *Reference `json:"ref,omitempty"` + Objects []*models.LFSMetaObjectBasic `json:"objects"` +} + +type Reference struct { + Name string `json:"name"` +} + +func packbatch(operation string, transfers []string, ref *Reference, metaObjects []*models.LFSMetaObject) (string, error) { + metaObjectsBasic := []*models.LFSMetaObjectBasic{} + for _, meta := range metaObjects { + metaBasic := &LFSMetaObjectBasic{meta.oid, meta.size} + metaObjectsBasic = append(metaObjectsBasic, metaBasic) + } + + reqobj := &BatchRequest{operation, transfers, ref, metaObjectsBasic} + + buf := &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(reqobj); err != nil { + return nil, fmt.Errorf("Failed to encode BatchRequest as json. Error: %v", err) + } + return buf, nil +} + +func BasicTransferAdapter(href string, size int64) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, href, nil) + if err != nil { + return err + } + req.Header.Set("Content-type", "application/octet-stream") + req.Header.Set("Content-Length", size) + + resp, err := client.Do(req) + if err != nil { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return decodeJSONError(resp).Err + } + return resp.Body +} + +func FetchLFSFilesToContentStore(ctx *context.Context, metaObjects []*models.LFSMetaObject, userName string, repo *models.Repository, LFSServer string) error { + client := http.DefaultClient + + rv, err := packbatch("download", nil, nil, metaObjects) + if err != nil { + return err + } + batchAPIURL := LFSServer + "/objects/batch" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, batchAPIURL, rv) + if err != nil { + return err + } + req.Header.Set("Content-type", metaMediaType) + req.Header.Set("Accept", metaMediaType) + + resp, err := client.Do(req) + if err != nil { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return decodeJSONError(resp).Err + } + + var respBatch BatchResponse + err = json.NewDecoder(resp.Body).Decode(&respBatch) + if err != nil { + return err + } + + if respBatch.Transfer == nil { + respBatch.Transfer = "basic" + } + + contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + + for _, rep := range respBatch.Objects { + rc := BasicTransferAdapter(rep.Actions["download"].Href, rep.Size) + meta, err := GetLFSMetaObjectByOid(rep.Oid) + if err != nil { + log.Error("Unable to get LFS OID[%s] Error: %v", rep.Oid, err) + return err + } + + // put LFS file to contentStore + exist, err := contentStore.Exists(meta) + if err != nil { + log.Error("Unable to check if LFS OID[%s] exist on %s/%s. Error: %v", meta.Oid, userName, repo.Name, err) + return err + } + + if exist { + // remove collision + if _, err := repo.RemoveLFSMetaObjectByOid(meta.Oid); err != nil { + return fmt.Errorf("Error whilst removing matched LFS object %s: %v", meta.Oid, err) + } + } + + if err := contentStore.Put(meta, rc); err != nil { + if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { + return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", meta.Oid, err2, err) + } + return err + } + } +} diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index 92b23ca51b43a..2adb772572356 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -21,6 +21,8 @@ type MigrateOptions struct { RepoName string `json:"repo_name" binding:"Required"` Mirror bool `json:"mirror"` LFS bool `json:"lfs"` + LFSServer string `json:"lfs_server"` + LFSFetchOlder bool `json:"lfs_fetch_older"` Private bool `json:"private"` Description string `json:"description"` OriginalURL string diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 6d2e70994297c..cb9e5a5466871 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -118,6 +118,8 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate GitServiceType: opts.GitServiceType, Mirror: repo.IsMirror, LFS: opts.LFS, + LFSServer: opts.LFSServer, + LFSFetchOlder: opts.LFSFetchOlder, CloneAddr: repo.CloneURL, Private: repo.IsPrivate, Wiki: opts.Wiki, diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 9a50fb1c4d0ea..ebec6706465c9 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -7,10 +7,7 @@ package repository import ( "context" "fmt" - "os" "path" - "path/filepath" - "strconv" "strings" "time" @@ -21,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/lfsclient" "gopkg.in/ini.v1" ) @@ -102,117 +100,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. defer gitRepo.Close() if opts.LFS { - lfsFetchDir := filepath.Join(setting.LFS.Path, "tmp") - _, err = git.NewCommand("config", "lfs.storage", lfsFetchDir).RunInDir(repoPath) + err := FetchLFSFilesToContentStore(ctx, repo, u.Name, gitRepo, opts.LFSServer, opts.LFSFetchOlder) if err != nil { - return repo, fmt.Errorf("Failed `git config lfs.storage lfsFetchDir`: %v", err) - } - - excludedPointersPaths := []string{} - - // scan repo for pointer files, exclude those exceeding MaxFileSize from being downloaded - if setting.LFS.MaxFileSize > 0 { - headBranch, _ := gitRepo.GetHEADBranch() - headCommit, _ := gitRepo.GetCommit(headBranch.Name) - entries, err := headCommit.Tree.ListEntriesRecursive() - if err != nil { - return repo, fmt.Errorf("Failed to access git files: %v", err) - } - - for _, entry := range entries { - entryString, _ := entry.Blob().GetBlobContent() - - if !strings.HasPrefix(entryString, models.LFSMetaFileIdentifier) { - continue - } - - splitLines := strings.Split(entryString, "\n") - if len(splitLines) < 3 { - continue - } - - size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) - if err != nil { - continue - } - - if size > setting.LFS.MaxFileSize { - excludedPointersPaths = append(excludedPointersPaths, entry.Name()) - log.Info("Denied LFS[%s] download of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", entry.Name(), entry.Blob().Size(), u.Name, repo.Name, setting.LFS.MaxFileSize) - } - } - } - - // fetch LFS files - cmd := git.NewCommand("lfs", "fetch", opts.CloneAddr) - if len(excludedPointersPaths) > 0 { - cmd.AddArguments("--exclude", strings.Join(excludedPointersPaths, ",")) - } - _, err = cmd.RunInDir(repoPath) - if err != nil { - return repo, fmt.Errorf("Failed `lfs fetch` %s: %v", opts.CloneAddr, err) - } - - lfsSrc := path.Join(lfsFetchDir, "objects") - lfsDst := path.Join(setting.LFS.Path) - - // rename LFS files - err = filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error { - var relSrcPath = strings.Replace(path, lfsSrc, "", 1) - if relSrcPath == "" { - return nil - } - if err != nil { - return err - } - lfsSrcFull := filepath.Join(lfsSrc, relSrcPath) - lfsDstFull := filepath.Join(lfsDst, relSrcPath) - - if _, err := os.Stat(lfsDstFull); !os.IsNotExist(err) { - return nil - } - - if info.IsDir() { - return os.Mkdir(lfsDstFull, 0755) - } - - // generate and associate LFS OIDs - file, err := os.Open(lfsSrcFull) - if err != nil { - return err - } - defer file.Close() - - oid, err := models.GenerateLFSOid(file) - if err != nil { - return err - } - fileInfo, err := file.Stat() - if err != nil { - return err - } - - lfsDstFull = filepath.Join(lfsDst, oid[0:2], oid[2:4], oid[4:]) - err = os.Rename(lfsSrcFull, lfsDstFull) - if err != nil { - return err - } - - _, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: repo.ID}) - if err != nil { - log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", oid, fileInfo.Size(), u.Name, repoPath, err) - return err - } - - return nil - }) - if err != nil { - return repo, fmt.Errorf("Failed to move LFS files %s: %v", lfsSrc, err) - } - - err = os.RemoveAll(lfsFetchDir) - if err != nil { - return repo, fmt.Errorf("Failed to delete temporary LFS directories %s: %v", repoPath, err) + return repo, fmt.Errorf("Failed to fetch LFS files: %v", err) } } @@ -284,6 +174,78 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. return repo, err } +func FetchLFSFilesToContentStore(ctx context.Context, repo *models.Repository, userName string, gitRepo *git.Repository, LFSServer string, LFSFetchOlder bool) error { + fetchingMetaObjects := []*models.LFSMetaObject{} + var err error + + // scan repo for pointer files + headBranch, _ := gitRepo.GetHEADBranch() + headCommit, _ := gitRepo.GetCommit(headBranch.Name) + + err = FindLFSMetaObjectsBelowMaxFileSize(headCommit, userName, repo, &fetchingMetaObjects) + if err != nil { + log.Error("Failed to access git LFS meta objects on commit %s: %v", headCommit.ID.String(), err) + return err + } + + if LFSFetchOlder { + opts := git.NewSearchCommitsOptions("before:" + headCommit.ID.String(), true) + commitIDsList, _ := headCommit.SearchCommits(opts) + var commitIDs = []string{} + for e := commitIDsList.Front(); e != nil; e = e.Next() { + commitIDs = append(commitIDs, e.Value.(string)) + } + commitsList := gitRepo.GetCommitsFromIDs(commitIDs) + + for e := commitsList.Front(); e != nil; e = e.Next() { + commit := e.Value.(*git.Commit) + err = FindLFSMetaObjectsBelowMaxFileSize(commit, userName, repo, &fetchingMetaObjects) + if err != nil { + log.Error("Failed to access git LFS meta objects on commit %s: %v", commit.ID.String(), err) + return err + } + } + } + + // fetch LFS files from external server + err = lfsclient.FetchLFSFilesToContentStore(ctx, fetchingMetaObjects, userName, repo, LFSServer) + if err != nil { + log.Error("Unable to fetch LFS files in %v/%v to content store. Error: %v", userName, repo.Name, err) + return err + } + + return nil +} + +func FindLFSMetaObjectsBelowMaxFileSize(commit *git.Commit, userName string, repo *models.Repository, fetchingMetaObjects *[]*models.LFSMetaObject) error { + entries, err := commit.Tree.ListEntriesRecursive() + if err != nil { + log.Error("Failed to access git commit %s tree: %v", commit.ID.String(), err) + return err + } + + for _, entry := range entries { + buf, _ := entry.Blob().GetBlobFirstBytes(1024) + meta := models.IsPointerFile(&buf) + if meta == nil { + continue + } + + if setting.LFS.MaxFileSize > 0 && meta.Size > setting.LFS.MaxFileSize { + log.Info("Denied LFS oid[%s] download of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", meta.Oid, meta.Size, userName, repo.Name, setting.LFS.MaxFileSize) + continue + } + + meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: meta.Oid, Size: meta.Size, RepositoryID: repo.ID}) + if err != nil { + log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", meta.Oid, meta.Size, userName, repo.Name, err) + return err + } + *fetchingMetaObjects = append(*fetchingMetaObjects, meta) + } + return nil +} + // cleanUpMigrateGitConfig removes mirror info which prevents "push --all". // This also removes possible user credentials. func cleanUpMigrateGitConfig(configPath string) error { diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 5bd5314cb1a4c..c76a143187201 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -254,6 +254,8 @@ type MigrateRepoOptions struct { Mirror bool `json:"mirror"` LFS bool `json:"lfs"` + LFSServer string `json:"lfs_server"` + LFSFetchOlder bool `json:"lfs_fetch_older"` Private bool `json:"private"` Description string `json:"description" binding:"MaxSize(255)"` Wiki bool `json:"wiki"` diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 5a5a1c4eeb727..cf8b17fef0072 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -759,6 +759,8 @@ migrate_service = Migration Service migrate_options_mirror_helper = This repository will be a mirror migrate_options_mirror_lfs = Mirror LFS data migrate_options_mirror_disabled = Your site administrator has disabled new mirrors. +migrate_options_lfs_server = LFS Server +migrate_options_lfs_fetch_older = Fetch LFS files of older commits migrate_items = Migration Items migrate_items_wiki = Wiki migrate_items_milestones = Milestones diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index f4c5b0ee1b7b8..6a068f69f7ba5 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -135,6 +135,8 @@ func Migrate(ctx *context.APIContext) { Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, LFS: form.LFS, + LFSServer: form.LFSServer, + LFSFetchOlder: form.LFSFetchOlder, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 06fb53e7244de..5977b5f94cb51 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -48,6 +48,7 @@ func Migrate(ctx *context.Context) { ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors ctx.Data["mirror"] = ctx.Query("mirror") == "1" ctx.Data["LFS"] = ctx.Query("lfs") == "1" + ctx.Data["LFSFetchOlder"] = ctx.Query("lfs_fetch_older") == "1" ctx.Data["wiki"] = ctx.Query("wiki") == "1" ctx.Data["milestones"] = ctx.Query("milestones") == "1" ctx.Data["labels"] = ctx.Query("labels") == "1" @@ -177,6 +178,8 @@ func MigratePost(ctx *context.Context) { Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror && !setting.Repository.DisableMirrors, LFS: form.LFS, + LFSServer: form.LFSServer, + LFSFetchOlder: form.LFSFetchOlder, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/templates/repo/migrate/github.tmpl b/templates/repo/migrate/github.tmpl index f0103f74a9c62..c871b32b6e706 100644 --- a/templates/repo/migrate/github.tmpl +++ b/templates/repo/migrate/github.tmpl @@ -43,6 +43,21 @@ {{end}} +
+
+ + +
+ +
+ +
+ + +
+
+
+ {{.i18n.Tr "repo.migrate.migrate_items_options"}}
diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index 09ab49b3e1e48..86e7772def416 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -4,14 +4,19 @@ const $pass = $('#auth_password'); const $token = $('#auth_token'); const $mirror = $('#mirror'); const $items = $('#migrate_items').find('input[type=checkbox]'); +const $lfs = $('#lfs'); +const $lfsserveritems = $('#lfs_server_items'); +const $lfsserver = $('#lfs_server'); export default function initMigration() { checkAuth(); + checkLFSInputs() $user.on('keyup', () => {checkItems(false)}); $pass.on('keyup', () => {checkItems(false)}); $token.on('keyup', () => {checkItems(true)}); $mirror.on('change', () => {checkItems(true)}); + $lfs.on('change', () => {checkLFSInputs()}); const $cloneAddr = $('#clone_addr'); $cloneAddr.on('change', () => { @@ -20,6 +25,13 @@ export default function initMigration() { $repoName.val($cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]); } }); + + $cloneAddr.on('keyup', () => { + const $repoName = $('#repo_name'); + if ($cloneAddr.val().length > 0) { + $lfsserver.val($cloneAddr.val() + '/info/lfs') + } + }); } function checkAuth() { @@ -39,10 +51,18 @@ function checkItems(tokenAuth) { if ($mirror.is(':checked')) { $items.not('[name="wiki"]').attr('disabled', true); $items.filter('[name="wiki"]').attr('disabled', false); - return; + } else { + $items.attr('disabled', false); } - $items.attr('disabled', false); } else { $items.attr('disabled', true); } } + +function checkLFSInputs() { + if ($lfs.is(':checked')) { + $lfsserveritems.css({'display': 'block'}); + } else { + $lfsserveritems.css({'display': 'none'}); + } +} From 61599e80313b35ae1cefb0dc5eb8393d381fbc2b Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Mon, 1 Mar 2021 09:00:07 +0500 Subject: [PATCH 07/13] Fix circular dependencies --- models/lfs.go | 65 +++++++++++++++++++ .../lfs_content_store.go | 11 ++-- modules/git/blob.go | 13 ++++ modules/lfs/pointers.go | 38 +---------- modules/lfs/server.go | 63 ++++++------------ modules/{lfs => lfsclient}/client.go | 46 ++++++++----- modules/repofiles/update.go | 4 +- modules/repofiles/upload.go | 3 +- routers/repo/lfs.go | 4 +- routers/repo/view.go | 4 +- services/pull/lfs.go | 3 +- 11 files changed, 141 insertions(+), 113 deletions(-) rename modules/lfs/content_store.go => models/lfs_content_store.go (89%) rename modules/{lfs => lfsclient}/client.go (70%) diff --git a/models/lfs.go b/models/lfs.go index 9e7fcb07e84fa..9e1b8d917d731 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -11,8 +11,13 @@ import ( "fmt" "io" "path" + "time" + "strings" + "strconv" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "xorm.io/builder" ) @@ -33,6 +38,38 @@ type LFSMetaObjectBasic struct { Size int64 `xorm:"NOT NULL"` } +// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file +func IsPointerFile(buf *[]byte) *LFSMetaObject { + if !setting.LFS.StartServer { + return nil + } + + headString := string(*buf) + if !strings.HasPrefix(headString, LFSMetaFileIdentifier) { + return nil + } + + splitLines := strings.Split(headString, "\n") + if len(splitLines) < 3 { + return nil + } + + oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix) + size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) + if len(oid) != 64 || err != nil { + return nil + } + + contentStore := &ContentStore{ObjectStorage: storage.LFS} + meta := &LFSMetaObject{Oid: oid, Size: size} + exist, err := contentStore.Exists(meta) + if err != nil || !exist { + return nil + } + + return meta +} + // RelativePath returns the relative path of the lfs object func (m *LFSMetaObject) RelativePath() string { if len(m.Oid) < 5 { @@ -240,3 +277,31 @@ func IterateLFS(f func(mo *LFSMetaObject) error) error { } } } + +// BatchResponse contains multiple object metadata Representation structures +// for use with the batch API. +type BatchResponse struct { + Transfer string `json:"transfer,omitempty"` + Objects []*Representation `json:"objects"` +} + +// Representation is object metadata as seen by clients of the lfs server. +type Representation struct { + Oid string `json:"oid"` + Size int64 `json:"size"` + Actions map[string]*Link `json:"actions"` + Error *ObjectError `json:"error,omitempty"` +} + +// ObjectError defines the JSON structure returned to the client in case of an error +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// Link provides a structure used to build a hypermedia representation of an HTTP link. +type Link struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} diff --git a/modules/lfs/content_store.go b/models/lfs_content_store.go similarity index 89% rename from modules/lfs/content_store.go rename to models/lfs_content_store.go index 247191a1bf846..0195c5ca6cd59 100644 --- a/modules/lfs/content_store.go +++ b/models/lfs_content_store.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package lfs +package models import ( "crypto/sha256" @@ -12,7 +12,6 @@ import ( "io" "os" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" ) @@ -44,7 +43,7 @@ type ContentStore struct { // Get takes a Meta object and retrieves the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte -func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) { +func (s *ContentStore) Get(meta *LFSMetaObject, fromByte int64) (io.ReadCloser, error) { f, err := s.Open(meta.RelativePath()) if err != nil { log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err) @@ -65,7 +64,7 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC } // Put takes a Meta object and an io.Reader and writes the content to the store. -func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { +func (s *ContentStore) Put(meta *LFSMetaObject, r io.Reader) error { hash := sha256.New() rd := io.TeeReader(r, hash) p := meta.RelativePath() @@ -94,7 +93,7 @@ func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { } // Exists returns true if the object exists in the content store. -func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) { +func (s *ContentStore) Exists(meta *LFSMetaObject) (bool, error) { _, err := s.ObjectStorage.Stat(meta.RelativePath()) if err != nil { if os.IsNotExist(err) { @@ -106,7 +105,7 @@ func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) { } // Verify returns true if the object exists in the content store and size is correct. -func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { +func (s *ContentStore) Verify(meta *LFSMetaObject) (bool, error) { p := meta.RelativePath() fi, err := s.ObjectStorage.Stat(p) if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) { diff --git a/modules/git/blob.go b/modules/git/blob.go index 674a6a9592778..e3fa0f2fdd672 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -32,6 +32,19 @@ func (b *Blob) GetBlobContent() (string, error) { return string(buf), nil } +// GetBlobFirstBytes gets limited content of the blob as bytes +func (b *Blob) GetBlobFirstBytes(limit int) ([]byte, error) { + dataRc, err := b.DataAsync() + buf := make([]byte, limit) + if err != nil { + return buf, err + } + defer dataRc.Close() + n, _ := dataRc.Read(buf) + buf = buf[:n] + return buf, nil +} + // GetBlobLineCount gets line count of lob as raw text func (b *Blob) GetBlobLineCount() (int, error) { reader, err := b.DataAsync() diff --git a/modules/lfs/pointers.go b/modules/lfs/pointers.go index c6fbf090e5164..1b22f9736e87d 100644 --- a/modules/lfs/pointers.go +++ b/modules/lfs/pointers.go @@ -6,8 +6,6 @@ package lfs import ( "io" - "strconv" - "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" @@ -29,43 +27,11 @@ func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) { return nil, nil } - return IsPointerFile(&buf), &buf -} - -// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file -func IsPointerFile(buf *[]byte) *models.LFSMetaObject { - if !setting.LFS.StartServer { - return nil - } - - headString := string(*buf) - if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) { - return nil - } - - splitLines := strings.Split(headString, "\n") - if len(splitLines) < 3 { - return nil - } - - oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix) - size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) - if len(oid) != 64 || err != nil { - return nil - } - - contentStore := &ContentStore{ObjectStorage: storage.LFS} - meta := &models.LFSMetaObject{Oid: oid, Size: size} - exist, err := contentStore.Exists(meta) - if err != nil || !exist { - return nil - } - - return meta + return models.IsPointerFile(&buf), &buf } // ReadMetaObject will read a models.LFSMetaObject and return a reader func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} return contentStore.Get(meta, 0) } diff --git a/modules/lfs/server.go b/modules/lfs/server.go index be21a4de82917..0f743b9cb42a5 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -14,7 +14,7 @@ import ( "regexp" "strconv" "strings" - "time" + "errors" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" @@ -29,6 +29,11 @@ const ( metaMediaType = "application/vnd.git-lfs+json" ) +var ( + errHashMismatch = errors.New("Content hash does not match OID") + errSizeMismatch = errors.New("Content size does not match") +) + // RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and // some headers are stored. type RequestVars struct { @@ -48,27 +53,6 @@ type BatchVars struct { Objects []*RequestVars `json:"objects"` } -// BatchResponse contains multiple object metadata Representation structures -// for use with the batch API. -type BatchResponse struct { - Transfer string `json:"transfer,omitempty"` - Objects []*Representation `json:"objects"` -} - -// Representation is object metadata as seen by clients of the lfs server. -type Representation struct { - Oid string `json:"oid"` - Size int64 `json:"size"` - Actions map[string]*link `json:"actions"` - Error *ObjectError `json:"error,omitempty"` -} - -// ObjectError defines the JSON structure returned to the client in case of an error -type ObjectError struct { - Code int `json:"code"` - Message string `json:"message"` -} - // Claims is a JWT Token Claims type Claims struct { RepoID int64 @@ -87,13 +71,6 @@ func (v *RequestVars) VerifyLink() string { return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/verify") } -// link provides a structure used to build a hypermedia representation of an HTTP link. -type link struct { - Href string `json:"href"` - Header map[string]string `json:"header,omitempty"` - ExpiresAt time.Time `json:"expires_at,omitempty"` -} - var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`) func isOidValid(oid string) bool { @@ -187,10 +164,10 @@ func getContentHandler(ctx *context.Context) { } } - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} content, err := contentStore.Get(meta, fromByte) if err != nil { - if IsErrRangeNotSatisfiable(err) { + if models.IsErrRangeNotSatisfiable(err) { writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable) } else { // Errors are logged in contentStore.Get @@ -292,7 +269,7 @@ func PostHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", metaMediaType) sentStatus := 202 - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} exist, err := contentStore.Exists(meta) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err) @@ -327,7 +304,7 @@ func BatchHandler(ctx *context.Context) { bv := unpackbatch(ctx) - var responseObjects []*Representation + var responseObjects []*models.Representation // Create a response object for _, object := range bv.Objects { @@ -353,7 +330,7 @@ func BatchHandler(ctx *context.Context) { return } - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} meta, err := repository.GetLFSMetaObjectByOid(object.Oid) if err == nil { // Object is found and exists @@ -392,7 +369,7 @@ func BatchHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", metaMediaType) - respobj := &BatchResponse{Objects: responseObjects} + respobj := &models.BatchResponse{Objects: responseObjects} enc := json.NewEncoder(ctx.Resp) if err := enc.Encode(respobj); err != nil { @@ -411,7 +388,7 @@ func PutHandler(ctx *context.Context) { return } - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} defer ctx.Req.Body.Close() if err := contentStore.Put(meta, ctx.Req.Body); err != nil { // Put will log the error itself @@ -452,7 +429,7 @@ func VerifyHandler(ctx *context.Context) { return } - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} ok, err := contentStore.Verify(meta) if err != nil { // Error will be logged in Verify @@ -470,11 +447,11 @@ func VerifyHandler(ctx *context.Context) { // Represent takes a RequestVars and Meta and turns it into a Representation suitable // for json encoding -func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation { - rep := &Representation{ +func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *models.Representation { + rep := &models.Representation{ Oid: meta.Oid, Size: meta.Size, - Actions: make(map[string]*link), + Actions: make(map[string]*models.Link), } header := make(map[string]string) @@ -487,11 +464,11 @@ func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload boo } if download { - rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header} + rep.Actions["download"] = &models.Link{Href: rv.ObjectLink(), Header: header} } if upload { - rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header} + rep.Actions["upload"] = &models.Link{Href: rv.ObjectLink(), Header: header} } if upload && !download { @@ -504,7 +481,7 @@ func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload boo // This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662 verifyHeader["Accept"] = metaMediaType - rep.Actions["verify"] = &link{Href: rv.VerifyLink(), Header: verifyHeader} + rep.Actions["verify"] = &models.Link{Href: rv.VerifyLink(), Header: verifyHeader} } return rep diff --git a/modules/lfs/client.go b/modules/lfsclient/client.go similarity index 70% rename from modules/lfs/client.go rename to modules/lfsclient/client.go index 34c82a4799725..c6df9b26fb7a3 100644 --- a/modules/lfs/client.go +++ b/modules/lfsclient/client.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package lfs +package lfsclient import ( "encoding/json" @@ -11,12 +11,17 @@ import ( "io" "net/http" "context" + "strconv" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" ) +const ( + metaMediaType = "application/vnd.git-lfs+json" +) + type BatchRequest struct { Operation string `json:"operation"` Transfers []string `json:"transfers,omitempty"` @@ -28,10 +33,10 @@ type Reference struct { Name string `json:"name"` } -func packbatch(operation string, transfers []string, ref *Reference, metaObjects []*models.LFSMetaObject) (string, error) { +func packbatch(operation string, transfers []string, ref *Reference, metaObjects []*models.LFSMetaObject) (*bytes.Buffer, error) { metaObjectsBasic := []*models.LFSMetaObjectBasic{} for _, meta := range metaObjects { - metaBasic := &LFSMetaObjectBasic{meta.oid, meta.size} + metaBasic := &models.LFSMetaObjectBasic{meta.Oid, meta.Size} metaObjectsBasic = append(metaObjectsBasic, metaBasic) } @@ -39,37 +44,37 @@ func packbatch(operation string, transfers []string, ref *Reference, metaObjects buf := &bytes.Buffer{} if err := json.NewEncoder(buf).Encode(reqobj); err != nil { - return nil, fmt.Errorf("Failed to encode BatchRequest as json. Error: %v", err) + return buf, fmt.Errorf("Failed to encode BatchRequest as json. Error: %v", err) } return buf, nil } -func BasicTransferAdapter(href string, size int64) (io.ReadCloser, error) { +func BasicTransferAdapter(ctx context.Context, client *http.Client, href string, size int64) (io.ReadCloser, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, href, nil) if err != nil { - return err + return nil, err } req.Header.Set("Content-type", "application/octet-stream") - req.Header.Set("Content-Length", size) + req.Header.Set("Content-Length", strconv.Itoa(int(size))) resp, err := client.Do(req) if err != nil { select { case <-ctx.Done(): - return ctx.Err() + return nil, ctx.Err() default: } - return err + return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return decodeJSONError(resp).Err + return nil, fmt.Errorf("Failed to query BasicTransferAdapter with response: %s", resp.Status) } - return resp.Body + return resp.Body, nil } -func FetchLFSFilesToContentStore(ctx *context.Context, metaObjects []*models.LFSMetaObject, userName string, repo *models.Repository, LFSServer string) error { +func FetchLFSFilesToContentStore(ctx context.Context, metaObjects []*models.LFSMetaObject, userName string, repo *models.Repository, LFSServer string) error { client := http.DefaultClient rv, err := packbatch("download", nil, nil, metaObjects) @@ -96,24 +101,28 @@ func FetchLFSFilesToContentStore(ctx *context.Context, metaObjects []*models.LFS defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return decodeJSONError(resp).Err + return fmt.Errorf("Failed to query Batch with response: %s", resp.Status) } - var respBatch BatchResponse + var respBatch models.BatchResponse err = json.NewDecoder(resp.Body).Decode(&respBatch) if err != nil { return err } - if respBatch.Transfer == nil { + if len(respBatch.Transfer) == 0 { respBatch.Transfer = "basic" } - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} for _, rep := range respBatch.Objects { - rc := BasicTransferAdapter(rep.Actions["download"].Href, rep.Size) - meta, err := GetLFSMetaObjectByOid(rep.Oid) + rc, err := BasicTransferAdapter(ctx, client, rep.Actions["download"].Href, rep.Size) + if err != nil { + log.Error("Unable to use BasicTransferAdapter. Error: %v", err) + return err + } + meta, err := repo.GetLFSMetaObjectByOid(rep.Oid) if err != nil { log.Error("Unable to get LFS OID[%s] Error: %v", rep.Oid, err) return err @@ -140,4 +149,5 @@ func FetchLFSFilesToContentStore(ctx *context.Context, metaObjects []*models.LFS return err } } + return nil } diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 0ee1ada34c1ca..881cedd8522b0 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -70,7 +70,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string buf = buf[:n] if setting.LFS.StartServer { - meta := lfs.IsPointerFile(&buf) + meta := models.IsPointerFile(&buf) if meta != nil { meta, err = repo.GetLFSMetaObjectByOid(meta.Oid) if err != nil && err != models.ErrLFSObjectNotExist { @@ -432,7 +432,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up if err != nil { return nil, err } - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} exist, err := contentStore.Exists(lfsMetaObject) if err != nil { return nil, err diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index c261e188c123f..d32e2d69233e4 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -11,7 +11,6 @@ import ( "strings" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" ) @@ -165,7 +164,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep // OK now we can insert the data into the store - there's no way to clean up the store // once it's in there, it's in there. - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} for _, uploadInfo := range infos { if uploadInfo.lfsMetaObject == nil { continue diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index fb0e3b10eae9a..f68091d05e7f1 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -493,7 +493,7 @@ type pointerResult struct { func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { defer wg.Done() defer catFileBatchReader.Close() - contentStore := lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := models.ContentStore{ObjectStorage: storage.LFS} bufferedReader := bufio.NewReader(catFileBatchReader) buf := make([]byte, 1025) @@ -526,7 +526,7 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := lfs.IsPointerFile(&pointerBuf) + pointer := models.IsPointerFile(&pointerBuf) if pointer == nil { continue } diff --git a/routers/repo/view.go b/routers/repo/view.go index a5e3cbe3e4323..27015a2cfffc3 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -273,7 +273,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { // FIXME: what happens when README file is an image? if isTextFile && setting.LFS.StartServer { - meta := lfs.IsPointerFile(&buf) + meta := models.IsPointerFile(&buf) if meta != nil { meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) if err != nil && err != models.ErrLFSObjectNotExist { @@ -399,7 +399,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st //Check for LFS meta file if isTextFile && setting.LFS.StartServer { - meta := lfs.IsPointerFile(&buf) + meta := models.IsPointerFile(&buf) if meta != nil { meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) if err != nil && err != models.ErrLFSObjectNotExist { diff --git a/services/pull/lfs.go b/services/pull/lfs.go index a1981b8253690..e796c9b854e26 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -13,7 +13,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git/pipeline" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" ) @@ -101,7 +100,7 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := lfs.IsPointerFile(&pointerBuf) + pointer := models.IsPointerFile(&pointerBuf) if pointer == nil { continue } From 4c649dbe420cbad31ec3aab4dc9bdde44edf4cad Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Mon, 1 Mar 2021 10:30:36 +0500 Subject: [PATCH 08/13] add fetchingMetaObjectsSet --- models/lfs.go | 4 ++-- modules/repository/repo.go | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 9e1b8d917d731..b1baed78cd8f6 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -34,8 +34,8 @@ type LFSMetaObject struct { // LFSMetaObjectBasic represents basic LFS metadata. type LFSMetaObjectBasic struct { - Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Size int64 `xorm:"NOT NULL"` + Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 `xorm:"NOT NULL"` } // IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file diff --git a/modules/repository/repo.go b/modules/repository/repo.go index ebec6706465c9..983904bad6b39 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -100,7 +100,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. defer gitRepo.Close() if opts.LFS { - err := FetchLFSFilesToContentStore(ctx, repo, u.Name, gitRepo, opts.LFSServer, opts.LFSFetchOlder) + err := FetchMissingLFSFilesToContentStore(ctx, repo, u.Name, gitRepo, opts.LFSServer, opts.LFSFetchOlder) if err != nil { return repo, fmt.Errorf("Failed to fetch LFS files: %v", err) } @@ -174,15 +174,15 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. return repo, err } -func FetchLFSFilesToContentStore(ctx context.Context, repo *models.Repository, userName string, gitRepo *git.Repository, LFSServer string, LFSFetchOlder bool) error { - fetchingMetaObjects := []*models.LFSMetaObject{} +func FetchMissingLFSFilesToContentStore(ctx context.Context, repo *models.Repository, userName string, gitRepo *git.Repository, LFSServer string, LFSFetchOlder bool) error { + fetchingMetaObjectsSet := make(map[string]*models.LFSMetaObject) var err error // scan repo for pointer files headBranch, _ := gitRepo.GetHEADBranch() headCommit, _ := gitRepo.GetCommit(headBranch.Name) - err = FindLFSMetaObjectsBelowMaxFileSize(headCommit, userName, repo, &fetchingMetaObjects) + err = FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(headCommit, userName, repo, &fetchingMetaObjectsSet) if err != nil { log.Error("Failed to access git LFS meta objects on commit %s: %v", headCommit.ID.String(), err) return err @@ -199,7 +199,7 @@ func FetchLFSFilesToContentStore(ctx context.Context, repo *models.Repository, u for e := commitsList.Front(); e != nil; e = e.Next() { commit := e.Value.(*git.Commit) - err = FindLFSMetaObjectsBelowMaxFileSize(commit, userName, repo, &fetchingMetaObjects) + err = FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(commit, userName, repo, &fetchingMetaObjectsSet) if err != nil { log.Error("Failed to access git LFS meta objects on commit %s: %v", commit.ID.String(), err) return err @@ -207,6 +207,11 @@ func FetchLFSFilesToContentStore(ctx context.Context, repo *models.Repository, u } } + fetchingMetaObjects := []*models.LFSMetaObject{} + for metaID := range fetchingMetaObjectsSet { + fetchingMetaObjects = append(fetchingMetaObjects, fetchingMetaObjectsSet[metaID]) + } + // fetch LFS files from external server err = lfsclient.FetchLFSFilesToContentStore(ctx, fetchingMetaObjects, userName, repo, LFSServer) if err != nil { @@ -217,7 +222,7 @@ func FetchLFSFilesToContentStore(ctx context.Context, repo *models.Repository, u return nil } -func FindLFSMetaObjectsBelowMaxFileSize(commit *git.Commit, userName string, repo *models.Repository, fetchingMetaObjects *[]*models.LFSMetaObject) error { +func FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(commit *git.Commit, userName string, repo *models.Repository, fetchingMetaObjectsSet *map[string]*models.LFSMetaObject) error { entries, err := commit.Tree.ListEntriesRecursive() if err != nil { log.Error("Failed to access git commit %s tree: %v", commit.ID.String(), err) @@ -241,7 +246,7 @@ func FindLFSMetaObjectsBelowMaxFileSize(commit *git.Commit, userName string, rep log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", meta.Oid, meta.Size, userName, repo.Name, err) return err } - *fetchingMetaObjects = append(*fetchingMetaObjects, meta) + (*fetchingMetaObjectsSet)[meta.Oid] = meta } return nil } From d35bfcc43459aaff096bc53c98123c784504604f Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Mon, 1 Mar 2021 10:50:03 +0500 Subject: [PATCH 09/13] fetch LFS on mirror update --- models/repo.go | 2 ++ modules/migrations/base/repo.go | 6 ++++-- modules/migrations/migrate.go | 2 ++ services/mirror/mirror.go | 19 +++++++++++++------ 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/models/repo.go b/models/repo.go index 29297c49542f6..874d2adce5866 100644 --- a/models/repo.go +++ b/models/repo.go @@ -219,6 +219,8 @@ type Repository struct { IsEmpty bool `xorm:"INDEX"` IsArchived bool `xorm:"INDEX"` IsMirror bool `xorm:"INDEX"` + LFS bool `yaml:"lfs"` + LFSServer string `yaml:"lfs_server"` *Mirror `xorm:"-"` Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` diff --git a/modules/migrations/base/repo.go b/modules/migrations/base/repo.go index 693a96314dc1f..1c8f91339028d 100644 --- a/modules/migrations/base/repo.go +++ b/modules/migrations/base/repo.go @@ -9,8 +9,10 @@ package base type Repository struct { Name string Owner string - IsPrivate bool `yaml:"is_private"` - IsMirror bool `yaml:"is_mirror"` + IsPrivate bool `yaml:"is_private"` + IsMirror bool `yaml:"is_mirror"` + LFS bool `yaml:"lfs"` + LFSServer string `yaml:"lfs_server"` Description string CloneURL string `yaml:"clone_url"` OriginalURL string `yaml:"original_url"` diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index b9c17478a9982..378045d18c6f4 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -140,6 +140,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } repo.IsPrivate = opts.Private repo.IsMirror = opts.Mirror + repo.LFS = opts.LFS + repo.LFSServer = opts.LFSServer if opts.Description != "" { repo.Description = opts.Description } diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index e4981b8c00e64..7f3178be75081 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -206,7 +206,7 @@ func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { } // runSync returns true if sync finished without error. -func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { +func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() wikiPath := m.Repo.WikiPath() timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second @@ -253,13 +253,20 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { log.Error("OpenRepository: %v", err) return nil, false } + defer gitRepo.Close() + + if m.Repo.LFS { + log.Trace("SyncMirrors [repo: %-v]: fetching LFS files...", m.Repo) + err := repo_module.FetchMissingLFSFilesToContentStore(ctx, m.Repo, Username(m), gitRepo, m.Repo.LFSServer, false) + if err != nil { + log.Error("Failed to fetch LFS files %v:\nErr: %v", m.Repo, err) + } + } log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { - gitRepo.Close() log.Error("Failed to synchronize tags to releases for repository: %v", err) } - gitRepo.Close() log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { @@ -378,12 +385,12 @@ func SyncMirrors(ctx context.Context) { mirrorQueue.Close() return case repoID := <-mirrorQueue.Queue(): - syncMirror(repoID) + syncMirror(ctx, repoID) } } } -func syncMirror(repoID string) { +func syncMirror(ctx context.Context, repoID string) { log.Trace("SyncMirrors [repo_id: %v]", repoID) defer func() { err := recover() @@ -403,7 +410,7 @@ func syncMirror(repoID string) { } log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) - results, ok := runSync(m) + results, ok := runSync(ctx, m) if !ok { return } From 183c6d20e30e71229b261468259e2f723dc9aceb Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Mon, 1 Mar 2021 12:37:04 +0500 Subject: [PATCH 10/13] do not fetch existing files with the same size & lint files --- integrations/lfs_getobject_test.go | 3 +- models/lfs.go | 6 +- models/repo.go | 12 +- modules/lfs/server.go | 2 +- modules/lfsclient/client.go | 171 ++++++++++++++--------------- modules/repository/repo.go | 55 +++++++--- services/mirror/mirror_test.go | 7 +- web_src/js/features/migration.js | 5 +- 8 files changed, 138 insertions(+), 123 deletions(-) diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index f364349ef140a..477fc4a61ae19 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -16,7 +16,6 @@ import ( "testing" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/routers/routes" @@ -50,7 +49,7 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string lfsID++ lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) assert.NoError(t, err) - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} exist, err := contentStore.Exists(lfsMetaObject) assert.NoError(t, err) if !exist { diff --git a/models/lfs.go b/models/lfs.go index b1baed78cd8f6..8cdbe29c983b8 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -11,13 +11,13 @@ import ( "fmt" "io" "path" - "time" - "strings" "strconv" + "strings" + "time" - "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" ) diff --git a/models/repo.go b/models/repo.go index 874d2adce5866..116faa7beda79 100644 --- a/models/repo.go +++ b/models/repo.go @@ -215,12 +215,12 @@ type Repository struct { NumClosedProjects int `xorm:"NOT NULL DEFAULT 0"` NumOpenProjects int `xorm:"-"` - IsPrivate bool `xorm:"INDEX"` - IsEmpty bool `xorm:"INDEX"` - IsArchived bool `xorm:"INDEX"` - IsMirror bool `xorm:"INDEX"` - LFS bool `yaml:"lfs"` - LFSServer string `yaml:"lfs_server"` + IsPrivate bool `xorm:"INDEX"` + IsEmpty bool `xorm:"INDEX"` + IsArchived bool `xorm:"INDEX"` + IsMirror bool `xorm:"INDEX"` + LFS bool `xorm:"INDEX"` + LFSServer string `xorm:"TEXT"` *Mirror `xorm:"-"` Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 0f743b9cb42a5..1681aac9950b8 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -7,6 +7,7 @@ package lfs import ( "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -14,7 +15,6 @@ import ( "regexp" "strconv" "strings" - "errors" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" diff --git a/modules/lfsclient/client.go b/modules/lfsclient/client.go index c6df9b26fb7a3..9ebf4cf948c49 100644 --- a/modules/lfsclient/client.go +++ b/modules/lfsclient/client.go @@ -5,149 +5,138 @@ package lfsclient import ( + "bytes" + "context" "encoding/json" - "bytes" "fmt" "io" "net/http" - "context" "strconv" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/storage" ) const ( metaMediaType = "application/vnd.git-lfs+json" ) +// BatchRequest encodes json object using in a lfs batch api request type BatchRequest struct { Operation string `json:"operation"` Transfers []string `json:"transfers,omitempty"` - Ref *Reference `json:"ref,omitempty"` - Objects []*models.LFSMetaObjectBasic `json:"objects"` + Ref *Reference `json:"ref,omitempty"` + Objects []*models.LFSMetaObjectBasic `json:"objects"` } +// Reference is a reference field of BatchRequest type Reference struct { - Name string `json:"name"` + Name string `json:"name"` } +// packbatch packs lfs batch request to json encoded as bytes func packbatch(operation string, transfers []string, ref *Reference, metaObjects []*models.LFSMetaObject) (*bytes.Buffer, error) { - metaObjectsBasic := []*models.LFSMetaObjectBasic{} + metaObjectsBasic := []*models.LFSMetaObjectBasic{} for _, meta := range metaObjects { - metaBasic := &models.LFSMetaObjectBasic{meta.Oid, meta.Size} + metaBasic := &models.LFSMetaObjectBasic{Oid: meta.Oid, Size: meta.Size} metaObjectsBasic = append(metaObjectsBasic, metaBasic) } - reqobj := &BatchRequest{operation, transfers, ref, metaObjectsBasic} + reqobj := &BatchRequest{operation, transfers, ref, metaObjectsBasic} - buf := &bytes.Buffer{} - if err := json.NewEncoder(buf).Encode(reqobj); err != nil { - return buf, fmt.Errorf("Failed to encode BatchRequest as json. Error: %v", err) + buf := &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(reqobj); err != nil { + return buf, fmt.Errorf("Failed to encode BatchRequest as json. Error: %v", err) } - return buf, nil + return buf, nil } +// BasicTransferAdapter makes request to lfs server and returns io.ReadCLoser func BasicTransferAdapter(ctx context.Context, client *http.Client, href string, size int64) (io.ReadCloser, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, href, nil) - if err != nil { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, href, nil) + if err != nil { return nil, err } req.Header.Set("Content-type", "application/octet-stream") req.Header.Set("Content-Length", strconv.Itoa(int(size))) - resp, err := client.Do(req) - if err != nil { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Failed to query BasicTransferAdapter with response: %s", resp.Status) + resp, err := client.Do(req) + if err != nil { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Failed to query BasicTransferAdapter with response: %s", resp.Status) } - return resp.Body, nil + return resp.Body, nil } -func FetchLFSFilesToContentStore(ctx context.Context, metaObjects []*models.LFSMetaObject, userName string, repo *models.Repository, LFSServer string) error { - client := http.DefaultClient +// FetchLFSFilesToContentStore downloads []LFSMetaObject from lfsServer to ContentStore +func FetchLFSFilesToContentStore(ctx context.Context, metaObjects []*models.LFSMetaObject, userName string, repo *models.Repository, lfsServer string, contentStore *models.ContentStore) error { + client := http.DefaultClient - rv, err := packbatch("download", nil, nil, metaObjects) - if err != nil { - return err - } - batchAPIURL := LFSServer + "/objects/batch" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, batchAPIURL, rv) - if err != nil { + rv, err := packbatch("download", nil, nil, metaObjects) + if err != nil { + return err + } + batchAPIURL := lfsServer + "/objects/batch" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, batchAPIURL, rv) + if err != nil { return err } req.Header.Set("Content-type", metaMediaType) req.Header.Set("Accept", metaMediaType) - resp, err := client.Do(req) - if err != nil { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("Failed to query Batch with response: %s", resp.Status) + resp, err := client.Do(req) + if err != nil { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return err } + defer resp.Body.Close() - var respBatch models.BatchResponse - err = json.NewDecoder(resp.Body).Decode(&respBatch) - if err != nil { - return err - } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Failed to query Batch with response: %s", resp.Status) + } - if len(respBatch.Transfer) == 0 { - respBatch.Transfer = "basic" + var respBatch models.BatchResponse + err = json.NewDecoder(resp.Body).Decode(&respBatch) + if err != nil { + return err } - contentStore := &models.ContentStore{ObjectStorage: storage.LFS} + if len(respBatch.Transfer) == 0 { + respBatch.Transfer = "basic" + } - for _, rep := range respBatch.Objects { - rc, err := BasicTransferAdapter(ctx, client, rep.Actions["download"].Href, rep.Size) + for _, rep := range respBatch.Objects { + rc, err := BasicTransferAdapter(ctx, client, rep.Actions["download"].Href, rep.Size) + if err != nil { + log.Error("Unable to use BasicTransferAdapter. Error: %v", err) + return err + } + meta, err := repo.GetLFSMetaObjectByOid(rep.Oid) if err != nil { - log.Error("Unable to use BasicTransferAdapter. Error: %v", err) - return err - } - meta, err := repo.GetLFSMetaObjectByOid(rep.Oid) - if err != nil { - log.Error("Unable to get LFS OID[%s] Error: %v", rep.Oid, err) - return err - } - - // put LFS file to contentStore - exist, err := contentStore.Exists(meta) - if err != nil { - log.Error("Unable to check if LFS OID[%s] exist on %s/%s. Error: %v", meta.Oid, userName, repo.Name, err) - return err - } - - if exist { - // remove collision - if _, err := repo.RemoveLFSMetaObjectByOid(meta.Oid); err != nil { - return fmt.Errorf("Error whilst removing matched LFS object %s: %v", meta.Oid, err) - } - } - - if err := contentStore.Put(meta, rc); err != nil { - if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { - return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", meta.Oid, err2, err) - } - return err - } - } + log.Error("Unable to get LFS OID[%s] Error: %v", rep.Oid, err) + return err + } + + // put LFS file to contentStore + if err := contentStore.Put(meta, rc); err != nil { + if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { + return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", meta.Oid, err2, err) + } + return err + } + } return nil } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 983904bad6b39..40cb74d97a80a 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -13,12 +13,13 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfsclient" "code.gitea.io/gitea/modules/log" migration "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/modules/lfsclient" "gopkg.in/ini.v1" ) @@ -174,22 +175,24 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. return repo, err } -func FetchMissingLFSFilesToContentStore(ctx context.Context, repo *models.Repository, userName string, gitRepo *git.Repository, LFSServer string, LFSFetchOlder bool) error { +// FetchMissingLFSFilesToContentStore downloads lfs files to a ContentStore +func FetchMissingLFSFilesToContentStore(ctx context.Context, repo *models.Repository, userName string, gitRepo *git.Repository, lfsServer string, lfsFetchOlder bool) error { fetchingMetaObjectsSet := make(map[string]*models.LFSMetaObject) var err error + contentStore := &models.ContentStore{ObjectStorage: storage.LFS} // scan repo for pointer files headBranch, _ := gitRepo.GetHEADBranch() headCommit, _ := gitRepo.GetCommit(headBranch.Name) - err = FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(headCommit, userName, repo, &fetchingMetaObjectsSet) + err = FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(headCommit, userName, repo, &fetchingMetaObjectsSet, contentStore) if err != nil { log.Error("Failed to access git LFS meta objects on commit %s: %v", headCommit.ID.String(), err) return err } - if LFSFetchOlder { - opts := git.NewSearchCommitsOptions("before:" + headCommit.ID.String(), true) + if lfsFetchOlder { + opts := git.NewSearchCommitsOptions("before:"+headCommit.ID.String(), true) commitIDsList, _ := headCommit.SearchCommits(opts) var commitIDs = []string{} for e := commitIDsList.Front(); e != nil; e = e.Next() { @@ -199,7 +202,7 @@ func FetchMissingLFSFilesToContentStore(ctx context.Context, repo *models.Reposi for e := commitsList.Front(); e != nil; e = e.Next() { commit := e.Value.(*git.Commit) - err = FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(commit, userName, repo, &fetchingMetaObjectsSet) + err = FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(commit, userName, repo, &fetchingMetaObjectsSet, contentStore) if err != nil { log.Error("Failed to access git LFS meta objects on commit %s: %v", commit.ID.String(), err) return err @@ -213,7 +216,7 @@ func FetchMissingLFSFilesToContentStore(ctx context.Context, repo *models.Reposi } // fetch LFS files from external server - err = lfsclient.FetchLFSFilesToContentStore(ctx, fetchingMetaObjects, userName, repo, LFSServer) + err = lfsclient.FetchLFSFilesToContentStore(ctx, fetchingMetaObjects, userName, repo, lfsServer, contentStore) if err != nil { log.Error("Unable to fetch LFS files in %v/%v to content store. Error: %v", userName, repo.Name, err) return err @@ -222,7 +225,8 @@ func FetchMissingLFSFilesToContentStore(ctx context.Context, repo *models.Reposi return nil } -func FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(commit *git.Commit, userName string, repo *models.Repository, fetchingMetaObjectsSet *map[string]*models.LFSMetaObject) error { +// FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles finds lfs pointers in a repo and adds them to a passed fetchingMetaObjectsSet +func FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(commit *git.Commit, userName string, repo *models.Repository, fetchingMetaObjectsSet *map[string]*models.LFSMetaObject, contentStore *models.ContentStore) error { entries, err := commit.Tree.ListEntriesRecursive() if err != nil { log.Error("Failed to access git commit %s tree: %v", commit.ID.String(), err) @@ -231,22 +235,45 @@ func FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(commit *git.Commit, user for _, entry := range entries { buf, _ := entry.Blob().GetBlobFirstBytes(1024) - meta := models.IsPointerFile(&buf) - if meta == nil { + metaBasic := models.IsPointerFile(&buf) + if metaBasic == nil { continue } - if setting.LFS.MaxFileSize > 0 && meta.Size > setting.LFS.MaxFileSize { - log.Info("Denied LFS oid[%s] download of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", meta.Oid, meta.Size, userName, repo.Name, setting.LFS.MaxFileSize) + if setting.LFS.MaxFileSize > 0 && metaBasic.Size > setting.LFS.MaxFileSize { + log.Info("Denied LFS oid[%s] download of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", metaBasic.Oid, metaBasic.Size, userName, repo.Name, setting.LFS.MaxFileSize) continue } - meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: meta.Oid, Size: meta.Size, RepositoryID: repo.ID}) + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: metaBasic.Oid, Size: metaBasic.Size, RepositoryID: repo.ID}) if err != nil { log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", meta.Oid, meta.Size, userName, repo.Name, err) return err } - (*fetchingMetaObjectsSet)[meta.Oid] = meta + + exist, err := contentStore.Exists(meta) + if err != nil { + log.Error("Unable to check if LFS OID[%s] exist on %s/%s. Error: %v", meta.Oid, userName, repo.Name, err) + return err + } + + if exist { + fileSizeValid, err := contentStore.Verify(meta) + if err != nil { + log.Error("Unable to verify LFS OID[%s] size on %s/%s. Error: %v", meta.Oid, userName, repo.Name, err) + return err + } + // remove collision if exists and size not matching + if !fileSizeValid { + if _, err := repo.RemoveLFSMetaObjectByOid(meta.Oid); err != nil { + return fmt.Errorf("Error whilst removing matched LFS object %s: %v", meta.Oid, err) + } + (*fetchingMetaObjectsSet)[meta.Oid] = meta + } + // if exists and size matching, do not fetch + } else { + (*fetchingMetaObjectsSet)[meta.Oid] = meta + } } return nil } diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go index ddfb6c676b6a7..16bb9071e05cf 100644 --- a/services/mirror/mirror_test.go +++ b/services/mirror/mirror_test.go @@ -48,7 +48,8 @@ func TestRelease_MirrorDelete(t *testing.T) { }) assert.NoError(t, err) - mirror, err := repository.MigrateRepositoryGitData(context.Background(), user, mirrorRepo, opts) + ctx := context.Background() + mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts) assert.NoError(t, err) gitRepo, err := git.OpenRepository(repoPath) @@ -74,7 +75,7 @@ func TestRelease_MirrorDelete(t *testing.T) { err = mirror.GetMirror() assert.NoError(t, err) - _, ok := runSync(mirror.Mirror) + _, ok := runSync(ctx, mirror.Mirror) assert.True(t, ok) count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) @@ -85,7 +86,7 @@ func TestRelease_MirrorDelete(t *testing.T) { assert.NoError(t, err) assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true)) - _, ok = runSync(mirror.Mirror) + _, ok = runSync(ctx, mirror.Mirror) assert.True(t, ok) count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions) diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index 86e7772def416..7f3c2f5709f2c 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -10,7 +10,7 @@ const $lfsserver = $('#lfs_server'); export default function initMigration() { checkAuth(); - checkLFSInputs() + checkLFSInputs(); $user.on('keyup', () => {checkItems(false)}); $pass.on('keyup', () => {checkItems(false)}); @@ -27,9 +27,8 @@ export default function initMigration() { }); $cloneAddr.on('keyup', () => { - const $repoName = $('#repo_name'); if ($cloneAddr.val().length > 0) { - $lfsserver.val($cloneAddr.val() + '/info/lfs') + $lfsserver.val(`${$cloneAddr.val()}/info/lfs`); } }); } From 5677bccbdc4db0eea23bc8ffe1ee7d309f1b509b Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Mon, 1 Mar 2021 13:24:21 +0500 Subject: [PATCH 11/13] add error ErrMirrorLFSServerNotValid --- models/error.go | 14 +++++++------- models/repo.go | 8 ++++---- options/locale/locale_en-US.ini | 2 +- routers/repo/migrate.go | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/models/error.go b/models/error.go index d3270aef39c03..1282a9a191aef 100644 --- a/models/error.go +++ b/models/error.go @@ -56,18 +56,18 @@ func (err ErrNamePatternNotAllowed) Error() string { return fmt.Sprintf("name pattern is not allowed [pattern: %s]", err.Pattern) } -// ErrLFSNotInstalled represents an "git-lfs not found" error. -type ErrLFSNotInstalled struct { +// ErrMirrorLFSServerNotValid represents an "LFS Server not valid" error. +type ErrMirrorLFSServerNotValid struct { } -// IsLFSNotInstalled checks if an error is an ErrLFSNotInstalled. -func IsLFSNotInstalled(err error) bool { - _, ok := err.(ErrLFSNotInstalled) +// IsMirrorLFSServerValid checks if an error is an ErrMirrorLFSServerNotValid. +func IsMirrorLFSServerValid(err error) bool { + _, ok := err.(ErrMirrorLFSServerNotValid) return ok } -func (err ErrLFSNotInstalled) Error() string { - return "git-lfs not found" +func (err ErrMirrorLFSServerNotValid) Error() string { + return "LFS Server not valid" } // ErrNameCharsNotAllowed represents a "character not allowed in name" error. diff --git a/models/repo.go b/models/repo.go index 116faa7beda79..eddb354fd22b3 100644 --- a/models/repo.go +++ b/models/repo.go @@ -18,7 +18,6 @@ import ( "net" "net/url" "os" - "os/exec" "path" "path/filepath" "sort" @@ -938,7 +937,7 @@ func (repo *Repository) CloneLink() (cl *CloneLink) { } // CheckCreateRepository check if could created a repository -func CheckCreateRepository(doer, u *User, name string, lfs bool, overwriteOrAdopt bool) error { +func CheckCreateRepository(doer, u *User, name string, lfs bool, lfsServer string, overwriteOrAdopt bool) error { if !doer.CanCreateRepo() { return ErrReachLimitOfRepo{u.MaxRepoCreation} } @@ -964,8 +963,9 @@ func CheckCreateRepository(doer, u *User, name string, lfs bool, overwriteOrAdop } if lfs { - if _, err := exec.LookPath("git-lfs"); err != nil { - return ErrLFSNotInstalled{} + u, err := url.ParseRequestURI(lfsServer) + if err != nil { + return ErrMirrorLFSServerNotValid{} } } return nil diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index cf8b17fef0072..06f4058bfc94e 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -392,7 +392,7 @@ user_not_exist = The user does not exist. team_not_exist = The team does not exist. last_org_owner = You cannot remove the last user from the 'owners' team. There must be at least one owner for an organization. cannot_add_org_to_team = An organization cannot be added as a team member. -lfs_not_installed = Please install git-lfs. +lfs_server_not_valid = LFS Server not valid. invalid_ssh_key = Can not verify your SSH key: %s invalid_gpg_key = Can not verify your GPG key: %s diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 5977b5f94cb51..a23af8eadd7c4 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -102,9 +102,9 @@ func handleMigrateError(ctx *context.Context, owner *models.User, err error, nam case models.IsErrNamePatternNotAllowed(err): ctx.Data["Err_RepoName"] = true ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form) - case models.IsLFSNotInstalled(err): - ctx.Data["Err_LFSNotInstalled"] = true - ctx.RenderWithErr(ctx.Tr("form.lfs_not_installed"), tpl, form) + case models.IsMirrorLFSServerValid(err): + ctx.Data["Err_LFSServerNotValid"] = true + ctx.RenderWithErr(ctx.Tr("form.lfs_server_not_valid"), tpl, form) default: remoteAddr, _ := auth.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword, owner) err = util.URLSanitizedError(err, remoteAddr) @@ -200,7 +200,7 @@ func MigratePost(ctx *context.Context) { opts.Releases = false } - err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName, opts.LFS, false) + err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName, opts.LFS, opts.LFSServer, false) if err != nil { handleMigrateError(ctx, ctxUser, err, "MigratePost", tpl, form) return From 9982edbc4ac6200c02d9a2b4d72c536e3671a73d Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Mon, 1 Mar 2021 21:48:36 +0500 Subject: [PATCH 12/13] update --- models/lfs.go | 34 ++++++++++++++++++++++++++---- models/repo.go | 4 ++-- modules/lfs/pointers.go | 2 +- modules/lfsclient/client.go | 5 ++--- modules/repofiles/update.go | 2 +- modules/repository/repo.go | 6 +++--- routers/repo/lfs.go | 2 +- routers/repo/view.go | 4 ++-- services/pull/lfs.go | 2 +- templates/repo/migrate/git.tmpl | 15 +++++++++++++ templates/repo/migrate/gitea.tmpl | 15 +++++++++++++ templates/repo/migrate/github.tmpl | 2 +- templates/repo/migrate/gitlab.tmpl | 15 +++++++++++++ templates/swagger/v1_json.tmpl | 16 ++++++++++++++ 14 files changed, 105 insertions(+), 19 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 8cdbe29c983b8..1f52678df78c3 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -34,12 +34,12 @@ type LFSMetaObject struct { // LFSMetaObjectBasic represents basic LFS metadata. type LFSMetaObjectBasic struct { - Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Size int64 `xorm:"NOT NULL"` + Oid string `json:"oid"` + Size int64 `json:"size"` } -// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file -func IsPointerFile(buf *[]byte) *LFSMetaObject { +// IsPointerFileAndStored will return a partially filled LFSMetaObject if the provided byte slice is a pointer file and stored in contentStore +func IsPointerFileAndStored(buf *[]byte) *LFSMetaObject { if !setting.LFS.StartServer { return nil } @@ -70,6 +70,32 @@ func IsPointerFile(buf *[]byte) *LFSMetaObject { return meta } +// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file +func IsPointerFile(buf *[]byte) *LFSMetaObjectBasic { + if !setting.LFS.StartServer { + return nil + } + + headString := string(*buf) + if !strings.HasPrefix(headString, LFSMetaFileIdentifier) { + return nil + } + + splitLines := strings.Split(headString, "\n") + if len(splitLines) < 3 { + return nil + } + + oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix) + size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) + if len(oid) != 64 || err != nil { + return nil + } + meta := &LFSMetaObjectBasic{Oid: oid, Size: size} + + return meta +} + // RelativePath returns the relative path of the lfs object func (m *LFSMetaObject) RelativePath() string { if len(m.Oid) < 5 { diff --git a/models/repo.go b/models/repo.go index eddb354fd22b3..e2a8dc415ccc6 100644 --- a/models/repo.go +++ b/models/repo.go @@ -963,9 +963,9 @@ func CheckCreateRepository(doer, u *User, name string, lfs bool, lfsServer strin } if lfs { - u, err := url.ParseRequestURI(lfsServer) + _, err := url.ParseRequestURI(lfsServer) if err != nil { - return ErrMirrorLFSServerNotValid{} + return ErrMirrorLFSServerNotValid{} } } return nil diff --git a/modules/lfs/pointers.go b/modules/lfs/pointers.go index 1b22f9736e87d..30850145a347c 100644 --- a/modules/lfs/pointers.go +++ b/modules/lfs/pointers.go @@ -27,7 +27,7 @@ func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) { return nil, nil } - return models.IsPointerFile(&buf), &buf + return models.IsPointerFileAndStored(&buf), &buf } // ReadMetaObject will read a models.LFSMetaObject and return a reader diff --git a/modules/lfsclient/client.go b/modules/lfsclient/client.go index 9ebf4cf948c49..417d7271340f2 100644 --- a/modules/lfsclient/client.go +++ b/modules/lfsclient/client.go @@ -69,7 +69,6 @@ func BasicTransferAdapter(ctx context.Context, client *http.Client, href string, } return nil, err } - defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("Failed to query BasicTransferAdapter with response: %s", resp.Status) @@ -81,12 +80,12 @@ func BasicTransferAdapter(ctx context.Context, client *http.Client, href string, func FetchLFSFilesToContentStore(ctx context.Context, metaObjects []*models.LFSMetaObject, userName string, repo *models.Repository, lfsServer string, contentStore *models.ContentStore) error { client := http.DefaultClient - rv, err := packbatch("download", nil, nil, metaObjects) + rv, err := packbatch("download", []string{"basic"}, nil, metaObjects) if err != nil { return err } batchAPIURL := lfsServer + "/objects/batch" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, batchAPIURL, rv) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, batchAPIURL, rv) if err != nil { return err } diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 881cedd8522b0..f565dc8952faf 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -70,7 +70,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string buf = buf[:n] if setting.LFS.StartServer { - meta := models.IsPointerFile(&buf) + meta := models.IsPointerFileAndStored(&buf) if meta != nil { meta, err = repo.GetLFSMetaObjectByOid(meta.Oid) if err != nil && err != models.ErrLFSObjectNotExist { diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 40cb74d97a80a..4094d26889c5c 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -263,10 +263,10 @@ func FindLFSMetaObjectsBelowMaxFileSizeWithMissingFiles(commit *git.Commit, user log.Error("Unable to verify LFS OID[%s] size on %s/%s. Error: %v", meta.Oid, userName, repo.Name, err) return err } - // remove collision if exists and size not matching + // remove file collision if exists and size not matching if !fileSizeValid { - if _, err := repo.RemoveLFSMetaObjectByOid(meta.Oid); err != nil { - return fmt.Errorf("Error whilst removing matched LFS object %s: %v", meta.Oid, err) + if err := contentStore.Delete(meta.RelativePath()); err != nil { + return fmt.Errorf("Error whilst deleting contentStore file by LFS oid %s: %v", meta.Oid, err) } (*fetchingMetaObjectsSet)[meta.Oid] = meta } diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index f68091d05e7f1..32a3910af70fa 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -526,7 +526,7 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := models.IsPointerFile(&pointerBuf) + pointer := models.IsPointerFileAndStored(&pointerBuf) if pointer == nil { continue } diff --git a/routers/repo/view.go b/routers/repo/view.go index 27015a2cfffc3..67c297cf5f07e 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -273,7 +273,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { // FIXME: what happens when README file is an image? if isTextFile && setting.LFS.StartServer { - meta := models.IsPointerFile(&buf) + meta := models.IsPointerFileAndStored(&buf) if meta != nil { meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) if err != nil && err != models.ErrLFSObjectNotExist { @@ -399,7 +399,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st //Check for LFS meta file if isTextFile && setting.LFS.StartServer { - meta := models.IsPointerFile(&buf) + meta := models.IsPointerFileAndStored(&buf) if meta != nil { meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) if err != nil && err != models.ErrLFSObjectNotExist { diff --git a/services/pull/lfs.go b/services/pull/lfs.go index e796c9b854e26..965510937e1f1 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -100,7 +100,7 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := models.IsPointerFile(&pointerBuf) + pointer := models.IsPointerFileAndStored(&pointerBuf) if pointer == nil { continue } diff --git a/templates/repo/migrate/git.tmpl b/templates/repo/migrate/git.tmpl index 26d2311471209..fb1c1c78d03fa 100644 --- a/templates/repo/migrate/git.tmpl +++ b/templates/repo/migrate/git.tmpl @@ -46,6 +46,21 @@ {{end}}
+
+
+ + +
+ +
+ +
+ + +
+
+
+
diff --git a/templates/repo/migrate/gitea.tmpl b/templates/repo/migrate/gitea.tmpl index b984e54cd69d6..adc3d2818e81b 100644 --- a/templates/repo/migrate/gitea.tmpl +++ b/templates/repo/migrate/gitea.tmpl @@ -43,6 +43,21 @@ {{end}}
+
+
+ + +
+ +
+ +
+ + +
+
+
+ {{.i18n.Tr "repo.migrate.migrate_items_options"}}
diff --git a/templates/repo/migrate/github.tmpl b/templates/repo/migrate/github.tmpl index c871b32b6e706..47c545e7c491b 100644 --- a/templates/repo/migrate/github.tmpl +++ b/templates/repo/migrate/github.tmpl @@ -46,7 +46,7 @@
- +
diff --git a/templates/repo/migrate/gitlab.tmpl b/templates/repo/migrate/gitlab.tmpl index ff8f3637370d6..fd07dd2d31e4f 100644 --- a/templates/repo/migrate/gitlab.tmpl +++ b/templates/repo/migrate/gitlab.tmpl @@ -43,6 +43,21 @@ {{end}}
+
+
+ + +
+ +
+ +
+ + +
+
+
+ {{.i18n.Tr "repo.migrate.migrate_items_options"}}
diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 8b3cb5ac3cca0..60b13ad0de548 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -14644,6 +14644,14 @@ "type": "boolean", "x-go-name": "LFS" }, + "lfs_fetch_older": { + "type": "boolean", + "x-go-name": "LFSFetchOlder" + }, + "lfs_server": { + "type": "string", + "x-go-name": "LFSServer" + }, "milestones": { "type": "boolean", "x-go-name": "Milestones" @@ -14727,6 +14735,14 @@ "type": "boolean", "x-go-name": "LFS" }, + "lfs_fetch_older": { + "type": "boolean", + "x-go-name": "LFSFetchOlder" + }, + "lfs_server": { + "type": "string", + "x-go-name": "LFSServer" + }, "milestones": { "type": "boolean", "x-go-name": "Milestones" From 670a4e7573fbd85587574d9a90dab24e33a65315 Mon Sep 17 00:00:00 2001 From: Nick Reiley Date: Mon, 1 Mar 2021 22:21:44 +0500 Subject: [PATCH 13/13] update --- modules/repofiles/upload.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index c3f6124733317..cadff5b3159a8 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" )