From f34664cb8bfdaf17a5fe9280a7be8cf62f7eec3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Fri, 28 Jan 2022 17:36:27 +0100 Subject: [PATCH 01/10] store the foreign ID of issues during migration So that it can be used to implement mirroring --- models/error.go | 43 +++++++++++++++ models/fixtures/foreign_reference.yml | 1 + models/foreignreference.go | 21 +++++++ models/issue.go | 40 ++++++++++++++ models/issue_test.go | 31 +++++++++++ models/migrate.go | 7 +++ models/migrate_test.go | 10 ++++ models/migrations/migrations.go | 2 + modules/migration/comment.go | 7 +++ modules/migration/downloader.go | 15 ++--- modules/migration/issue.go | 58 ++++++++------------ modules/migration/null_downloader.go | 11 +++- modules/migration/pullrequest.go | 7 ++- modules/migration/retry_downloader.go | 8 +-- modules/migration/review.go | 6 ++ services/migrations/codebase.go | 38 +++---------- services/migrations/codebase_test.go | 6 +- services/migrations/gitbucket.go | 5 -- services/migrations/gitea_downloader.go | 24 ++++---- services/migrations/gitea_downloader_test.go | 6 +- services/migrations/gitea_uploader.go | 6 ++ services/migrations/github.go | 28 ++++------ services/migrations/github_test.go | 12 ++-- services/migrations/gitlab.go | 46 +++++----------- services/migrations/gitlab_test.go | 24 ++++---- services/migrations/gogs.go | 8 +-- services/migrations/gogs_test.go | 4 +- services/migrations/migrate.go | 15 ++--- services/migrations/onedev.go | 48 ++++++---------- services/migrations/onedev_test.go | 36 ++++-------- services/migrations/restore.go | 13 ++--- 31 files changed, 326 insertions(+), 260 deletions(-) create mode 100644 models/fixtures/foreign_reference.yml create mode 100644 models/foreignreference.go diff --git a/models/error.go b/models/error.go index f0e8751d75fdc..db52523b72bd7 100644 --- a/models/error.go +++ b/models/error.go @@ -1385,3 +1385,46 @@ func (err ErrNotValidReviewRequest) Error() string { err.UserID, err.RepoID) } + +// +// ___________ .__ __________ _____ +// \_ _____/__________ ____ |__| ____ ____\______ \ _____/ ____\___________ ____ ____ ____ ____ +// | __)/ _ \_ __ \_/ __ \| |/ ___\ / \| _// __ \ __\/ __ \_ __ \_/ __ \ / \_/ ___\/ __ \ +// | \( <_> ) | \/\ ___/| / /_/ > | \ | \ ___/| | \ ___/| | \/\ ___/| | \ \__\ ___/ +// \___ / \____/|__| \___ >__\___ /|___| /____|_ /\___ >__| \___ >__| \___ >___| /\___ >___ > +// \/ \/ /_____/ \/ \/ \/ \/ \/ \/ \/ \/ +// + +// ErrLocalIDNotExist represents a "LocalIDNotExist" kind of error. +type ErrLocalIDNotExist struct { + RepoID int64 + ForeignID int64 + Type string +} + +// ErrLocalIDNotExist checks if an error is a ErrLocalIDNotExist. +func IsErrLocalIDNotExist(err error) bool { + _, ok := err.(ErrLocalIDNotExist) + return ok +} + +func (err ErrLocalIDNotExist) Error() string { + return fmt.Sprintf("repository %d has no LocalID for ForeignID %d of type %s", err.RepoID, err.ForeignID, err.Type) +} + +// ErrForeignIDNotExist represents a "ForeignIDNotExist" kind of error. +type ErrForeignIDNotExist struct { + RepoID int64 + LocalID int64 + Type string +} + +// ErrForeignIDNotExist checks if an error is a ErrForeignIDNotExist. +func IsErrForeignIDNotExist(err error) bool { + _, ok := err.(ErrForeignIDNotExist) + return ok +} + +func (err ErrForeignIDNotExist) Error() string { + return fmt.Sprintf("repository %d has no ForeignID for LocalID %d of type %s", err.RepoID, err.LocalID, err.Type) +} diff --git a/models/fixtures/foreign_reference.yml b/models/fixtures/foreign_reference.yml new file mode 100644 index 0000000000000..ca780a73aa0c1 --- /dev/null +++ b/models/fixtures/foreign_reference.yml @@ -0,0 +1 @@ +[] # empty diff --git a/models/foreignreference.go b/models/foreignreference.go new file mode 100644 index 0000000000000..4a9bef1dc1792 --- /dev/null +++ b/models/foreignreference.go @@ -0,0 +1,21 @@ +// Copyright 2022 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "code.gitea.io/gitea/models/db" +) + +// ForeignReference represents external references +type ForeignReference struct { + LocalID int64 `xorm:"INDEX"` + ForeignID int64 `xorm:"INDEX UNIQUE(external_reference_index)"` + RepoID int64 `xorm:"INDEX UNIQUE(external_reference_index)"` + Type string `xorm:"INDEX UNIQUE(external_reference_index)"` +} + +func init() { + db.RegisterModel(new(ForeignReference)) +} diff --git a/models/issue.go b/models/issue.go index fd59ac0a4b495..a7138690f7be3 100644 --- a/models/issue.go +++ b/models/issue.go @@ -72,6 +72,7 @@ type Issue struct { Reactions ReactionList `xorm:"-"` TotalTrackedTime int64 `xorm:"-"` Assignees []*user_model.User `xorm:"-"` + ForeignReference *ForeignReference `xorm:"-"` // IsLocked limits commenting abilities to users on an issue // with write access @@ -271,6 +272,25 @@ func (issue *Issue) loadReactions(ctx context.Context) (err error) { return nil } +func (issue *Issue) loadForeignReference(ctx context.Context) (err error) { + if issue.ForeignReference != nil { + return nil + } + reference := &ForeignReference{ + LocalID: issue.ID, + RepoID: issue.RepoID, + Type: "issue", + } + has, err := db.GetEngine(ctx).Get(reference) + if err != nil { + return err + } else if !has { + return ErrForeignIDNotExist{issue.RepoID, issue.ID, "issue"} + } + issue.ForeignReference = reference + return nil +} + func (issue *Issue) loadMilestone(e db.Engine) (err error) { if (issue.Milestone == nil || issue.Milestone.ID != issue.MilestoneID) && issue.MilestoneID > 0 { issue.Milestone, err = getMilestoneByRepoID(e, issue.RepoID, issue.MilestoneID) @@ -332,6 +352,10 @@ func (issue *Issue) loadAttributes(ctx context.Context) (err error) { } } + if err = issue.loadForeignReference(ctx); err != nil && !IsErrForeignIDNotExist(err) { + return err + } + return issue.loadReactions(ctx) } @@ -1110,6 +1134,22 @@ func GetIssueByIndex(repoID, index int64) (*Issue, error) { return issue, nil } +// GetIssueByForeignID returns raw issue by foreign ID +func GetIssueByForeignID(ctx context.Context, repoID, foreignID int64) (*Issue, error) { + reference := &ForeignReference{ + RepoID: repoID, + ForeignID: foreignID, + Type: "issue", + } + has, err := db.GetEngine(ctx).Get(reference) + if err != nil { + return nil, err + } else if !has { + return nil, ErrLocalIDNotExist{repoID, foreignID, "issue"} + } + return getIssueByID(db.GetEngine(ctx), reference.LocalID) +} + // GetIssueWithAttrsByIndex returns issue by index in a repository. func GetIssueWithAttrsByIndex(repoID, index int64) (*Issue, error) { issue, err := GetIssueByIndex(repoID, index) diff --git a/models/issue_test.go b/models/issue_test.go index 7cc0aa61b0da9..906de757fa4ec 100644 --- a/models/issue_test.go +++ b/models/issue_test.go @@ -534,3 +534,34 @@ func TestCorrectIssueStats(t *testing.T) { assert.NoError(t, err) assert.EqualValues(t, issueStats.OpenCount, issueAmount) } + +func TestIssueForeignReference(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + issue := unittest.AssertExistsAndLoadBean(t, &Issue{ID: 1}).(*Issue) + + // it is fine for an issue to not have a foreign reference + err := issue.LoadAttributes() + assert.NoError(t, err) + assert.Nil(t, issue.ForeignReference) + + var foreignID int64 = 12345 + _, err = GetIssueByForeignID(context.Background(), issue.RepoID, foreignID) + assert.True(t, IsErrLocalIDNotExist(err)) + + _, err = db.GetEngine(db.DefaultContext).Insert(&ForeignReference{ + LocalID: issue.ID, + ForeignID: foreignID, + RepoID: issue.RepoID, + Type: "issue", + }) + assert.NoError(t, err) + + err = issue.LoadAttributes() + assert.NoError(t, err) + + assert.EqualValues(t, issue.ForeignReference.ForeignID, foreignID) + + found, err := GetIssueByForeignID(context.Background(), issue.RepoID, foreignID) + assert.NoError(t, err) + assert.EqualValues(t, found.ID, issue.ID) +} diff --git a/models/migrate.go b/models/migrate.go index bbfba1fa1e61a..7c9de3a4b7516 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -83,6 +83,13 @@ func insertIssue(ctx context.Context, issue *Issue) error { } } + if issue.ForeignReference != nil { + issue.ForeignReference.LocalID = issue.ID + if _, err := sess.Insert(issue.ForeignReference); err != nil { + return err + } + } + return nil } diff --git a/models/migrate_test.go b/models/migrate_test.go index d85dcbfeef25d..703e8d09074b9 100644 --- a/models/migrate_test.go +++ b/models/migrate_test.go @@ -45,6 +45,7 @@ func assertCreateIssues(t *testing.T, isPull bool) { UserID: owner.ID, } + foreignID := int64(12345) title := "issuetitle1" is := &Issue{ RepoID: repo.ID, @@ -58,11 +59,20 @@ func assertCreateIssues(t *testing.T, isPull bool) { IsClosed: true, Labels: []*Label{label}, Reactions: []*Reaction{reaction}, + ForeignReference: &ForeignReference{ + ForeignID: foreignID, + RepoID: repo.ID, + Type: "issue", + }, } err := InsertIssues(is) assert.NoError(t, err) i := unittest.AssertExistsAndLoadBean(t, &Issue{Title: title}).(*Issue) + assert.Nil(t, i.ForeignReference) + err = i.LoadAttributes() + assert.NoError(t, err) + assert.EqualValues(t, foreignID, i.ForeignReference.ForeignID) unittest.AssertExistsAndLoadBean(t, &Reaction{Type: "heart", UserID: owner.ID, IssueID: i.ID}) } diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 31b172a68d66f..5ed358822b689 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -373,6 +373,8 @@ var migrations = []Migration{ NewMigration("Increase WebAuthentication CredentialID size to 410 - NO-OPED", increaseCredentialIDTo410), // v210 -> v211 NewMigration("v208 was completely broken - remigrate", remigrateU2FCredentials), + // v211 -> v212 + NewMigration("Create ForeignReference table", createForeignReferenceTable), } // GetCurrentDBVersion returns the current db version diff --git a/modules/migration/comment.go b/modules/migration/comment.go index f364ffc93a728..9e895fbb2f401 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -7,6 +7,13 @@ package migration import "time" +// Commentable can be commented upon +type Commentable interface { + GetLocalID() int64 + GetForeignID() int64 + GetContext() DownloaderContext +} + // Comment is a standard comment information type Comment struct { IssueIndex int64 `yaml:"issue_index"` diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 90e149fb1a40c..7759c96056a92 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -11,13 +11,6 @@ import ( "code.gitea.io/gitea/modules/structs" ) -// GetCommentOptions represents an options for get comment -type GetCommentOptions struct { - Context IssueContext - Page int - PageSize int -} - // Downloader downloads the site repo information type Downloader interface { SetContext(context.Context) @@ -27,10 +20,11 @@ type Downloader interface { GetReleases() ([]*Release, error) GetLabels() ([]*Label, error) GetIssues(page, perPage int) ([]*Issue, bool, error) - GetComments(opts GetCommentOptions) ([]*Comment, bool, error) + GetComments(commentable Commentable) ([]*Comment, bool, error) + GetAllComments(page, perPage int) ([]*Comment, bool, error) SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(pullRequestContext IssueContext) ([]*Review, error) + GetReviews(reviewable Reviewable) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) } @@ -39,3 +33,6 @@ type DownloaderFactory interface { New(ctx context.Context, opts MigrateOptions) (Downloader, error) GitServiceType() structs.GitServiceType } + +// DownloaderContext has opaque information only relevant to a given downloader +type DownloaderContext interface{} diff --git a/modules/migration/issue.go b/modules/migration/issue.go index 984f07d8c9eed..fd2d161435b6c 100644 --- a/modules/migration/issue.go +++ b/modules/migration/issue.go @@ -7,44 +7,26 @@ package migration import "time" -// IssueContext is used to map between local and foreign issue/PR ids. -type IssueContext interface { - LocalID() int64 - ForeignID() int64 -} - -// BasicIssueContext is a 1:1 mapping between local and foreign ids. -type BasicIssueContext int64 - -// LocalID gets the local id. -func (c BasicIssueContext) LocalID() int64 { - return int64(c) -} - -// ForeignID gets the foreign id. -func (c BasicIssueContext) ForeignID() int64 { - return int64(c) -} - // Issue is a standard issue information type Issue struct { - Number int64 `json:"number"` - PosterID int64 `yaml:"poster_id" json:"poster_id"` - PosterName string `yaml:"poster_name" json:"poster_name"` - PosterEmail string `yaml:"poster_email" json:"poster_email"` - Title string `json:"title"` - Content string `json:"content"` - Ref string `json:"ref"` - Milestone string `json:"milestone"` - State string `json:"state"` // closed, open - IsLocked bool `yaml:"is_locked" json:"is_locked"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - Closed *time.Time `json:"closed"` - Labels []*Label `json:"labels"` - Reactions []*Reaction `json:"reactions"` - Assignees []string `json:"assignees"` - Context IssueContext `yaml:"-"` + Number int64 `json:"number"` + PosterID int64 `yaml:"poster_id" json:"poster_id"` + PosterName string `yaml:"poster_name" json:"poster_name"` + PosterEmail string `yaml:"poster_email" json:"poster_email"` + Title string `json:"title"` + Content string `json:"content"` + Ref string `json:"ref"` + Milestone string `json:"milestone"` + State string `json:"state"` // closed, open + IsLocked bool `yaml:"is_locked" json:"is_locked"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Closed *time.Time `json:"closed"` + Labels []*Label `json:"labels"` + Reactions []*Reaction `json:"reactions"` + Assignees []string `json:"assignees"` + ForeignID int64 `json:"foreign_id"` + Context DownloaderContext `yaml:"-"` } // GetExternalName ExternalUserMigrated interface @@ -52,3 +34,7 @@ func (i *Issue) GetExternalName() string { return i.PosterName } // GetExternalID ExternalUserMigrated interface func (i *Issue) GetExternalID() int64 { return i.PosterID } + +func (i *Issue) GetLocalID() int64 { return i.Number } +func (i *Issue) GetForeignID() int64 { return i.ForeignID } +func (i *Issue) GetContext() DownloaderContext { return i.Context } diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index 6192870873525..32da720f1601f 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -47,18 +47,23 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { return nil, false, &ErrNotSupported{Entity: "Issues"} } -// GetComments returns comments according the options -func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) { +// GetComments returns comments of an issue or PR +func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { return nil, false, &ErrNotSupported{Entity: "Comments"} } +// GetAllComments returns paginated comments +func (n NullDownloader) GetAllComments(page, perPage int) ([]*Comment, bool, error) { + return nil, false, &ErrNotSupported{Entity: "AllComments"} +} + // GetPullRequests returns pull requests according page and perPage func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { return nil, false, &ErrNotSupported{Entity: "PullRequests"} } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) { +func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { return nil, &ErrNotSupported{Entity: "Reviews"} } diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index 7a681940a7eeb..b75903bd42ff5 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -35,9 +35,14 @@ type PullRequest struct { Assignees []string IsLocked bool `yaml:"is_locked"` Reactions []*Reaction - Context IssueContext `yaml:"-"` + ForeignID int64 + Context DownloaderContext `yaml:"-"` } +func (p *PullRequest) GetLocalID() int64 { return p.Number } +func (p *PullRequest) GetForeignID() int64 { return p.ForeignID } +func (p *PullRequest) GetContext() DownloaderContext { return p.Context } + // IsForkPullRequest returns true if the pull request from a forked repository but not the same repository func (p *PullRequest) IsForkPullRequest() bool { return p.Head.RepoPath() != p.Base.RepoPath() diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go index 1095a26891ba9..2e40c102bea88 100644 --- a/modules/migration/retry_downloader.go +++ b/modules/migration/retry_downloader.go @@ -148,7 +148,7 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns a repository's comments with retry -func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) { +func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { var ( comments []*Comment isEnd bool @@ -156,7 +156,7 @@ func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, ) err = d.retry(func() error { - comments, isEnd, err = d.Downloader.GetComments(opts) + comments, isEnd, err = d.Downloader.GetComments(commentable) return err }) @@ -180,14 +180,14 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) { +func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { var ( reviews []*Review err error ) err = d.retry(func() error { - reviews, err = d.Downloader.GetReviews(pullRequestContext) + reviews, err = d.Downloader.GetReviews(reviewable) return err }) diff --git a/modules/migration/review.go b/modules/migration/review.go index 85795385e97e4..fc698f1de3a88 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -6,6 +6,12 @@ package migration import "time" +// Reviewable can be reviewed +type Reviewable interface { + GetLocalID() int64 + GetForeignID() int64 +} + // enumerate all review states const ( ReviewStatePending = "PENDING" diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index be0b5d40043ac..d70ee58224c5c 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -266,17 +266,7 @@ func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) { } type codebaseIssueContext struct { - foreignID int64 - localID int64 - Comments []*base.Comment -} - -func (c codebaseIssueContext) LocalID() int64 { - return c.localID -} - -func (c codebaseIssueContext) ForeignID() int64 { - return c.foreignID + Comments []*base.Comment } // GetIssues returns issues, limits are not supported @@ -402,10 +392,9 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, Labels: []*base.Label{ {Name: issue.Type.Name}, }, + ForeignID: issue.TicketID.Value, Context: codebaseIssueContext{ - foreignID: issue.TicketID.Value, - localID: issue.TicketID.Value, - Comments: comments[1:], + Comments: comments[1:], }, }) @@ -418,10 +407,10 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments -func (d *CodebaseDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - context, ok := opts.Context.(codebaseIssueContext) +func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { + context, ok := commentable.GetContext().(codebaseIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context) + return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) } return context.Comments, true, nil @@ -570,10 +559,9 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq SHA: d.getHeadCommit(rawMergeRequest.TargetRef), RepoName: d.repoName, }, + ForeignID: rawMergeRequest.ID.Value, Context: codebaseIssueContext{ - foreignID: rawMergeRequest.ID.Value, - localID: number, - Comments: comments[1:], + Comments: comments[1:], }, }) } @@ -581,16 +569,6 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq return pullRequests, true, nil } -// GetReviews returns pull requests reviews -func (d *CodebaseDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { - return []*base.Review{}, nil -} - -// GetTopics return repository topics -func (d *CodebaseDownloader) GetTopics() ([]string, error) { - return []string{}, nil -} - func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { if len(d.userMap) == 0 { var rawUsers struct { diff --git a/services/migrations/codebase_test.go b/services/migrations/codebase_test.go index ef39b9f146b86..cb70a2bf75936 100644 --- a/services/migrations/codebase_test.go +++ b/services/migrations/codebase_test.go @@ -108,9 +108,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: issues[0].Context, - }) + comments, _, err := downloader.GetComments(issues[0]) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -148,7 +146,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(prs[0].Context) + rvs, err := downloader.GetReviews(prs[0]) assert.NoError(t, err) assert.Empty(t, rvs) } diff --git a/services/migrations/gitbucket.go b/services/migrations/gitbucket.go index c4fb0df93a7e9..92b6cac73806b 100644 --- a/services/migrations/gitbucket.go +++ b/services/migrations/gitbucket.go @@ -64,8 +64,3 @@ func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, to func (g *GitBucketDownloader) SupportGetRepoComments() bool { return false } - -// GetReviews is not supported -func (g *GitBucketDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { - return nil, &base.ErrNotSupported{Entity: "Reviews"} -} diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index be3c6c1202fea..fded3cdf8ec9e 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -430,7 +430,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err Labels: labels, Assignees: assignees, IsLocked: issue.IsLocked, - Context: base.BasicIssueContext(issue.Index), + ForeignID: issue.Index, }) } @@ -442,7 +442,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err } // GetComments returns comments according issueNumber -func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { +func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, g.maxPerPage) for i := 1; ; i++ { @@ -453,26 +453,26 @@ func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comm default: } - comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ + comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ PageSize: g.maxPerPage, Page: i, }}) if err != nil { - return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err) + return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", commentable.GetForeignID(), err) } for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err) + log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignID(), comment.ID, g.repoOwner, g.repoName, err) if err2 := admin_model.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { + fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { log.Error("create repository notice failed: ", err2) } } allComments = append(allComments, &base.Comment{ - IssueIndex: opts.Context.LocalID(), + IssueIndex: commentable.GetLocalID(), Index: comment.ID, PosterID: comment.Poster.ID, PosterName: comment.Poster.UserName, @@ -602,7 +602,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques RepoName: g.repoName, OwnerName: g.repoOwner, }, - Context: base.BasicIssueContext(pr.Index), + ForeignID: pr.Index, }) } @@ -614,7 +614,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { +func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") return nil, nil @@ -630,7 +630,7 @@ func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, default: } - prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ + prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, reviewable.GetForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ Page: i, PageSize: g.maxPerPage, }}) @@ -640,7 +640,7 @@ func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, for _, pr := range prl { - rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID) + rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, reviewable.GetForeignID(), pr.ID) if err != nil { return nil, err } @@ -666,7 +666,7 @@ func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, allReviews = append(allReviews, &base.Review{ ID: pr.ID, - IssueIndex: context.LocalID(), + IssueIndex: reviewable.GetLocalID(), ReviewerID: pr.Reviewer.ID, ReviewerName: pr.Reviewer.UserName, Official: pr.Official, diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index 2c70dc421356f..acf93764788c8 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -198,9 +198,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: base.BasicIssueContext(4), - }) + comments, _, err := downloader.GetComments(&base.Issue{Number: 4, ForeignID: 4}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -265,7 +263,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(base.BasicIssueContext(7)) + reviews, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignID: 7}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 21c2dc8f8eb5d..70026a3a1b9ac 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -373,6 +373,12 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { Labels: labels, CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()), UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()), + ForeignReference: &models.ForeignReference{ + LocalID: issue.GetLocalID(), + ForeignID: issue.GetForeignID(), + RepoID: g.repo.ID, + Type: "issue", + }, } if err := g.remapUser(issue, &is); err != nil { diff --git a/services/migrations/github.go b/services/migrations/github.go index f86ba94393b42..ca9a167a1e912 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -461,7 +461,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, Closed: issue.ClosedAt, IsLocked: issue.GetLocked(), Assignees: assignees, - Context: base.BasicIssueContext(*issue.Number), + ForeignID: int64(*issue.Number), }) } @@ -474,16 +474,12 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { } // GetComments returns comments according issueNumber -func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - if opts.Context != nil { - comments, err := g.getComments(opts.Context) - return comments, false, err - } - - return g.GetAllComments(opts.Page, opts.PageSize) +func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { + comments, err := g.getComments(commentable) + return comments, false, err } -func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -498,7 +494,7 @@ func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*bas } for { g.waitAndPickClient() - comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt) + comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } @@ -531,7 +527,7 @@ func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*bas } allComments = append(allComments, &base.Comment{ - IssueIndex: issueContext.LocalID(), + IssueIndex: commentable.GetLocalID(), Index: comment.GetID(), PosterID: comment.GetUser().GetID(), PosterName: comment.GetUser().GetLogin(), @@ -711,7 +707,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, PatchURL: pr.GetPatchURL(), Reactions: reactions, - Context: base.BasicIssueContext(*pr.Number), + ForeignID: int64(*pr.Number), }) } @@ -777,28 +773,28 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) { +func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { allReviews := make([]*base.Review, 0, g.maxPerPage) opt := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.waitAndPickClient() - reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt) + reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } g.setRate(&resp.Rate) for _, review := range reviews { r := convertGithubReview(review) - r.IssueIndex = context.LocalID() + r.IssueIndex = reviewable.GetLocalID() // retrieve all review comments opt2 := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.waitAndPickClient() - reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2) + reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignID()), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } diff --git a/services/migrations/github_test.go b/services/migrations/github_test.go index 7540037d92fab..a9e71a24b8078 100644 --- a/services/migrations/github_test.go +++ b/services/migrations/github_test.go @@ -215,9 +215,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: base.BasicIssueContext(2), - }) + comments, _, err := downloader.GetComments(&base.Issue{Number: 2, ForeignID: 2}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -286,7 +284,7 @@ func TestGitHubDownloadRepo(t *testing.T) { Merged: true, MergedTime: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)), MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", - Context: base.BasicIssueContext(3), + ForeignID: 3, }, { Number: 4, @@ -333,11 +331,11 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, - Context: base.BasicIssueContext(4), + ForeignID: 4, }, }, prs) - reviews, err := downloader.GetReviews(base.BasicIssueContext(3)) + reviews, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignID: 3}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -369,7 +367,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(base.BasicIssueContext(4)) + reviews, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignID: 4}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index c05d081e9aadc..f3ed00dd60d87 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -349,19 +349,9 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { } type gitlabIssueContext struct { - foreignID int64 - localID int64 IsMergeRequest bool } -func (c gitlabIssueContext) LocalID() int64 { - return c.localID -} - -func (c gitlabIssueContext) ForeignID() int64 { - return c.foreignID -} - // GetIssues returns issues according start and limit // Note: issue label description and colors are not supported by the go-gitlab library at this time func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { @@ -434,11 +424,8 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Closed: issue.ClosedAt, IsLocked: issue.DiscussionLocked, Updated: *issue.UpdatedAt, - Context: gitlabIssueContext{ - foreignID: int64(issue.IID), - localID: int64(issue.IID), - IsMergeRequest: false, - }, + ForeignID: int64(issue.IID), + Context: gitlabIssueContext{IsMergeRequest: false}, }) // increment issueCount, to be used in GetPullRequests() @@ -450,10 +437,10 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions -func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - context, ok := opts.Context.(gitlabIssueContext) +func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { + context, ok := commentable.GetContext().(gitlabIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context) + return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) } allComments := make([]*base.Comment, 0, g.maxPerPage) @@ -465,12 +452,12 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com var resp *gitlab.Response var err error if !context.IsMergeRequest { - comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListIssueDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignID()), &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } else { - comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) @@ -484,7 +471,7 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com if !comment.IndividualNote { for _, note := range comment.Notes { allComments = append(allComments, &base.Comment{ - IssueIndex: context.LocalID(), + IssueIndex: commentable.GetLocalID(), Index: int64(note.ID), PosterID: int64(note.Author.ID), PosterName: note.Author.Username, @@ -496,7 +483,7 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com } else { c := comment.Notes[0] allComments = append(allComments, &base.Comment{ - IssueIndex: context.LocalID(), + IssueIndex: commentable.GetLocalID(), Index: int64(c.ID), PosterID: int64(c.Author.ID), PosterName: c.Author.Username, @@ -619,12 +606,9 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque RepoName: g.repoName, OwnerName: pr.Author.Username, }, - PatchURL: pr.WebURL + ".patch", - Context: gitlabIssueContext{ - foreignID: int64(pr.IID), - localID: newPRNumber, - IsMergeRequest: true, - }, + PatchURL: pr.WebURL + ".patch", + ForeignID: int64(pr.IID), + Context: gitlabIssueContext{IsMergeRequest: true}, }) } @@ -632,8 +616,8 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { - approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(context.ForeignID()), gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { + approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignID()), gitlab.WithContext(g.ctx)) if err != nil { if resp != nil && resp.StatusCode == 404 { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) @@ -654,7 +638,7 @@ func (g *GitlabDownloader) GetReviews(context base.IssueContext) ([]*base.Review reviews := make([]*base.Review, 0, len(approvals.ApprovedBy)) for _, user := range approvals.ApprovedBy { reviews = append(reviews, &base.Review{ - IssueIndex: context.LocalID(), + IssueIndex: reviewable.GetLocalID(), ReviewerID: int64(user.User.ID), ReviewerName: user.User.Username, CreatedAt: createdAt, diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index ad61577653c17..89b822588edd1 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -214,12 +214,10 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: gitlabIssueContext{ - foreignID: 2, - localID: 2, - IsMergeRequest: false, - }, + comments, _, err := downloader.GetComments(&base.Issue{ + Number: 2, + ForeignID: 2, + Context: gitlabIssueContext{IsMergeRequest: false}, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -301,15 +299,12 @@ func TestGitlabDownloadRepo(t *testing.T) { Merged: false, MergedTime: nil, MergeCommitSHA: "", - Context: gitlabIssueContext{ - foreignID: 2, - localID: 4, - IsMergeRequest: true, - }, + ForeignID: 2, + Context: gitlabIssueContext{IsMergeRequest: true}, }, }, prs) - rvs, err := downloader.GetReviews(base.BasicIssueContext(1)) + rvs, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignID: 1}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -328,7 +323,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, rvs) - rvs, err = downloader.GetReviews(base.BasicIssueContext(2)) + rvs, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignID: 2}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -469,7 +464,8 @@ func TestGitlabGetReviews(t *testing.T) { mock, review := convertTestCase(testCase) mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock) - rvs, err := downloader.GetReviews(base.BasicIssueContext(testCase.prID)) + id := int64(testCase.prID) + rvs, err := downloader.GetReviews(&base.Issue{Number: id, ForeignID: id}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{&review}, rvs) } diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index 0ef39484b7a03..92dfcce455056 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -223,10 +223,10 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { +func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, 100) - comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID()) + comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignID()) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } @@ -235,7 +235,7 @@ func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comme continue } allComments = append(allComments, &base.Comment{ - IssueIndex: opts.Context.LocalID(), + IssueIndex: commentable.GetLocalID(), Index: comment.ID, PosterID: comment.Poster.ID, PosterName: comment.Poster.Login, @@ -300,7 +300,7 @@ func convertGogsIssue(issue *gogs.Issue) *base.Issue { Updated: issue.Updated, Labels: labels, Closed: closed, - Context: base.BasicIssueContext(issue.Index), + ForeignID: issue.Index, } } diff --git a/services/migrations/gogs_test.go b/services/migrations/gogs_test.go index f9d74d3be3ecf..8be3472b6625f 100644 --- a/services/migrations/gogs_test.go +++ b/services/migrations/gogs_test.go @@ -111,9 +111,7 @@ func TestGogsDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: base.BasicIssueContext(1), - }) + comments, _, err := downloader.GetComments(&base.Issue{Number: 1, ForeignID: 1}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 7bca128ac52f7..b550be4ce75f4 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -325,9 +325,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts allComments := make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { log.Trace("migrating issue %d's comments", issue.Number) - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: issue.Context, - }) + comments, _, err := downloader.GetComments(issue) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -383,9 +381,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts allComments := make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { log.Trace("migrating pull request %d's comments", pr.Number) - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: pr.Context, - }) + comments, _, err := downloader.GetComments(pr) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -412,7 +408,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts // migrate reviews allReviews := make([]*base.Review, 0, reviewBatchSize) for _, pr := range prs { - reviews, err := downloader.GetReviews(pr.Context) + reviews, err := downloader.GetReviews(pr) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -446,10 +442,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts if opts.Comments && supportAllComments { log.Trace("migrating comments") for i := 1; ; i++ { - comments, isEnd, err := downloader.GetComments(base.GetCommentOptions{ - Page: i, - PageSize: commentBatchSize, - }) + comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize) if err != nil { return err } diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index d27cbbed4f850..d53ce24e60bc1 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -262,19 +262,9 @@ func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { } type onedevIssueContext struct { - foreignID int64 - localID int64 IsPullRequest bool } -func (c onedevIssueContext) LocalID() int64 { - return c.localID -} - -func (c onedevIssueContext) ForeignID() int64 { - return c.foreignID -} - // GetIssues returns issues func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { rawIssues := make([]struct { @@ -356,11 +346,8 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Created: issue.SubmitDate, Updated: issue.SubmitDate, Labels: []*base.Label{label}, - Context: onedevIssueContext{ - foreignID: issue.ID, - localID: issue.Number, - IsPullRequest: false, - }, + ForeignID: issue.ID, + Context: onedevIssueContext{IsPullRequest: false}, }) if d.maxIssueIndex < issue.Number { @@ -372,10 +359,10 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er } // GetComments returns comments -func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - context, ok := opts.Context.(onedevIssueContext) +func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { + context, ok := commentable.GetContext().(onedevIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context) + return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) } rawComments := make([]struct { @@ -387,9 +374,9 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com var endpoint string if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.ForeignID()) + endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", commentable.GetForeignID()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/comments", context.ForeignID()) + endpoint = fmt.Sprintf("/api/issues/%d/comments", commentable.GetForeignID()) } err := d.callAPI( @@ -408,9 +395,9 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com }, 0, 100) if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.ForeignID()) + endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", commentable.GetForeignID()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/changes", context.ForeignID()) + endpoint = fmt.Sprintf("/api/issues/%d/changes", commentable.GetForeignID()) } err = d.callAPI( @@ -429,7 +416,7 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com } poster := d.tryGetUser(comment.UserID) comments = append(comments, &base.Comment{ - IssueIndex: context.LocalID(), + IssueIndex: commentable.GetLocalID(), Index: comment.ID, PosterID: poster.ID, PosterName: poster.Name, @@ -454,7 +441,7 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com poster := d.tryGetUser(change.UserID) comments = append(comments, &base.Comment{ - IssueIndex: context.LocalID(), + IssueIndex: commentable.GetLocalID(), PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, @@ -552,11 +539,8 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque SHA: mergePreview.TargetHeadCommitHash, RepoName: d.repoName, }, - Context: onedevIssueContext{ - foreignID: pr.ID, - localID: number, - IsPullRequest: true, - }, + ForeignID: pr.ID, + Context: onedevIssueContext{IsPullRequest: true}, }) } @@ -564,7 +548,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests reviews -func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { +func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { rawReviews := make([]struct { ID int64 `json:"id"` UserID int64 `json:"userId"` @@ -576,7 +560,7 @@ func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review }, 0, 100) err := d.callAPI( - fmt.Sprintf("/api/pull-requests/%d/reviews", context.ForeignID()), + fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignID()), nil, &rawReviews, ) @@ -600,7 +584,7 @@ func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review poster := d.tryGetUser(review.UserID) reviews = append(reviews, &base.Review{ - IssueIndex: context.LocalID(), + IssueIndex: reviewable.GetLocalID(), ReviewerID: poster.ID, ReviewerName: poster.Name, Content: content, diff --git a/services/migrations/onedev_test.go b/services/migrations/onedev_test.go index 59b7cae5feffe..3c8798124bcb9 100644 --- a/services/migrations/onedev_test.go +++ b/services/migrations/onedev_test.go @@ -74,11 +74,8 @@ func TestOneDevDownloadRepo(t *testing.T) { Name: "Improvement", }, }, - Context: onedevIssueContext{ - foreignID: 398, - localID: 4, - IsPullRequest: false, - }, + ForeignID: 398, + Context: onedevIssueContext{IsPullRequest: false}, }, { Number: 3, @@ -94,20 +91,15 @@ func TestOneDevDownloadRepo(t *testing.T) { Name: "New Feature", }, }, - Context: onedevIssueContext{ - foreignID: 397, - localID: 3, - IsPullRequest: false, - }, + ForeignID: 397, + Context: onedevIssueContext{IsPullRequest: false}, }, }, issues) - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: onedevIssueContext{ - foreignID: 398, - localID: 4, - IsPullRequest: false, - }, + comments, _, err := downloader.GetComments(&base.Issue{ + Number: 4, + ForeignID: 398, + Context: onedevIssueContext{IsPullRequest: false}, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -141,18 +133,12 @@ func TestOneDevDownloadRepo(t *testing.T) { SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", RepoName: "go-gitea-test_repo", }, - Context: onedevIssueContext{ - foreignID: 186, - localID: 5, - IsPullRequest: true, - }, + ForeignID: 186, + Context: onedevIssueContext{IsPullRequest: true}, }, }, prs) - rvs, err := downloader.GetReviews(onedevIssueContext{ - foreignID: 186, - localID: 5, - }) + rvs, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignID: 186}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/restore.go b/services/migrations/restore.go index d30d90a6c48f4..f8083de57bf01 100644 --- a/services/migrations/restore.go +++ b/services/migrations/restore.go @@ -193,17 +193,13 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, } return nil, false, err } - - for _, issue := range issues { - issue.Context = base.BasicIssueContext(issue.Number) - } return issues, true, nil } // GetComments returns comments according issueNumber -func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { +func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { comments := make([]*base.Comment, 0, 10) - p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.Context.ForeignID())) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignID())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -247,15 +243,14 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } for _, pr := range pulls { pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) - pr.Context = base.BasicIssueContext(pr.Number) } return pulls, true, nil } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) { +func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { reviews := make([]*base.Review, 0, 10) - p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", context.ForeignID())) + p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignID())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { From 0a80569d8a28831d89568aa1d89e2db7157c4380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Thu, 3 Feb 2022 13:36:28 +0100 Subject: [PATCH 02/10] ForeignID in the database is a string to support uuids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Loïc Dachary --- models/foreignreference.go | 2 +- models/issue.go | 2 +- models/issue_test.go | 5 +++-- models/migrate_test.go | 5 +++-- services/migrations/gitea_uploader.go | 5 +++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/models/foreignreference.go b/models/foreignreference.go index 4a9bef1dc1792..be5a7139c7de3 100644 --- a/models/foreignreference.go +++ b/models/foreignreference.go @@ -11,7 +11,7 @@ import ( // ForeignReference represents external references type ForeignReference struct { LocalID int64 `xorm:"INDEX"` - ForeignID int64 `xorm:"INDEX UNIQUE(external_reference_index)"` + ForeignID string `xorm:"INDEX UNIQUE(external_reference_index)"` RepoID int64 `xorm:"INDEX UNIQUE(external_reference_index)"` Type string `xorm:"INDEX UNIQUE(external_reference_index)"` } diff --git a/models/issue.go b/models/issue.go index a7138690f7be3..501a46097e5c2 100644 --- a/models/issue.go +++ b/models/issue.go @@ -1138,7 +1138,7 @@ func GetIssueByIndex(repoID, index int64) (*Issue, error) { func GetIssueByForeignID(ctx context.Context, repoID, foreignID int64) (*Issue, error) { reference := &ForeignReference{ RepoID: repoID, - ForeignID: foreignID, + ForeignID: strconv.FormatInt(foreignID, 10), Type: "issue", } has, err := db.GetEngine(ctx).Get(reference) diff --git a/models/issue_test.go b/models/issue_test.go index 906de757fa4ec..52c18adcd8371 100644 --- a/models/issue_test.go +++ b/models/issue_test.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "sort" + "strconv" "sync" "testing" "time" @@ -550,7 +551,7 @@ func TestIssueForeignReference(t *testing.T) { _, err = db.GetEngine(db.DefaultContext).Insert(&ForeignReference{ LocalID: issue.ID, - ForeignID: foreignID, + ForeignID: strconv.FormatInt(foreignID, 10), RepoID: issue.RepoID, Type: "issue", }) @@ -559,7 +560,7 @@ func TestIssueForeignReference(t *testing.T) { err = issue.LoadAttributes() assert.NoError(t, err) - assert.EqualValues(t, issue.ForeignReference.ForeignID, foreignID) + assert.EqualValues(t, issue.ForeignReference.ForeignID, strconv.FormatInt(foreignID, 10)) found, err := GetIssueByForeignID(context.Background(), issue.RepoID, foreignID) assert.NoError(t, err) diff --git a/models/migrate_test.go b/models/migrate_test.go index 703e8d09074b9..33ab7a2f18d43 100644 --- a/models/migrate_test.go +++ b/models/migrate_test.go @@ -5,6 +5,7 @@ package models import ( + "strconv" "testing" repo_model "code.gitea.io/gitea/models/repo" @@ -60,7 +61,7 @@ func assertCreateIssues(t *testing.T, isPull bool) { Labels: []*Label{label}, Reactions: []*Reaction{reaction}, ForeignReference: &ForeignReference{ - ForeignID: foreignID, + ForeignID: strconv.FormatInt(foreignID, 10), RepoID: repo.ID, Type: "issue", }, @@ -72,7 +73,7 @@ func assertCreateIssues(t *testing.T, isPull bool) { assert.Nil(t, i.ForeignReference) err = i.LoadAttributes() assert.NoError(t, err) - assert.EqualValues(t, foreignID, i.ForeignReference.ForeignID) + assert.EqualValues(t, strconv.FormatInt(foreignID, 10), i.ForeignReference.ForeignID) unittest.AssertExistsAndLoadBean(t, &Reaction{Type: "heart", UserID: owner.ID, IssueID: i.ID}) } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 70026a3a1b9ac..7646c8f60ed05 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2019-2022 The Gitea Authors. All rights reserved. // Copyright 2018 Jonas Franz. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -11,6 +11,7 @@ import ( "io" "os" "path/filepath" + "strconv" "strings" "time" @@ -375,7 +376,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()), ForeignReference: &models.ForeignReference{ LocalID: issue.GetLocalID(), - ForeignID: issue.GetForeignID(), + ForeignID: strconv.FormatInt(issue.GetForeignID(), 10), RepoID: g.repo.ID, Type: "issue", }, From 09d1a53d09a363a3cbf288fa1869eb0b037ff488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Thu, 10 Feb 2022 16:35:28 +0100 Subject: [PATCH 03/10] LocalID is issue.Index --- models/issue.go | 4 ++-- models/issue_test.go | 4 ++-- models/migrate.go | 2 +- services/migrations/gitea_uploader.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/issue.go b/models/issue.go index 501a46097e5c2..d0f2974dc17ad 100644 --- a/models/issue.go +++ b/models/issue.go @@ -277,7 +277,7 @@ func (issue *Issue) loadForeignReference(ctx context.Context) (err error) { return nil } reference := &ForeignReference{ - LocalID: issue.ID, + LocalID: issue.Index, RepoID: issue.RepoID, Type: "issue", } @@ -285,7 +285,7 @@ func (issue *Issue) loadForeignReference(ctx context.Context) (err error) { if err != nil { return err } else if !has { - return ErrForeignIDNotExist{issue.RepoID, issue.ID, "issue"} + return ErrForeignIDNotExist{issue.RepoID, issue.Index, "issue"} } issue.ForeignReference = reference return nil diff --git a/models/issue_test.go b/models/issue_test.go index 52c18adcd8371..8811d29708b2c 100644 --- a/models/issue_test.go +++ b/models/issue_test.go @@ -550,7 +550,7 @@ func TestIssueForeignReference(t *testing.T) { assert.True(t, IsErrLocalIDNotExist(err)) _, err = db.GetEngine(db.DefaultContext).Insert(&ForeignReference{ - LocalID: issue.ID, + LocalID: issue.Index, ForeignID: strconv.FormatInt(foreignID, 10), RepoID: issue.RepoID, Type: "issue", @@ -564,5 +564,5 @@ func TestIssueForeignReference(t *testing.T) { found, err := GetIssueByForeignID(context.Background(), issue.RepoID, foreignID) assert.NoError(t, err) - assert.EqualValues(t, found.ID, issue.ID) + assert.EqualValues(t, found.Index, issue.Index) } diff --git a/models/migrate.go b/models/migrate.go index 7c9de3a4b7516..d0813be503a76 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -84,7 +84,7 @@ func insertIssue(ctx context.Context, issue *Issue) error { } if issue.ForeignReference != nil { - issue.ForeignReference.LocalID = issue.ID + issue.ForeignReference.LocalID = issue.Index if _, err := sess.Insert(issue.ForeignReference); err != nil { return err } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 7646c8f60ed05..00587bc8608bd 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -1,4 +1,4 @@ -// Copyright 2019-2022 The Gitea Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Copyright 2018 Jonas Franz. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. From a63a669984156f1ff79a4c2d276afdb4c796a943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Thu, 10 Feb 2022 16:53:14 +0100 Subject: [PATCH 04/10] migrations: better indices, 8 char Type field, ForeignTypeIssue --- models/foreignreference.go | 11 +++++++---- models/issue.go | 6 +++--- models/issue_test.go | 2 +- models/migrate_test.go | 2 +- services/migrations/gitea_uploader.go | 2 +- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/models/foreignreference.go b/models/foreignreference.go index be5a7139c7de3..eec3b9de3811c 100644 --- a/models/foreignreference.go +++ b/models/foreignreference.go @@ -8,12 +8,15 @@ import ( "code.gitea.io/gitea/models/db" ) +const ForeignTypeIssue = "issue" + // ForeignReference represents external references type ForeignReference struct { - LocalID int64 `xorm:"INDEX"` - ForeignID string `xorm:"INDEX UNIQUE(external_reference_index)"` - RepoID int64 `xorm:"INDEX UNIQUE(external_reference_index)"` - Type string `xorm:"INDEX UNIQUE(external_reference_index)"` + // RepoID is the first column in all indices. now we only need 2 indices: (repo, local) and (repo, foreign, type) + RepoID int64 `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" ` + LocalID int64 `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID. + ForeignID string `xorm:"INDEX UNIQUE(repo_foreign_type)"` + Type string `xorm:"VARCHAR(8) INDEX UNIQUE(repo_foreign_type)"` } func init() { diff --git a/models/issue.go b/models/issue.go index d0f2974dc17ad..a3b79606c0c2f 100644 --- a/models/issue.go +++ b/models/issue.go @@ -279,13 +279,13 @@ func (issue *Issue) loadForeignReference(ctx context.Context) (err error) { reference := &ForeignReference{ LocalID: issue.Index, RepoID: issue.RepoID, - Type: "issue", + Type: ForeignTypeIssue, } has, err := db.GetEngine(ctx).Get(reference) if err != nil { return err } else if !has { - return ErrForeignIDNotExist{issue.RepoID, issue.Index, "issue"} + return ErrForeignIDNotExist{issue.RepoID, issue.Index, ForeignTypeIssue} } issue.ForeignReference = reference return nil @@ -1139,7 +1139,7 @@ func GetIssueByForeignID(ctx context.Context, repoID, foreignID int64) (*Issue, reference := &ForeignReference{ RepoID: repoID, ForeignID: strconv.FormatInt(foreignID, 10), - Type: "issue", + Type: ForeignTypeIssue, } has, err := db.GetEngine(ctx).Get(reference) if err != nil { diff --git a/models/issue_test.go b/models/issue_test.go index 8811d29708b2c..2d1df134b65bb 100644 --- a/models/issue_test.go +++ b/models/issue_test.go @@ -553,7 +553,7 @@ func TestIssueForeignReference(t *testing.T) { LocalID: issue.Index, ForeignID: strconv.FormatInt(foreignID, 10), RepoID: issue.RepoID, - Type: "issue", + Type: ForeignTypeIssue, }) assert.NoError(t, err) diff --git a/models/migrate_test.go b/models/migrate_test.go index 33ab7a2f18d43..b9a16d4bce2e5 100644 --- a/models/migrate_test.go +++ b/models/migrate_test.go @@ -63,7 +63,7 @@ func assertCreateIssues(t *testing.T, isPull bool) { ForeignReference: &ForeignReference{ ForeignID: strconv.FormatInt(foreignID, 10), RepoID: repo.ID, - Type: "issue", + Type: ForeignTypeIssue, }, } err := InsertIssues(is) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 00587bc8608bd..5f96f06e9f068 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -378,7 +378,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { LocalID: issue.GetLocalID(), ForeignID: strconv.FormatInt(issue.GetForeignID(), 10), RepoID: g.repo.ID, - Type: "issue", + Type: models.ForeignTypeIssue, }, } From e860f35d1acd6f41fda76c033502807759a3f2d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Fri, 11 Feb 2022 15:09:19 +0100 Subject: [PATCH 05/10] use GetIssueByIndex and fix test to avoid false positive --- models/issue.go | 4 ++-- models/issue_test.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/models/issue.go b/models/issue.go index a3b79606c0c2f..6508d609ee77f 100644 --- a/models/issue.go +++ b/models/issue.go @@ -1145,9 +1145,9 @@ func GetIssueByForeignID(ctx context.Context, repoID, foreignID int64) (*Issue, if err != nil { return nil, err } else if !has { - return nil, ErrLocalIDNotExist{repoID, foreignID, "issue"} + return nil, ErrLocalIDNotExist{repoID, foreignID, ForeignTypeIssue} } - return getIssueByID(db.GetEngine(ctx), reference.LocalID) + return GetIssueByIndex(repoID, reference.LocalID) } // GetIssueWithAttrsByIndex returns issue by index in a repository. diff --git a/models/issue_test.go b/models/issue_test.go index 2d1df134b65bb..a2de314129532 100644 --- a/models/issue_test.go +++ b/models/issue_test.go @@ -538,7 +538,8 @@ func TestCorrectIssueStats(t *testing.T) { func TestIssueForeignReference(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - issue := unittest.AssertExistsAndLoadBean(t, &Issue{ID: 1}).(*Issue) + issue := unittest.AssertExistsAndLoadBean(t, &Issue{ID: 4}).(*Issue) + assert.NotEqualValues(t, issue.Index, issue.ID) // make sure they are different to avoid false positive // it is fine for an issue to not have a foreign reference err := issue.LoadAttributes() From fdd4770dbfbc66afca523b70c743401128f513cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Fri, 11 Feb 2022 15:11:30 +0100 Subject: [PATCH 06/10] add comment for ForeignTypeIssue --- models/foreignreference.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/foreignreference.go b/models/foreignreference.go index eec3b9de3811c..3298cb39c5a7d 100644 --- a/models/foreignreference.go +++ b/models/foreignreference.go @@ -8,6 +8,7 @@ import ( "code.gitea.io/gitea/models/db" ) +// ForeignTypeIssue is to be used for the Type field in ForeignReference const ForeignTypeIssue = "issue" // ForeignReference represents external references From 7197bf2290384663857e550c1dadcdee5ed1d880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Sun, 13 Feb 2022 21:08:59 +0100 Subject: [PATCH 07/10] s/GlobalID/GlobalIndex/ s/LocalID/LocalIndex/ --- models/error.go | 40 ++++++++-------- models/foreignreference.go | 8 ++-- models/issue.go | 24 +++++----- models/issue_test.go | 18 +++---- models/migrate.go | 2 +- models/migrate_test.go | 10 ++-- modules/migration/comment.go | 4 +- modules/migration/issue.go | 40 ++++++++-------- modules/migration/pullrequest.go | 6 +-- modules/migration/review.go | 4 +- services/migrations/codebase.go | 4 +- services/migrations/gitea_downloader.go | 50 ++++++++++---------- services/migrations/gitea_downloader_test.go | 4 +- services/migrations/gitea_uploader.go | 8 ++-- services/migrations/github.go | 48 +++++++++---------- services/migrations/github_test.go | 10 ++-- services/migrations/gitlab.go | 48 +++++++++---------- services/migrations/gitlab_test.go | 14 +++--- services/migrations/gogs.go | 30 ++++++------ services/migrations/gogs_test.go | 2 +- services/migrations/onedev.go | 44 ++++++++--------- services/migrations/onedev_test.go | 20 ++++---- services/migrations/restore.go | 4 +- 23 files changed, 221 insertions(+), 221 deletions(-) diff --git a/models/error.go b/models/error.go index db52523b72bd7..87d0c7a77f2df 100644 --- a/models/error.go +++ b/models/error.go @@ -1395,36 +1395,36 @@ func (err ErrNotValidReviewRequest) Error() string { // \/ \/ /_____/ \/ \/ \/ \/ \/ \/ \/ \/ // -// ErrLocalIDNotExist represents a "LocalIDNotExist" kind of error. -type ErrLocalIDNotExist struct { - RepoID int64 - ForeignID int64 - Type string +// ErrLocalIndexNotExist represents a "LocalIndexNotExist" kind of error. +type ErrLocalIndexNotExist struct { + RepoID int64 + ForeignIndex int64 + Type string } -// ErrLocalIDNotExist checks if an error is a ErrLocalIDNotExist. -func IsErrLocalIDNotExist(err error) bool { - _, ok := err.(ErrLocalIDNotExist) +// ErrLocalIndexNotExist checks if an error is a ErrLocalIndexNotExist. +func IsErrLocalIndexNotExist(err error) bool { + _, ok := err.(ErrLocalIndexNotExist) return ok } -func (err ErrLocalIDNotExist) Error() string { - return fmt.Sprintf("repository %d has no LocalID for ForeignID %d of type %s", err.RepoID, err.ForeignID, err.Type) +func (err ErrLocalIndexNotExist) Error() string { + return fmt.Sprintf("repository %d has no LocalIndex for ForeignIndex %d of type %s", err.RepoID, err.ForeignIndex, err.Type) } -// ErrForeignIDNotExist represents a "ForeignIDNotExist" kind of error. -type ErrForeignIDNotExist struct { - RepoID int64 - LocalID int64 - Type string +// ErrForeignIndexNotExist represents a "ForeignIndexNotExist" kind of error. +type ErrForeignIndexNotExist struct { + RepoID int64 + LocalIndex int64 + Type string } -// ErrForeignIDNotExist checks if an error is a ErrForeignIDNotExist. -func IsErrForeignIDNotExist(err error) bool { - _, ok := err.(ErrForeignIDNotExist) +// ErrForeignIndexNotExist checks if an error is a ErrForeignIndexNotExist. +func IsErrForeignIndexNotExist(err error) bool { + _, ok := err.(ErrForeignIndexNotExist) return ok } -func (err ErrForeignIDNotExist) Error() string { - return fmt.Sprintf("repository %d has no ForeignID for LocalID %d of type %s", err.RepoID, err.LocalID, err.Type) +func (err ErrForeignIndexNotExist) Error() string { + return fmt.Sprintf("repository %d has no ForeignIndex for LocalIndex %d of type %s", err.RepoID, err.LocalIndex, err.Type) } diff --git a/models/foreignreference.go b/models/foreignreference.go index 3298cb39c5a7d..fb82aca32e1d8 100644 --- a/models/foreignreference.go +++ b/models/foreignreference.go @@ -14,10 +14,10 @@ const ForeignTypeIssue = "issue" // ForeignReference represents external references type ForeignReference struct { // RepoID is the first column in all indices. now we only need 2 indices: (repo, local) and (repo, foreign, type) - RepoID int64 `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" ` - LocalID int64 `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID. - ForeignID string `xorm:"INDEX UNIQUE(repo_foreign_type)"` - Type string `xorm:"VARCHAR(8) INDEX UNIQUE(repo_foreign_type)"` + RepoID int64 `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" ` + LocalIndex int64 `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID. + ForeignIndex string `xorm:"INDEX UNIQUE(repo_foreign_type)"` + Type string `xorm:"VARCHAR(8) INDEX UNIQUE(repo_foreign_type)"` } func init() { diff --git a/models/issue.go b/models/issue.go index 6508d609ee77f..94a95327e65e5 100644 --- a/models/issue.go +++ b/models/issue.go @@ -277,15 +277,15 @@ func (issue *Issue) loadForeignReference(ctx context.Context) (err error) { return nil } reference := &ForeignReference{ - LocalID: issue.Index, - RepoID: issue.RepoID, - Type: ForeignTypeIssue, + LocalIndex: issue.Index, + RepoID: issue.RepoID, + Type: ForeignTypeIssue, } has, err := db.GetEngine(ctx).Get(reference) if err != nil { return err } else if !has { - return ErrForeignIDNotExist{issue.RepoID, issue.Index, ForeignTypeIssue} + return ErrForeignIndexNotExist{issue.RepoID, issue.Index, ForeignTypeIssue} } issue.ForeignReference = reference return nil @@ -352,7 +352,7 @@ func (issue *Issue) loadAttributes(ctx context.Context) (err error) { } } - if err = issue.loadForeignReference(ctx); err != nil && !IsErrForeignIDNotExist(err) { + if err = issue.loadForeignReference(ctx); err != nil && !IsErrForeignIndexNotExist(err) { return err } @@ -1134,20 +1134,20 @@ func GetIssueByIndex(repoID, index int64) (*Issue, error) { return issue, nil } -// GetIssueByForeignID returns raw issue by foreign ID -func GetIssueByForeignID(ctx context.Context, repoID, foreignID int64) (*Issue, error) { +// GetIssueByForeignIndex returns raw issue by foreign ID +func GetIssueByForeignIndex(ctx context.Context, repoID, foreignIndex int64) (*Issue, error) { reference := &ForeignReference{ - RepoID: repoID, - ForeignID: strconv.FormatInt(foreignID, 10), - Type: ForeignTypeIssue, + RepoID: repoID, + ForeignIndex: strconv.FormatInt(foreignIndex, 10), + Type: ForeignTypeIssue, } has, err := db.GetEngine(ctx).Get(reference) if err != nil { return nil, err } else if !has { - return nil, ErrLocalIDNotExist{repoID, foreignID, ForeignTypeIssue} + return nil, ErrLocalIndexNotExist{repoID, foreignIndex, ForeignTypeIssue} } - return GetIssueByIndex(repoID, reference.LocalID) + return GetIssueByIndex(repoID, reference.LocalIndex) } // GetIssueWithAttrsByIndex returns issue by index in a repository. diff --git a/models/issue_test.go b/models/issue_test.go index a2de314129532..b2799190f925a 100644 --- a/models/issue_test.go +++ b/models/issue_test.go @@ -546,24 +546,24 @@ func TestIssueForeignReference(t *testing.T) { assert.NoError(t, err) assert.Nil(t, issue.ForeignReference) - var foreignID int64 = 12345 - _, err = GetIssueByForeignID(context.Background(), issue.RepoID, foreignID) - assert.True(t, IsErrLocalIDNotExist(err)) + var foreignIndex int64 = 12345 + _, err = GetIssueByForeignIndex(context.Background(), issue.RepoID, foreignIndex) + assert.True(t, IsErrLocalIndexNotExist(err)) _, err = db.GetEngine(db.DefaultContext).Insert(&ForeignReference{ - LocalID: issue.Index, - ForeignID: strconv.FormatInt(foreignID, 10), - RepoID: issue.RepoID, - Type: ForeignTypeIssue, + LocalIndex: issue.Index, + ForeignIndex: strconv.FormatInt(foreignIndex, 10), + RepoID: issue.RepoID, + Type: ForeignTypeIssue, }) assert.NoError(t, err) err = issue.LoadAttributes() assert.NoError(t, err) - assert.EqualValues(t, issue.ForeignReference.ForeignID, strconv.FormatInt(foreignID, 10)) + assert.EqualValues(t, issue.ForeignReference.ForeignIndex, strconv.FormatInt(foreignIndex, 10)) - found, err := GetIssueByForeignID(context.Background(), issue.RepoID, foreignID) + found, err := GetIssueByForeignIndex(context.Background(), issue.RepoID, foreignIndex) assert.NoError(t, err) assert.EqualValues(t, found.Index, issue.Index) } diff --git a/models/migrate.go b/models/migrate.go index d0813be503a76..2470fd4336bd7 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -84,7 +84,7 @@ func insertIssue(ctx context.Context, issue *Issue) error { } if issue.ForeignReference != nil { - issue.ForeignReference.LocalID = issue.Index + issue.ForeignReference.LocalIndex = issue.Index if _, err := sess.Insert(issue.ForeignReference); err != nil { return err } diff --git a/models/migrate_test.go b/models/migrate_test.go index b9a16d4bce2e5..a6c8c37131730 100644 --- a/models/migrate_test.go +++ b/models/migrate_test.go @@ -46,7 +46,7 @@ func assertCreateIssues(t *testing.T, isPull bool) { UserID: owner.ID, } - foreignID := int64(12345) + foreignIndex := int64(12345) title := "issuetitle1" is := &Issue{ RepoID: repo.ID, @@ -61,9 +61,9 @@ func assertCreateIssues(t *testing.T, isPull bool) { Labels: []*Label{label}, Reactions: []*Reaction{reaction}, ForeignReference: &ForeignReference{ - ForeignID: strconv.FormatInt(foreignID, 10), - RepoID: repo.ID, - Type: ForeignTypeIssue, + ForeignIndex: strconv.FormatInt(foreignIndex, 10), + RepoID: repo.ID, + Type: ForeignTypeIssue, }, } err := InsertIssues(is) @@ -73,7 +73,7 @@ func assertCreateIssues(t *testing.T, isPull bool) { assert.Nil(t, i.ForeignReference) err = i.LoadAttributes() assert.NoError(t, err) - assert.EqualValues(t, strconv.FormatInt(foreignID, 10), i.ForeignReference.ForeignID) + assert.EqualValues(t, strconv.FormatInt(foreignIndex, 10), i.ForeignReference.ForeignIndex) unittest.AssertExistsAndLoadBean(t, &Reaction{Type: "heart", UserID: owner.ID, IssueID: i.ID}) } diff --git a/modules/migration/comment.go b/modules/migration/comment.go index 9e895fbb2f401..0447689b74cd6 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -9,8 +9,8 @@ import "time" // Commentable can be commented upon type Commentable interface { - GetLocalID() int64 - GetForeignID() int64 + GetLocalIndex() int64 + GetForeignIndex() int64 GetContext() DownloaderContext } diff --git a/modules/migration/issue.go b/modules/migration/issue.go index fd2d161435b6c..78f648dd2db57 100644 --- a/modules/migration/issue.go +++ b/modules/migration/issue.go @@ -9,24 +9,24 @@ import "time" // Issue is a standard issue information type Issue struct { - Number int64 `json:"number"` - PosterID int64 `yaml:"poster_id" json:"poster_id"` - PosterName string `yaml:"poster_name" json:"poster_name"` - PosterEmail string `yaml:"poster_email" json:"poster_email"` - Title string `json:"title"` - Content string `json:"content"` - Ref string `json:"ref"` - Milestone string `json:"milestone"` - State string `json:"state"` // closed, open - IsLocked bool `yaml:"is_locked" json:"is_locked"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - Closed *time.Time `json:"closed"` - Labels []*Label `json:"labels"` - Reactions []*Reaction `json:"reactions"` - Assignees []string `json:"assignees"` - ForeignID int64 `json:"foreign_id"` - Context DownloaderContext `yaml:"-"` + Number int64 `json:"number"` + PosterID int64 `yaml:"poster_id" json:"poster_id"` + PosterName string `yaml:"poster_name" json:"poster_name"` + PosterEmail string `yaml:"poster_email" json:"poster_email"` + Title string `json:"title"` + Content string `json:"content"` + Ref string `json:"ref"` + Milestone string `json:"milestone"` + State string `json:"state"` // closed, open + IsLocked bool `yaml:"is_locked" json:"is_locked"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Closed *time.Time `json:"closed"` + Labels []*Label `json:"labels"` + Reactions []*Reaction `json:"reactions"` + Assignees []string `json:"assignees"` + ForeignIndex int64 `json:"foreign_id"` + Context DownloaderContext `yaml:"-"` } // GetExternalName ExternalUserMigrated interface @@ -35,6 +35,6 @@ func (i *Issue) GetExternalName() string { return i.PosterName } // GetExternalID ExternalUserMigrated interface func (i *Issue) GetExternalID() int64 { return i.PosterID } -func (i *Issue) GetLocalID() int64 { return i.Number } -func (i *Issue) GetForeignID() int64 { return i.ForeignID } +func (i *Issue) GetLocalIndex() int64 { return i.Number } +func (i *Issue) GetForeignIndex() int64 { return i.ForeignIndex } func (i *Issue) GetContext() DownloaderContext { return i.Context } diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index b75903bd42ff5..eaa0dd45e2c4b 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -35,12 +35,12 @@ type PullRequest struct { Assignees []string IsLocked bool `yaml:"is_locked"` Reactions []*Reaction - ForeignID int64 + ForeignIndex int64 Context DownloaderContext `yaml:"-"` } -func (p *PullRequest) GetLocalID() int64 { return p.Number } -func (p *PullRequest) GetForeignID() int64 { return p.ForeignID } +func (p *PullRequest) GetLocalIndex() int64 { return p.Number } +func (p *PullRequest) GetForeignIndex() int64 { return p.ForeignIndex } func (p *PullRequest) GetContext() DownloaderContext { return p.Context } // IsForkPullRequest returns true if the pull request from a forked repository but not the same repository diff --git a/modules/migration/review.go b/modules/migration/review.go index fc698f1de3a88..e4db33d98fd38 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -8,8 +8,8 @@ import "time" // Reviewable can be reviewed type Reviewable interface { - GetLocalID() int64 - GetForeignID() int64 + GetLocalIndex() int64 + GetForeignIndex() int64 } // enumerate all review states diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index d70ee58224c5c..bb74c0a49d1dd 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -392,7 +392,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, Labels: []*base.Label{ {Name: issue.Type.Name}, }, - ForeignID: issue.TicketID.Value, + ForeignIndex: issue.TicketID.Value, Context: codebaseIssueContext{ Comments: comments[1:], }, @@ -559,7 +559,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq SHA: d.getHeadCommit(rawMergeRequest.TargetRef), RepoName: d.repoName, }, - ForeignID: rawMergeRequest.ID.Value, + ForeignIndex: rawMergeRequest.ID.Value, Context: codebaseIssueContext{ Comments: comments[1:], }, diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index fded3cdf8ec9e..3c02e112ca73e 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -415,22 +415,22 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err } allIssues = append(allIssues, &base.Issue{ - Title: issue.Title, - Number: issue.Index, - PosterID: issue.Poster.ID, - PosterName: issue.Poster.UserName, - PosterEmail: issue.Poster.Email, - Content: issue.Body, - Milestone: milestone, - State: string(issue.State), - Created: issue.Created, - Updated: issue.Updated, - Closed: issue.Closed, - Reactions: reactions, - Labels: labels, - Assignees: assignees, - IsLocked: issue.IsLocked, - ForeignID: issue.Index, + Title: issue.Title, + Number: issue.Index, + PosterID: issue.Poster.ID, + PosterName: issue.Poster.UserName, + PosterEmail: issue.Poster.Email, + Content: issue.Body, + Milestone: milestone, + State: string(issue.State), + Created: issue.Created, + Updated: issue.Updated, + Closed: issue.Closed, + Reactions: reactions, + Labels: labels, + Assignees: assignees, + IsLocked: issue.IsLocked, + ForeignIndex: issue.Index, }) } @@ -453,26 +453,26 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com default: } - comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ + comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ PageSize: g.maxPerPage, Page: i, }}) if err != nil { - return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", commentable.GetForeignID(), err) + return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", commentable.GetForeignIndex(), err) } for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignID(), comment.ID, g.repoOwner, g.repoName, err) + log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err) if err2 := admin_model.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { + fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { log.Error("create repository notice failed: ", err2) } } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalID(), + IssueIndex: commentable.GetLocalIndex(), Index: comment.ID, PosterID: comment.Poster.ID, PosterName: comment.Poster.UserName, @@ -602,7 +602,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques RepoName: g.repoName, OwnerName: g.repoOwner, }, - ForeignID: pr.Index, + ForeignIndex: pr.Index, }) } @@ -630,7 +630,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review default: } - prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, reviewable.GetForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ + prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, reviewable.GetForeignIndex(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ Page: i, PageSize: g.maxPerPage, }}) @@ -640,7 +640,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review for _, pr := range prl { - rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, reviewable.GetForeignID(), pr.ID) + rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID) if err != nil { return nil, err } @@ -666,7 +666,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review allReviews = append(allReviews, &base.Review{ ID: pr.ID, - IssueIndex: reviewable.GetLocalID(), + IssueIndex: reviewable.GetLocalIndex(), ReviewerID: pr.Reviewer.ID, ReviewerName: pr.Reviewer.UserName, Official: pr.Official, diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index acf93764788c8..dc6903e854742 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -198,7 +198,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{Number: 4, ForeignID: 4}) + comments, _, err := downloader.GetComments(&base.Issue{Number: 4, ForeignIndex: 4}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -263,7 +263,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignID: 7}) + reviews, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignIndex: 7}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 5f96f06e9f068..0b6e047deb250 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -375,10 +375,10 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()), UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()), ForeignReference: &models.ForeignReference{ - LocalID: issue.GetLocalID(), - ForeignID: strconv.FormatInt(issue.GetForeignID(), 10), - RepoID: g.repo.ID, - Type: models.ForeignTypeIssue, + LocalIndex: issue.GetLocalIndex(), + ForeignIndex: strconv.FormatInt(issue.GetForeignIndex(), 10), + RepoID: g.repo.ID, + Type: models.ForeignTypeIssue, }, } diff --git a/services/migrations/github.go b/services/migrations/github.go index ca9a167a1e912..faf0cf0794179 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -446,22 +446,22 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, } allIssues = append(allIssues, &base.Issue{ - Title: *issue.Title, - Number: int64(*issue.Number), - PosterID: issue.GetUser().GetID(), - PosterName: issue.GetUser().GetLogin(), - PosterEmail: issue.GetUser().GetEmail(), - Content: issue.GetBody(), - Milestone: issue.GetMilestone().GetTitle(), - State: issue.GetState(), - Created: issue.GetCreatedAt(), - Updated: issue.GetUpdatedAt(), - Labels: labels, - Reactions: reactions, - Closed: issue.ClosedAt, - IsLocked: issue.GetLocked(), - Assignees: assignees, - ForeignID: int64(*issue.Number), + Title: *issue.Title, + Number: int64(*issue.Number), + PosterID: issue.GetUser().GetID(), + PosterName: issue.GetUser().GetLogin(), + PosterEmail: issue.GetUser().GetEmail(), + Content: issue.GetBody(), + Milestone: issue.GetMilestone().GetTitle(), + State: issue.GetState(), + Created: issue.GetCreatedAt(), + Updated: issue.GetUpdatedAt(), + Labels: labels, + Reactions: reactions, + Closed: issue.ClosedAt, + IsLocked: issue.GetLocked(), + Assignees: assignees, + ForeignIndex: int64(*issue.Number), }) } @@ -494,7 +494,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. } for { g.waitAndPickClient() - comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignID()), opt) + comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } @@ -527,7 +527,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalID(), + IssueIndex: commentable.GetLocalIndex(), Index: comment.GetID(), PosterID: comment.GetUser().GetID(), PosterName: comment.GetUser().GetLogin(), @@ -705,9 +705,9 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq RepoName: pr.GetBase().GetRepo().GetName(), OwnerName: pr.GetBase().GetUser().GetLogin(), }, - PatchURL: pr.GetPatchURL(), - Reactions: reactions, - ForeignID: int64(*pr.Number), + PatchURL: pr.GetPatchURL(), + Reactions: reactions, + ForeignIndex: int64(*pr.Number), }) } @@ -780,21 +780,21 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } for { g.waitAndPickClient() - reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignID()), opt) + reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } g.setRate(&resp.Rate) for _, review := range reviews { r := convertGithubReview(review) - r.IssueIndex = reviewable.GetLocalID() + r.IssueIndex = reviewable.GetLocalIndex() // retrieve all review comments opt2 := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.waitAndPickClient() - reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignID()), review.GetID(), opt2) + reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } diff --git a/services/migrations/github_test.go b/services/migrations/github_test.go index a9e71a24b8078..90c1fcaef5b6b 100644 --- a/services/migrations/github_test.go +++ b/services/migrations/github_test.go @@ -215,7 +215,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 2, ForeignID: 2}) + comments, _, err := downloader.GetComments(&base.Issue{Number: 2, ForeignIndex: 2}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -284,7 +284,7 @@ func TestGitHubDownloadRepo(t *testing.T) { Merged: true, MergedTime: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)), MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", - ForeignID: 3, + ForeignIndex: 3, }, { Number: 4, @@ -331,11 +331,11 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, - ForeignID: 4, + ForeignIndex: 4, }, }, prs) - reviews, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignID: 3}) + reviews, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignIndex: 3}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -367,7 +367,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignID: 4}) + reviews, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignIndex: 4}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index f3ed00dd60d87..d3a034e27c52a 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -411,21 +411,21 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er } allIssues = append(allIssues, &base.Issue{ - Title: issue.Title, - Number: int64(issue.IID), - PosterID: int64(issue.Author.ID), - PosterName: issue.Author.Username, - Content: issue.Description, - Milestone: milestone, - State: issue.State, - Created: *issue.CreatedAt, - Labels: labels, - Reactions: reactions, - Closed: issue.ClosedAt, - IsLocked: issue.DiscussionLocked, - Updated: *issue.UpdatedAt, - ForeignID: int64(issue.IID), - Context: gitlabIssueContext{IsMergeRequest: false}, + Title: issue.Title, + Number: int64(issue.IID), + PosterID: int64(issue.Author.ID), + PosterName: issue.Author.Username, + Content: issue.Description, + Milestone: milestone, + State: issue.State, + Created: *issue.CreatedAt, + Labels: labels, + Reactions: reactions, + Closed: issue.ClosedAt, + IsLocked: issue.DiscussionLocked, + Updated: *issue.UpdatedAt, + ForeignIndex: int64(issue.IID), + Context: gitlabIssueContext{IsMergeRequest: false}, }) // increment issueCount, to be used in GetPullRequests() @@ -452,12 +452,12 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co var resp *gitlab.Response var err error if !context.IsMergeRequest { - comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignID()), &gitlab.ListIssueDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } else { - comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) @@ -471,7 +471,7 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co if !comment.IndividualNote { for _, note := range comment.Notes { allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalID(), + IssueIndex: commentable.GetLocalIndex(), Index: int64(note.ID), PosterID: int64(note.Author.ID), PosterName: note.Author.Username, @@ -483,7 +483,7 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co } else { c := comment.Notes[0] allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalID(), + IssueIndex: commentable.GetLocalIndex(), Index: int64(c.ID), PosterID: int64(c.Author.ID), PosterName: c.Author.Username, @@ -606,9 +606,9 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque RepoName: g.repoName, OwnerName: pr.Author.Username, }, - PatchURL: pr.WebURL + ".patch", - ForeignID: int64(pr.IID), - Context: gitlabIssueContext{IsMergeRequest: true}, + PatchURL: pr.WebURL + ".patch", + ForeignIndex: int64(pr.IID), + Context: gitlabIssueContext{IsMergeRequest: true}, }) } @@ -617,7 +617,7 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque // GetReviews returns pull requests review func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { - approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignID()), gitlab.WithContext(g.ctx)) + approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(g.ctx)) if err != nil { if resp != nil && resp.StatusCode == 404 { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) @@ -638,7 +638,7 @@ func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie reviews := make([]*base.Review, 0, len(approvals.ApprovedBy)) for _, user := range approvals.ApprovedBy { reviews = append(reviews, &base.Review{ - IssueIndex: reviewable.GetLocalID(), + IssueIndex: reviewable.GetLocalIndex(), ReviewerID: int64(user.User.ID), ReviewerName: user.User.Username, CreatedAt: createdAt, diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index 89b822588edd1..52edb2af8fdfc 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -215,9 +215,9 @@ func TestGitlabDownloadRepo(t *testing.T) { }, issues) comments, _, err := downloader.GetComments(&base.Issue{ - Number: 2, - ForeignID: 2, - Context: gitlabIssueContext{IsMergeRequest: false}, + Number: 2, + ForeignIndex: 2, + Context: gitlabIssueContext{IsMergeRequest: false}, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -299,12 +299,12 @@ func TestGitlabDownloadRepo(t *testing.T) { Merged: false, MergedTime: nil, MergeCommitSHA: "", - ForeignID: 2, + ForeignIndex: 2, Context: gitlabIssueContext{IsMergeRequest: true}, }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignID: 1}) + rvs, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignIndex: 1}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -323,7 +323,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, rvs) - rvs, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignID: 2}) + rvs, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignIndex: 2}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -465,7 +465,7 @@ func TestGitlabGetReviews(t *testing.T) { mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock) id := int64(testCase.prID) - rvs, err := downloader.GetReviews(&base.Issue{Number: id, ForeignID: id}) + rvs, err := downloader.GetReviews(&base.Issue{Number: id, ForeignIndex: id}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{&review}, rvs) } diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index 92dfcce455056..a28033218eac5 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -226,7 +226,7 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, 100) - comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignID()) + comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex()) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } @@ -235,7 +235,7 @@ func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comm continue } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalID(), + IssueIndex: commentable.GetLocalIndex(), Index: comment.ID, PosterID: comment.Poster.ID, PosterName: comment.Poster.Login, @@ -288,19 +288,19 @@ func convertGogsIssue(issue *gogs.Issue) *base.Issue { } return &base.Issue{ - Title: issue.Title, - Number: issue.Index, - PosterID: issue.Poster.ID, - PosterName: issue.Poster.Login, - PosterEmail: issue.Poster.Email, - Content: issue.Body, - Milestone: milestone, - State: string(issue.State), - Created: issue.Created, - Updated: issue.Updated, - Labels: labels, - Closed: closed, - ForeignID: issue.Index, + Title: issue.Title, + Number: issue.Index, + PosterID: issue.Poster.ID, + PosterName: issue.Poster.Login, + PosterEmail: issue.Poster.Email, + Content: issue.Body, + Milestone: milestone, + State: string(issue.State), + Created: issue.Created, + Updated: issue.Updated, + Labels: labels, + Closed: closed, + ForeignIndex: issue.Index, } } diff --git a/services/migrations/gogs_test.go b/services/migrations/gogs_test.go index 8be3472b6625f..501161b610dad 100644 --- a/services/migrations/gogs_test.go +++ b/services/migrations/gogs_test.go @@ -111,7 +111,7 @@ func TestGogsDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 1, ForeignID: 1}) + comments, _, err := downloader.GetComments(&base.Issue{Number: 1, ForeignIndex: 1}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index d53ce24e60bc1..d4b30939ce954 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -336,18 +336,18 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er } poster := d.tryGetUser(issue.SubmitterID) issues = append(issues, &base.Issue{ - Title: issue.Title, - Number: issue.Number, - PosterName: poster.Name, - PosterEmail: poster.Email, - Content: issue.Description, - Milestone: d.milestoneMap[milestoneID], - State: state, - Created: issue.SubmitDate, - Updated: issue.SubmitDate, - Labels: []*base.Label{label}, - ForeignID: issue.ID, - Context: onedevIssueContext{IsPullRequest: false}, + Title: issue.Title, + Number: issue.Number, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: issue.Description, + Milestone: d.milestoneMap[milestoneID], + State: state, + Created: issue.SubmitDate, + Updated: issue.SubmitDate, + Labels: []*base.Label{label}, + ForeignIndex: issue.ID, + Context: onedevIssueContext{IsPullRequest: false}, }) if d.maxIssueIndex < issue.Number { @@ -374,9 +374,9 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co var endpoint string if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", commentable.GetForeignID()) + endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", commentable.GetForeignIndex()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/comments", commentable.GetForeignID()) + endpoint = fmt.Sprintf("/api/issues/%d/comments", commentable.GetForeignIndex()) } err := d.callAPI( @@ -395,9 +395,9 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co }, 0, 100) if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", commentable.GetForeignID()) + endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", commentable.GetForeignIndex()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/changes", commentable.GetForeignID()) + endpoint = fmt.Sprintf("/api/issues/%d/changes", commentable.GetForeignIndex()) } err = d.callAPI( @@ -416,7 +416,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } poster := d.tryGetUser(comment.UserID) comments = append(comments, &base.Comment{ - IssueIndex: commentable.GetLocalID(), + IssueIndex: commentable.GetLocalIndex(), Index: comment.ID, PosterID: poster.ID, PosterName: poster.Name, @@ -441,7 +441,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co poster := d.tryGetUser(change.UserID) comments = append(comments, &base.Comment{ - IssueIndex: commentable.GetLocalID(), + IssueIndex: commentable.GetLocalIndex(), PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, @@ -539,8 +539,8 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque SHA: mergePreview.TargetHeadCommitHash, RepoName: d.repoName, }, - ForeignID: pr.ID, - Context: onedevIssueContext{IsPullRequest: true}, + ForeignIndex: pr.ID, + Context: onedevIssueContext{IsPullRequest: true}, }) } @@ -560,7 +560,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie }, 0, 100) err := d.callAPI( - fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignID()), + fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()), nil, &rawReviews, ) @@ -584,7 +584,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie poster := d.tryGetUser(review.UserID) reviews = append(reviews, &base.Review{ - IssueIndex: reviewable.GetLocalID(), + IssueIndex: reviewable.GetLocalIndex(), ReviewerID: poster.ID, ReviewerName: poster.Name, Content: content, diff --git a/services/migrations/onedev_test.go b/services/migrations/onedev_test.go index 3c8798124bcb9..55ae7da1fcc66 100644 --- a/services/migrations/onedev_test.go +++ b/services/migrations/onedev_test.go @@ -74,8 +74,8 @@ func TestOneDevDownloadRepo(t *testing.T) { Name: "Improvement", }, }, - ForeignID: 398, - Context: onedevIssueContext{IsPullRequest: false}, + ForeignIndex: 398, + Context: onedevIssueContext{IsPullRequest: false}, }, { Number: 3, @@ -91,15 +91,15 @@ func TestOneDevDownloadRepo(t *testing.T) { Name: "New Feature", }, }, - ForeignID: 397, - Context: onedevIssueContext{IsPullRequest: false}, + ForeignIndex: 397, + Context: onedevIssueContext{IsPullRequest: false}, }, }, issues) comments, _, err := downloader.GetComments(&base.Issue{ - Number: 4, - ForeignID: 398, - Context: onedevIssueContext{IsPullRequest: false}, + Number: 4, + ForeignIndex: 398, + Context: onedevIssueContext{IsPullRequest: false}, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -133,12 +133,12 @@ func TestOneDevDownloadRepo(t *testing.T) { SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", RepoName: "go-gitea-test_repo", }, - ForeignID: 186, - Context: onedevIssueContext{IsPullRequest: true}, + ForeignIndex: 186, + Context: onedevIssueContext{IsPullRequest: true}, }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignID: 186}) + rvs, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignIndex: 186}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/restore.go b/services/migrations/restore.go index f8083de57bf01..8c9654a7e3c2b 100644 --- a/services/migrations/restore.go +++ b/services/migrations/restore.go @@ -199,7 +199,7 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, // GetComments returns comments according issueNumber func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { comments := make([]*base.Comment, 0, 10) - p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignID())) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignIndex())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -250,7 +250,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq // GetReviews returns pull requests review func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { reviews := make([]*base.Review, 0, 10) - p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignID())) + p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignIndex())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { From 00c2098d661d5899c661156d7b13da0746f62259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Sun, 13 Feb 2022 21:15:03 +0100 Subject: [PATCH 08/10] define all ForeignType and increase the size from 8 to 16 --- models/foreignreference.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/models/foreignreference.go b/models/foreignreference.go index fb82aca32e1d8..69e6846f9a592 100644 --- a/models/foreignreference.go +++ b/models/foreignreference.go @@ -8,8 +8,15 @@ import ( "code.gitea.io/gitea/models/db" ) -// ForeignTypeIssue is to be used for the Type field in ForeignReference -const ForeignTypeIssue = "issue" +// ForeignType* are valid values for the Type field of ForeignReference +const ( + ForeignTypeIssue = "issue" + ForeignTypePullRequest = "pull_request" + ForeignTypeComment = "comment" + ForeignTypeReview = "review" + ForeignTypeReviewComment = "review_comment" + ForeignTypeRelease = "release" +) // ForeignReference represents external references type ForeignReference struct { @@ -17,7 +24,7 @@ type ForeignReference struct { RepoID int64 `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" ` LocalIndex int64 `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID. ForeignIndex string `xorm:"INDEX UNIQUE(repo_foreign_type)"` - Type string `xorm:"VARCHAR(8) INDEX UNIQUE(repo_foreign_type)"` + Type string `xorm:"VARCHAR(16) INDEX UNIQUE(repo_foreign_type)"` } func init() { From 53bd9779a1dbf895f2134029564e1d1d3d47b499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Mon, 14 Feb 2022 23:36:36 +0100 Subject: [PATCH 09/10] move foreignreference.go & error to models/foreignreference --- models/error.go | 43 ------------------- models/foreignreference/error.go | 43 +++++++++++++++++++ .../foreignreference.go | 16 +++---- models/issue.go | 37 ++++++++++------ models/issue_test.go | 7 +-- models/migrate_test.go | 5 ++- services/migrations/gitea_uploader.go | 5 ++- 7 files changed, 84 insertions(+), 72 deletions(-) create mode 100644 models/foreignreference/error.go rename models/{ => foreignreference}/foreignreference.go (71%) diff --git a/models/error.go b/models/error.go index 87d0c7a77f2df..f0e8751d75fdc 100644 --- a/models/error.go +++ b/models/error.go @@ -1385,46 +1385,3 @@ func (err ErrNotValidReviewRequest) Error() string { err.UserID, err.RepoID) } - -// -// ___________ .__ __________ _____ -// \_ _____/__________ ____ |__| ____ ____\______ \ _____/ ____\___________ ____ ____ ____ ____ -// | __)/ _ \_ __ \_/ __ \| |/ ___\ / \| _// __ \ __\/ __ \_ __ \_/ __ \ / \_/ ___\/ __ \ -// | \( <_> ) | \/\ ___/| / /_/ > | \ | \ ___/| | \ ___/| | \/\ ___/| | \ \__\ ___/ -// \___ / \____/|__| \___ >__\___ /|___| /____|_ /\___ >__| \___ >__| \___ >___| /\___ >___ > -// \/ \/ /_____/ \/ \/ \/ \/ \/ \/ \/ \/ -// - -// ErrLocalIndexNotExist represents a "LocalIndexNotExist" kind of error. -type ErrLocalIndexNotExist struct { - RepoID int64 - ForeignIndex int64 - Type string -} - -// ErrLocalIndexNotExist checks if an error is a ErrLocalIndexNotExist. -func IsErrLocalIndexNotExist(err error) bool { - _, ok := err.(ErrLocalIndexNotExist) - return ok -} - -func (err ErrLocalIndexNotExist) Error() string { - return fmt.Sprintf("repository %d has no LocalIndex for ForeignIndex %d of type %s", err.RepoID, err.ForeignIndex, err.Type) -} - -// ErrForeignIndexNotExist represents a "ForeignIndexNotExist" kind of error. -type ErrForeignIndexNotExist struct { - RepoID int64 - LocalIndex int64 - Type string -} - -// ErrForeignIndexNotExist checks if an error is a ErrForeignIndexNotExist. -func IsErrForeignIndexNotExist(err error) bool { - _, ok := err.(ErrForeignIndexNotExist) - return ok -} - -func (err ErrForeignIndexNotExist) Error() string { - return fmt.Sprintf("repository %d has no ForeignIndex for LocalIndex %d of type %s", err.RepoID, err.LocalIndex, err.Type) -} diff --git a/models/foreignreference/error.go b/models/foreignreference/error.go new file mode 100644 index 0000000000000..d783a087301ea --- /dev/null +++ b/models/foreignreference/error.go @@ -0,0 +1,43 @@ +// Copyright 2022 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package foreignreference + +import ( + "fmt" +) + +// ErrLocalIndexNotExist represents a "LocalIndexNotExist" kind of error. +type ErrLocalIndexNotExist struct { + RepoID int64 + ForeignIndex int64 + Type string +} + +// ErrLocalIndexNotExist checks if an error is a ErrLocalIndexNotExist. +func IsErrLocalIndexNotExist(err error) bool { + _, ok := err.(ErrLocalIndexNotExist) + return ok +} + +func (err ErrLocalIndexNotExist) Error() string { + return fmt.Sprintf("repository %d has no LocalIndex for ForeignIndex %d of type %s", err.RepoID, err.ForeignIndex, err.Type) +} + +// ErrForeignIndexNotExist represents a "ForeignIndexNotExist" kind of error. +type ErrForeignIndexNotExist struct { + RepoID int64 + LocalIndex int64 + Type string +} + +// ErrForeignIndexNotExist checks if an error is a ErrForeignIndexNotExist. +func IsErrForeignIndexNotExist(err error) bool { + _, ok := err.(ErrForeignIndexNotExist) + return ok +} + +func (err ErrForeignIndexNotExist) Error() string { + return fmt.Sprintf("repository %d has no ForeignIndex for LocalIndex %d of type %s", err.RepoID, err.LocalIndex, err.Type) +} diff --git a/models/foreignreference.go b/models/foreignreference/foreignreference.go similarity index 71% rename from models/foreignreference.go rename to models/foreignreference/foreignreference.go index 69e6846f9a592..cc657a0fbec6c 100644 --- a/models/foreignreference.go +++ b/models/foreignreference/foreignreference.go @@ -2,20 +2,20 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package models +package foreignreference import ( "code.gitea.io/gitea/models/db" ) -// ForeignType* are valid values for the Type field of ForeignReference +// Type* are valid values for the Type field of ForeignReference const ( - ForeignTypeIssue = "issue" - ForeignTypePullRequest = "pull_request" - ForeignTypeComment = "comment" - ForeignTypeReview = "review" - ForeignTypeReviewComment = "review_comment" - ForeignTypeRelease = "release" + TypeIssue = "issue" + TypePullRequest = "pull_request" + TypeComment = "comment" + TypeReview = "review" + TypeReviewComment = "review_comment" + TypeRelease = "release" ) // ForeignReference represents external references diff --git a/models/issue.go b/models/issue.go index 94a95327e65e5..ed94195ed1e6a 100644 --- a/models/issue.go +++ b/models/issue.go @@ -15,6 +15,7 @@ import ( admin_model "code.gitea.io/gitea/models/admin" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/foreignreference" "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" @@ -67,12 +68,12 @@ type Issue struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` ClosedUnix timeutil.TimeStamp `xorm:"INDEX"` - Attachments []*repo_model.Attachment `xorm:"-"` - Comments []*Comment `xorm:"-"` - Reactions ReactionList `xorm:"-"` - TotalTrackedTime int64 `xorm:"-"` - Assignees []*user_model.User `xorm:"-"` - ForeignReference *ForeignReference `xorm:"-"` + Attachments []*repo_model.Attachment `xorm:"-"` + Comments []*Comment `xorm:"-"` + Reactions ReactionList `xorm:"-"` + TotalTrackedTime int64 `xorm:"-"` + Assignees []*user_model.User `xorm:"-"` + ForeignReference *foreignreference.ForeignReference `xorm:"-"` // IsLocked limits commenting abilities to users on an issue // with write access @@ -276,16 +277,20 @@ func (issue *Issue) loadForeignReference(ctx context.Context) (err error) { if issue.ForeignReference != nil { return nil } - reference := &ForeignReference{ - LocalIndex: issue.Index, + reference := &foreignreference.ForeignReference{ RepoID: issue.RepoID, - Type: ForeignTypeIssue, + LocalIndex: issue.Index, + Type: foreignreference.TypeIssue, } has, err := db.GetEngine(ctx).Get(reference) if err != nil { return err } else if !has { - return ErrForeignIndexNotExist{issue.RepoID, issue.Index, ForeignTypeIssue} + return foreignreference.ErrForeignIndexNotExist{ + RepoID: issue.RepoID, + LocalIndex: issue.Index, + Type: foreignreference.TypeIssue, + } } issue.ForeignReference = reference return nil @@ -352,7 +357,7 @@ func (issue *Issue) loadAttributes(ctx context.Context) (err error) { } } - if err = issue.loadForeignReference(ctx); err != nil && !IsErrForeignIndexNotExist(err) { + if err = issue.loadForeignReference(ctx); err != nil && !foreignreference.IsErrForeignIndexNotExist(err) { return err } @@ -1136,16 +1141,20 @@ func GetIssueByIndex(repoID, index int64) (*Issue, error) { // GetIssueByForeignIndex returns raw issue by foreign ID func GetIssueByForeignIndex(ctx context.Context, repoID, foreignIndex int64) (*Issue, error) { - reference := &ForeignReference{ + reference := &foreignreference.ForeignReference{ RepoID: repoID, ForeignIndex: strconv.FormatInt(foreignIndex, 10), - Type: ForeignTypeIssue, + Type: foreignreference.TypeIssue, } has, err := db.GetEngine(ctx).Get(reference) if err != nil { return nil, err } else if !has { - return nil, ErrLocalIndexNotExist{repoID, foreignIndex, ForeignTypeIssue} + return nil, foreignreference.ErrLocalIndexNotExist{ + RepoID: repoID, + ForeignIndex: foreignIndex, + Type: foreignreference.TypeIssue, + } } return GetIssueByIndex(repoID, reference.LocalIndex) } diff --git a/models/issue_test.go b/models/issue_test.go index b2799190f925a..9afbce67c1f9e 100644 --- a/models/issue_test.go +++ b/models/issue_test.go @@ -14,6 +14,7 @@ import ( "time" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/foreignreference" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" @@ -548,13 +549,13 @@ func TestIssueForeignReference(t *testing.T) { var foreignIndex int64 = 12345 _, err = GetIssueByForeignIndex(context.Background(), issue.RepoID, foreignIndex) - assert.True(t, IsErrLocalIndexNotExist(err)) + assert.True(t, foreignreference.IsErrLocalIndexNotExist(err)) - _, err = db.GetEngine(db.DefaultContext).Insert(&ForeignReference{ + _, err = db.GetEngine(db.DefaultContext).Insert(&foreignreference.ForeignReference{ LocalIndex: issue.Index, ForeignIndex: strconv.FormatInt(foreignIndex, 10), RepoID: issue.RepoID, - Type: ForeignTypeIssue, + Type: foreignreference.TypeIssue, }) assert.NoError(t, err) diff --git a/models/migrate_test.go b/models/migrate_test.go index a6c8c37131730..f4af7ffe373f8 100644 --- a/models/migrate_test.go +++ b/models/migrate_test.go @@ -8,6 +8,7 @@ import ( "strconv" "testing" + "code.gitea.io/gitea/models/foreignreference" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" @@ -60,10 +61,10 @@ func assertCreateIssues(t *testing.T, isPull bool) { IsClosed: true, Labels: []*Label{label}, Reactions: []*Reaction{reaction}, - ForeignReference: &ForeignReference{ + ForeignReference: &foreignreference.ForeignReference{ ForeignIndex: strconv.FormatInt(foreignIndex, 10), RepoID: repo.ID, - Type: ForeignTypeIssue, + Type: foreignreference.TypeIssue, }, } err := InsertIssues(is) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 0b6e047deb250..2faa0a1f2a4dd 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/foreignreference" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" @@ -374,11 +375,11 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { Labels: labels, CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()), UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()), - ForeignReference: &models.ForeignReference{ + ForeignReference: &foreignreference.ForeignReference{ LocalIndex: issue.GetLocalIndex(), ForeignIndex: strconv.FormatInt(issue.GetForeignIndex(), 10), RepoID: g.repo.ID, - Type: models.ForeignTypeIssue, + Type: foreignreference.TypeIssue, }, } From ea0394c6bad569bf9e350057a5c78a5f50c812af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Tue, 15 Feb 2022 10:18:09 +0100 Subject: [PATCH 10/10] foreignreference Type VARCHAR(16) must also be set in migrations --- models/migrations/v211.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 models/migrations/v211.go diff --git a/models/migrations/v211.go b/models/migrations/v211.go new file mode 100644 index 0000000000000..26ccfd20376c1 --- /dev/null +++ b/models/migrations/v211.go @@ -0,0 +1,26 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + + "xorm.io/xorm" +) + +func createForeignReferenceTable(x *xorm.Engine) error { + type ForeignReference struct { + // RepoID is the first column in all indices. now we only need 2 indices: (repo, local) and (repo, foreign, type) + RepoID int64 `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" ` + LocalIndex int64 `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID. + ForeignIndex string `xorm:"INDEX UNIQUE(repo_foreign_type)"` + Type string `xorm:"VARCHAR(16) INDEX UNIQUE(repo_foreign_type)"` + } + + if err := x.Sync2(new(ForeignReference)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + return nil +}