From e048b8835b6238b9f8b20cc2e6b46e0900ef7ee5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 23 Sep 2021 23:32:30 +0800 Subject: [PATCH 1/4] multiple tokens support for migrating from github --- modules/migrations/github.go | 149 +++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 57 deletions(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 97e1b672accd..97515d2a889b 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -68,14 +68,15 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { // from github via APIv3 type GithubDownloaderV3 struct { base.NullDownloader - ctx context.Context - client *github.Client - repoOwner string - repoName string - userName string - password string - rate *github.Rate - maxPerPage int + ctx context.Context + clients []*github.Client + repoOwner string + repoName string + userName string + password string + rates []*github.Rate + curClientIdx int + maxPerPage int } // NewGithubDownloaderV3 creates a github Downloader via github v3 API @@ -97,15 +98,31 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok }, }, } + if token != "" { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - client = oauth2.NewClient(downloader.ctx, ts) - } - downloader.client = github.NewClient(client) - if baseURL != "https://github.com" { - downloader.client, _ = github.NewEnterpriseClient(baseURL, baseURL, client) + tokens := strings.Split(token, ",") + for _, token := range tokens { + token = strings.TrimSpace(token) + + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + client = oauth2.NewClient(downloader.ctx, ts) + + githubClient := github.NewClient(client) + if baseURL != "https://github.com" { + githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client) + } + downloader.clients = append(downloader.clients, githubClient) + downloader.rates = append(downloader.rates, nil) + } + } else { + githubClient := github.NewClient(client) + if baseURL != "https://github.com" { + githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client) + } + downloader.clients = append(downloader.clients, githubClient) + downloader.rates = append(downloader.rates, nil) } return &downloader } @@ -116,8 +133,18 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) { } func (g *GithubDownloaderV3) sleep() { - for g.rate != nil && g.rate.Remaining <= GithubLimitRateRemaining { - timer := time.NewTimer(time.Until(g.rate.Reset.Time)) + var recentIdx int + var maxRemaining int + for i := 0; i < len(g.clients); i++ { + if g.rates[i] != nil && g.rates[i].Remaining > maxRemaining { + maxRemaining = g.rates[i].Remaining + recentIdx = i + } + } + g.curClientIdx = recentIdx + + for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining { + timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time)) select { case <-g.ctx.Done(): util.StopTimer(timer) @@ -127,35 +154,43 @@ func (g *GithubDownloaderV3) sleep() { err := g.RefreshRate() if err != nil { - log.Error("g.client.RateLimits: %s", err) + log.Error("g.getClient().RateLimits: %s", err) } } } // RefreshRate update the current rate (doesn't count in rate limit) func (g *GithubDownloaderV3) RefreshRate() error { - rates, _, err := g.client.RateLimits(g.ctx) + rates, _, err := g.getClient().RateLimits(g.ctx) if err != nil { // if rate limit is not enabled, ignore it if strings.Contains(err.Error(), "404") { - g.rate = nil + g.setRate(nil) return nil } return err } - g.rate = rates.GetCore() + g.setRate(rates.GetCore()) return nil } +func (g *GithubDownloaderV3) getClient() *github.Client { + return g.clients[g.curClientIdx] +} + +func (g *GithubDownloaderV3) setRate(rate *github.Rate) { + g.rates[g.curClientIdx] = rate +} + // GetRepoInfo returns a repository information func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { g.sleep() - gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) + gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) // convert github repo to stand Repo return &base.Repository{ @@ -172,11 +207,11 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { // GetTopics return github topics func (g *GithubDownloaderV3) GetTopics() ([]string, error) { g.sleep() - r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) + r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) return r.Topics, nil } @@ -186,7 +221,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { var milestones = make([]*base.Milestone, 0, perPage) for i := 1; ; i++ { g.sleep() - ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, + ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, &github.MilestoneListOptions{ State: "all", ListOptions: github.ListOptions{ @@ -196,7 +231,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { if err != nil { return nil, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) for _, m := range ms { var state = "open" @@ -234,7 +269,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { var labels = make([]*base.Label, 0, perPage) for i := 1; ; i++ { g.sleep() - ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, + ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, @@ -242,7 +277,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { if err != nil { return nil, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) for _, label := range ls { labels = append(labels, convertGithubLabel(label)) @@ -291,12 +326,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) Updated: asset.UpdatedAt.Time, DownloadFunc: func() (io.ReadCloser, error) { g.sleep() - asset, redirectURL, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil) + asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil) if err != nil { return nil, err } if err := g.RefreshRate(); err != nil { - log.Error("g.client.RateLimits: %s", err) + log.Error("g.getClient().RateLimits: %s", err) } if asset == nil { if redirectURL != "" { @@ -308,7 +343,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) resp, err := httpClient.Do(req) err1 := g.RefreshRate() if err1 != nil { - log.Error("g.client.RateLimits: %s", err1) + log.Error("g.getClient().RateLimits: %s", err1) } if err != nil { return nil, err @@ -330,7 +365,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { var releases = make([]*base.Release, 0, perPage) for i := 1; ; i++ { g.sleep() - ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, + ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, @@ -338,7 +373,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { if err != nil { return nil, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) for _, release := range ls { releases = append(releases, g.convertGithubRelease(release)) @@ -367,12 +402,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, var allIssues = make([]*base.Issue, 0, perPage) g.sleep() - issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) + issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) - g.rate = &resp.Rate + g.setRate(&resp.Rate) for _, issue := range issues { if issue.IsPullRequest() { continue @@ -387,14 +422,14 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, var reactions []*base.Reaction for i := 1; ; i++ { g.sleep() - res, resp, err := g.client.Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ + res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ Page: i, PerPage: perPage, }) if err != nil { return nil, false, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) if len(res) == 0 { break } @@ -465,24 +500,24 @@ func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*bas } for { g.sleep() - comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt) + comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } - g.rate = &resp.Rate + g.setRate(&resp.Rate) for _, comment := range comments { // get reactions var reactions []*base.Reaction for i := 1; ; i++ { g.sleep() - res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ + res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ Page: i, PerPage: g.maxPerPage, }) if err != nil { return nil, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) if len(res) == 0 { break } @@ -534,27 +569,27 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, } g.sleep() - comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt) + comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } var isEnd = resp.NextPage == 0 log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage) - g.rate = &resp.Rate + g.setRate(&resp.Rate) for _, comment := range comments { // get reactions var reactions []*base.Reaction for i := 1; ; i++ { g.sleep() - res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ + res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ Page: i, PerPage: g.maxPerPage, }) if err != nil { return nil, false, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) if len(res) == 0 { break } @@ -599,12 +634,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq } var allPRs = make([]*base.PullRequest, 0, perPage) g.sleep() - prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) + prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) - g.rate = &resp.Rate + g.setRate(&resp.Rate) for _, pr := range prs { var labels = make([]*base.Label, 0, len(pr.Labels)) for _, l := range pr.Labels { @@ -615,14 +650,14 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq var reactions []*base.Reaction for i := 1; ; i++ { g.sleep() - res, resp, err := g.client.Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{ + res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{ Page: i, PerPage: perPage, }) if err != nil { return nil, false, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) if len(res) == 0 { break } @@ -693,14 +728,14 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques var reactions []*base.Reaction for i := 1; ; i++ { g.sleep() - res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{ + res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{ Page: i, PerPage: g.maxPerPage, }) if err != nil { return nil, err } - g.rate = &resp.Rate + g.setRate(&resp.Rate) if len(res) == 0 { break } @@ -738,11 +773,11 @@ func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Revi } for { g.sleep() - reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt) + reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } - g.rate = &resp.Rate + g.setRate(&resp.Rate) for _, review := range reviews { r := convertGithubReview(review) r.IssueIndex = context.LocalID() @@ -752,11 +787,11 @@ func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Revi } for { g.sleep() - reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2) + reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } - g.rate = &resp.Rate + g.setRate(&resp.Rate) cs, err := g.convertGithubReviewComments(reviewComments) if err != nil { From 526852fc2deeced722054512250e8bea54ff69e0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 29 Sep 2021 20:51:47 +0800 Subject: [PATCH 2/4] improve code and token description --- modules/migrations/github.go | 23 +++++++++++------------ options/locale/locale_en-US.ini | 1 + templates/repo/migrate/github.tmpl | 5 ++++- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 97515d2a889b..aeaf9fb5ab2b 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -109,24 +109,23 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok ) client = oauth2.NewClient(downloader.ctx, ts) - githubClient := github.NewClient(client) - if baseURL != "https://github.com" { - githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client) - } - downloader.clients = append(downloader.clients, githubClient) - downloader.rates = append(downloader.rates, nil) + downloader.addClient(client, baseURL) } } else { - githubClient := github.NewClient(client) - if baseURL != "https://github.com" { - githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client) - } - downloader.clients = append(downloader.clients, githubClient) - downloader.rates = append(downloader.rates, nil) + downloader.addClient(client, baseURL) } return &downloader } +func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) { + githubClient := github.NewClient(client) + if baseURL != "https://github.com" { + githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client) + } + g.clients = append(g.clients, githubClient) + g.rates = append(g.rates, nil) +} + // SetContext set context func (g *GithubDownloaderV3) SetContext(ctx context.Context) { g.ctx = ctx diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1324303fc064..472d443263dd 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -887,6 +887,7 @@ migrate_items_releases = Releases migrate_repo = Migrate Repository migrate.clone_address = Migrate / Clone From URL migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repository +migrate.github_token_desc = You can put one or more tokens with comma separated here to make migrating faster because of Github API rate limit migrate.clone_local_path = or a local server path migrate.permission_denied = You are not allowed to import local repositories. migrate.permission_denied_blocked = You are not allowed to import from blocked hosts. diff --git a/templates/repo/migrate/github.tmpl b/templates/repo/migrate/github.tmpl index 156f8896fce0..9bd7228a43b4 100644 --- a/templates/repo/migrate/github.tmpl +++ b/templates/repo/migrate/github.tmpl @@ -14,7 +14,7 @@ - {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} + {{.i18n.Tr "repo.migrate.clone_address_desc"}} @@ -22,6 +22,9 @@ {{svg "octicon-question"}} + + {{.i18n.Tr "repo.migrate.github_token_desc"}} + {{template "repo/migrate/options" .}} From eb98d2a27f6a47e8fb1079b3887c750cee3955c2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 15 Oct 2021 09:46:55 +0800 Subject: [PATCH 3/4] Fix bug --- modules/migrations/github.go | 72 +++++++++++++++++++-------------- options/locale/locale_en-US.ini | 2 +- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index aeaf9fb5ab2b..a7498c28c1a1 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -90,28 +90,37 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok maxPerPage: 100, } - client := &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - req.SetBasicAuth(userName, password) - return proxy.Proxy()(req) - }, - }, - } - if token != "" { tokens := strings.Split(token, ",") for _, token := range tokens { token = strings.TrimSpace(token) - ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ) - client = oauth2.NewClient(downloader.ctx, ts) + var client = &http.Client{ + Transport: &oauth2.Transport{ + Base: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, + Proxy: func(req *http.Request) (*url.URL, error) { + return proxy.Proxy()(req) + }, + }, + Source: oauth2.ReuseTokenSource(nil, ts), + }, + } downloader.addClient(client, baseURL) } } else { + var client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return proxy.Proxy()(req) + }, + }, + } downloader.addClient(client, baseURL) } return &downloader @@ -131,7 +140,7 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) { g.ctx = ctx } -func (g *GithubDownloaderV3) sleep() { +func (g *GithubDownloaderV3) waitAndPickClient() { var recentIdx int var maxRemaining int for i := 0; i < len(g.clients); i++ { @@ -184,7 +193,7 @@ func (g *GithubDownloaderV3) setRate(rate *github.Rate) { // GetRepoInfo returns a repository information func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { - g.sleep() + g.waitAndPickClient() gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) if err != nil { return nil, err @@ -205,7 +214,7 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { // GetTopics return github topics func (g *GithubDownloaderV3) GetTopics() ([]string, error) { - g.sleep() + g.waitAndPickClient() r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) if err != nil { return nil, err @@ -219,7 +228,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { var perPage = g.maxPerPage var milestones = make([]*base.Milestone, 0, perPage) for i := 1; ; i++ { - g.sleep() + g.waitAndPickClient() ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, &github.MilestoneListOptions{ State: "all", @@ -267,7 +276,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { var perPage = g.maxPerPage var labels = make([]*base.Label, 0, perPage) for i := 1; ; i++ { - g.sleep() + g.waitAndPickClient() ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, @@ -324,7 +333,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) Created: asset.CreatedAt.Time, Updated: asset.UpdatedAt.Time, DownloadFunc: func() (io.ReadCloser, error) { - g.sleep() + g.waitAndPickClient() asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil) if err != nil { return nil, err @@ -334,7 +343,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) } if asset == nil { if redirectURL != "" { - g.sleep() + g.waitAndPickClient() req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil) if err != nil { return nil, err @@ -363,7 +372,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { var perPage = g.maxPerPage var releases = make([]*base.Release, 0, perPage) for i := 1; ; i++ { - g.sleep() + g.waitAndPickClient() ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, @@ -400,7 +409,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, } var allIssues = make([]*base.Issue, 0, perPage) - g.sleep() + g.waitAndPickClient() issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) @@ -420,7 +429,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, // get reactions var reactions []*base.Reaction for i := 1; ; i++ { - g.sleep() + g.waitAndPickClient() res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ Page: i, PerPage: perPage, @@ -498,7 +507,7 @@ func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*bas }, } for { - g.sleep() + g.waitAndPickClient() comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) @@ -508,7 +517,7 @@ func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*bas // get reactions var reactions []*base.Reaction for i := 1; ; i++ { - g.sleep() + g.waitAndPickClient() res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ Page: i, PerPage: g.maxPerPage, @@ -567,7 +576,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, }, } - g.sleep() + g.waitAndPickClient() comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) @@ -580,7 +589,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, // get reactions var reactions []*base.Reaction for i := 1; ; i++ { - g.sleep() + g.waitAndPickClient() res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ Page: i, PerPage: g.maxPerPage, @@ -632,7 +641,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, } var allPRs = make([]*base.PullRequest, 0, perPage) - g.sleep() + g.waitAndPickClient() prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) @@ -648,7 +657,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq // get reactions var reactions []*base.Reaction for i := 1; ; i++ { - g.sleep() + g.waitAndPickClient() res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{ Page: i, PerPage: perPage, @@ -669,6 +678,9 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq } } + // download patch and saved as tmp file + g.waitAndPickClient() + allPRs = append(allPRs, &base.PullRequest{ Title: pr.GetTitle(), Number: int64(pr.GetNumber()), @@ -726,7 +738,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques // get reactions var reactions []*base.Reaction for i := 1; ; i++ { - g.sleep() + g.waitAndPickClient() res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{ Page: i, PerPage: g.maxPerPage, @@ -771,7 +783,7 @@ func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Revi PerPage: g.maxPerPage, } for { - g.sleep() + g.waitAndPickClient() reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) @@ -785,7 +797,7 @@ func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Revi PerPage: g.maxPerPage, } for { - g.sleep() + g.waitAndPickClient() reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 472d443263dd..67ca8ff558b3 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -887,7 +887,7 @@ migrate_items_releases = Releases migrate_repo = Migrate Repository migrate.clone_address = Migrate / Clone From URL migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repository -migrate.github_token_desc = You can put one or more tokens with comma separated here to make migrating faster because of Github API rate limit +migrate.github_token_desc = You can put one or more tokens with comma separated here to make migrating faster because of Github API rate limit. WARN: Abusing this feature may violate the service provider's policy and lead to account blocking. migrate.clone_local_path = or a local server path migrate.permission_denied = You are not allowed to import local repositories. migrate.permission_denied_blocked = You are not allowed to import from blocked hosts. From 01b50f17f798ad7be3899322f6ede79d2b893b83 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 15 Oct 2021 12:34:01 +0800 Subject: [PATCH 4/4] Add comment for get client --- modules/migrations/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index a7498c28c1a1..1a228c84a49d 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -149,7 +149,7 @@ func (g *GithubDownloaderV3) waitAndPickClient() { recentIdx = i } } - g.curClientIdx = recentIdx + g.curClientIdx = recentIdx // if no max remain, it will always pick the first client. for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining { timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))