mirror
/
gitea
mirror of https://github.com/go-gitea/gitea
2
0
Fork 0
Code Issues Releases Wiki Activity

Fix bug isEnd detection on getIssues/getPullRequests (#13299) (#13301)

Browse Source
pull/13303/head
Lunny Xiao 4 years ago committed by GitHub
parent e177728a82
commit 151daf73a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 50 additions and 32 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 52
      modules/migrations/github.go
  2. 30
      modules/migrations/gitlab.go

52
modules/migrations/github.go
Unescape View File

@ -65,23 +65,25 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// GithubDownloaderV3 implements a Downloader interface to get repository informations
// from github via APIv3
type GithubDownloaderV3 struct {
ctx context.Context
client *github.Client
repoOwner string
repoName string
userName string
password string
rate *github.Rate
ctx context.Context
client *github.Client
repoOwner string
repoName string
userName string
password string
rate *github.Rate
maxPerPage int
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
var downloader = GithubDownloaderV3{
userName: userName,
password: password,
ctx: ctx,
repoOwner: repoOwner,
repoName: repoName,
userName: userName,
password: password,
ctx: ctx,
repoOwner: repoOwner,
repoName: repoName,
maxPerPage: 100,
}
client := &http.Client{
@ -177,7 +179,7 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
// GetMilestones returns milestones
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100
var perPage = g.maxPerPage
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
g.sleep()
@ -233,7 +235,7 @@ func convertGithubLabel(label *github.Label) *base.Label {
// GetLabels returns labels
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = 100
var perPage = g.maxPerPage
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
g.sleep()
@ -304,7 +306,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
// GetReleases returns releases
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
var perPage = 100
var perPage = g.maxPerPage
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
g.sleep()
@ -342,6 +344,9 @@ func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, err
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &github.IssueListByRepoOptions{
Sort: "created",
Direction: "asc",
@ -429,7 +434,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
// GetComments returns comments according issueNumber
func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) {
var (
allComments = make([]*base.Comment, 0, 100)
allComments = make([]*base.Comment, 0, g.maxPerPage)
created = "created"
asc = "asc"
)
@ -437,7 +442,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
Sort: &created,
Direction: &asc,
ListOptions: github.ListOptions{
PerPage: 100,
PerPage: g.maxPerPage,
},
}
for {
@ -459,7 +464,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
g.sleep()
res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i,
PerPage: 100,
PerPage: g.maxPerPage,
})
if err != nil {
return nil, err
@ -497,6 +502,9 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
// GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &github.PullRequestListOptions{
Sort: "created",
Direction: "asc",
@ -650,7 +658,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
g.sleep()
res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
Page: i,
PerPage: 100,
PerPage: g.maxPerPage,
})
if err != nil {
return nil, err
@ -687,9 +695,9 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
// GetReviews returns pull requests review
func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
var allReviews = make([]*base.Review, 0, 100)
var allReviews = make([]*base.Review, 0, g.maxPerPage)
opt := &github.ListOptions{
PerPage: 100,
PerPage: g.maxPerPage,
}
for {
g.sleep()
@ -703,7 +711,7 @@ func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review
r.IssueIndex = pullRequestNumber
// retrieve all review comments
opt2 := &github.ListOptions{
PerPage: 100,
PerPage: g.maxPerPage,
}
for {
g.sleep()

30
modules/migrations/gitlab.go
Unescape View File

@ -68,6 +68,7 @@ type GitlabDownloader struct {
repoName string
issueCount int64
fetchPRcomments bool
maxPerPage int
}
// NewGitlabDownloader creates a gitlab Downloader via gitlab API
@ -99,10 +100,11 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw
}
return &GitlabDownloader{
ctx: ctx,
client: gitlabClient,
repoID: gr.ID,
repoName: gr.Name,
ctx: ctx,
client: gitlabClient,
repoID: gr.ID,
repoName: gr.Name,
maxPerPage: 100,
}, nil
}
@ -159,7 +161,7 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) {
// GetMilestones returns milestones
func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100
var perPage = g.maxPerPage
var state = "all"
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
@ -230,7 +232,7 @@ func (g *GitlabDownloader) normalizeColor(val string) string {
// GetLabels returns labels
func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
var perPage = 100
var perPage = g.maxPerPage
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
@ -281,7 +283,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
// GetReleases returns releases
func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
var perPage = 100
var perPage = g.maxPerPage
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
@ -330,6 +332,10 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
state := "all"
sort := "asc"
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &gitlab.ListProjectIssuesOptions{
State: &state,
Sort: &sort,
@ -401,7 +407,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
// GetComments returns comments according issueNumber
// TODO: figure out how to transfer comment reactions
func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
var allComments = make([]*base.Comment, 0, 100)
var allComments = make([]*base.Comment, 0, g.maxPerPage)
var page = 1
var realIssueNumber int64
@ -415,14 +421,14 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
realIssueNumber = issueNumber
comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{
Page: page,
PerPage: 100,
PerPage: g.maxPerPage,
}, nil, gitlab.WithContext(g.ctx))
} else {
// If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below
realIssueNumber = issueNumber - g.issueCount
comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{
Page: page,
PerPage: 100,
PerPage: g.maxPerPage,
}, nil, gitlab.WithContext(g.ctx))
}
@ -465,6 +471,10 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
// GetPullRequests returns pull requests according page and perPage
func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &gitlab.ListProjectMergeRequestsOptions{
ListOptions: gitlab.ListOptions{
PerPage: perPage,

Write Preview
Loading…
Cancel
Save