From f9942add5042ad93f9b692f4760fd69d6816d0b8 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 12 Oct 2020 08:14:02 +0200 Subject: [PATCH] Backport Migration Option AuthToken (#13101) fix #13085 Backport #12862 Backport Parts of #12672 --- modules/migrations/gitea.go | 5 ++++- modules/migrations/gitea_test.go | 2 +- modules/migrations/github.go | 34 ++++++++++++++----------------- modules/migrations/github_test.go | 2 +- modules/migrations/gitlab.go | 18 ++++++++-------- modules/migrations/gitlab_test.go | 2 +- modules/migrations/migrate.go | 6 ++++++ modules/structs/repo.go | 1 + 8 files changed, 37 insertions(+), 33 deletions(-) diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index b51dc99037a..3c711feb8b8 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -93,12 +93,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate } var remoteAddr = repo.CloneURL - if len(opts.AuthUsername) > 0 { + if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { u, err := url.Parse(repo.CloneURL) if err != nil { return err } u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) + if len(opts.AuthToken) > 0 { + u.User = url.UserPassword("oauth2", opts.AuthToken) + } remoteAddr = u.String() } diff --git a/modules/migrations/gitea_test.go b/modules/migrations/gitea_test.go index 3f03cbab313..7834dc26eec 100644 --- a/modules/migrations/gitea_test.go +++ b/modules/migrations/gitea_test.go @@ -26,7 +26,7 @@ func TestGiteaUploadRepo(t *testing.T) { user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User) var ( - downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder") + downloader = NewGithubDownloaderV3("", "", "", "go-xorm", "builder") repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) ) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index a6b8fab59de..570c91bfd62 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -60,7 +60,7 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download log.Trace("Create github downloader: %s/%s", oldOwner, oldName) - return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil + return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil } // GitServiceType returns the type of git service @@ -81,7 +81,7 @@ type GithubDownloaderV3 struct { } // NewGithubDownloaderV3 creates a github Downloader via github v3 API -func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *GithubDownloaderV3 { +func NewGithubDownloaderV3(userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { var downloader = GithubDownloaderV3{ userName: userName, password: password, @@ -90,23 +90,19 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith repoName: repoName, } - var client *http.Client - if userName != "" { - if password == "" { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: userName}, - ) - client = oauth2.NewClient(downloader.ctx, ts) - } else { - client = &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - req.SetBasicAuth(userName, password) - return nil, nil - }, - }, - } - } + client := &http.Client{ + Transport: &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, + }, + } + if token != "" { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + client = oauth2.NewClient(downloader.ctx, ts) } downloader.client = github.NewClient(client) return &downloader diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index 708ea6dc91b..bffacabc5d5 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -64,7 +64,7 @@ func assertLabelEqual(t *testing.T, name, color, description string, label *base func TestGitHubDownloadRepo(t *testing.T) { GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in // - downloader := NewGithubDownloaderV3(os.Getenv("GITHUB_READ_TOKEN"), "", "go-gitea", "test_repo") + downloader := NewGithubDownloaderV3("", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo") err := downloader.RefreshRate() assert.NoError(t, err) diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index 09e1a77d36e..02b5aa58877 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -56,10 +56,11 @@ func (f *GitlabDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader baseURL := u.Scheme + "://" + u.Host repoNameSpace := strings.TrimPrefix(u.Path, "/") + repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git") log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace) - return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword), nil + return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken), nil } // GitServiceType returns the type of git service @@ -85,15 +86,12 @@ type GitlabDownloader struct { // NewGitlabDownloader creates a gitlab Downloader via gitlab API // Use either a username/password, personal token entered into the username field, or anonymous/public access // Note: Public access only allows very basic access -func NewGitlabDownloader(baseURL, repoPath, username, password string) *GitlabDownloader { - var gitlabClient *gitlab.Client - var err error - if username != "" { - if password == "" { - gitlabClient, err = gitlab.NewClient(username, gitlab.WithBaseURL(baseURL)) - } else { - gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL)) - } +func NewGitlabDownloader(baseURL, repoPath, username, password, token string) *GitlabDownloader { + gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL)) + // Only use basic auth if token is blank and password is NOT + // Basic auth will fail with empty strings, but empty token will allow anonymous public API usage + if token == "" && password != "" { + gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL)) } if err != nil { diff --git a/modules/migrations/gitlab_test.go b/modules/migrations/gitlab_test.go index a06646e2cc4..2c7b19046fe 100644 --- a/modules/migrations/gitlab_test.go +++ b/modules/migrations/gitlab_test.go @@ -27,7 +27,7 @@ func TestGitlabDownloadRepo(t *testing.T) { t.Skipf("Can't access test repo, skipping %s", t.Name()) } - downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", gitlabPersonalAccessToken, "") + downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken) if downloader == nil { t.Fatal("NewGitlabDownloader is nil") } diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index c970ba69208..2a0e1eb68fa 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -36,6 +36,12 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, theFactory base.DownloaderFactory ) + // determine if user is token + if len(opts.AuthUsername) != 0 && len(opts.AuthPassword) == 0 { + opts.AuthToken = opts.AuthUsername + opts.AuthUsername = "" + } + for _, factory := range factories { if match, err := factory.Match(opts); err != nil { return nil, err diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 70de9b74694..778dc0f5b85 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -213,6 +213,7 @@ type MigrateRepoOption struct { CloneAddr string `json:"clone_addr" binding:"Required"` AuthUsername string `json:"auth_username"` AuthPassword string `json:"auth_password"` + AuthToken string `json:"auth_token"` // required: true UID int `json:"uid" binding:"Required"` // required: true