mirror of https://github.com/go-gitea/gitea
Support migration from AWS CodeCommit (#31981)
This PR adds support for migrating repos from [AWS CodeCommit](https://docs.aws.amazon.com/codecommit/latest/userguide/welcome.html). The access key ID and secret access key are required to get repository information and pull requests. And [HTTPS Git credentials](https://docs.aws.amazon.com/codecommit/latest/userguide/setting-up-gc.html) are required to clone the repository. <img src="https://github.com/user-attachments/assets/82ecb2d0-8d43-42b0-b5af-f5347a13b9d0" width="680" /> The AWS CodeCommit icon is from [AWS Architecture Icons](https://aws.amazon.com/architecture/icons/). <img src="https://github.com/user-attachments/assets/3c44d21f-d753-40f5-9eae-5d3589e0d50d" width="320" />pull/32025/head^2
Zettat123
2 months ago
committed by
GitHub
parent
d9a7748cdc
commit
def1c9670b
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 1.9 KiB |
@ -0,0 +1,269 @@ |
|||||||
|
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package migrations |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
"net/url" |
||||||
|
"strconv" |
||||||
|
"strings" |
||||||
|
|
||||||
|
git_module "code.gitea.io/gitea/modules/git" |
||||||
|
"code.gitea.io/gitea/modules/log" |
||||||
|
base "code.gitea.io/gitea/modules/migration" |
||||||
|
"code.gitea.io/gitea/modules/structs" |
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go-v2/credentials" |
||||||
|
"github.com/aws/aws-sdk-go-v2/service/codecommit" |
||||||
|
"github.com/aws/aws-sdk-go-v2/service/codecommit/types" |
||||||
|
"github.com/aws/aws-sdk-go/aws" |
||||||
|
) |
||||||
|
|
||||||
|
var ( |
||||||
|
_ base.Downloader = &CodeCommitDownloader{} |
||||||
|
_ base.DownloaderFactory = &CodeCommitDownloaderFactory{} |
||||||
|
) |
||||||
|
|
||||||
|
func init() { |
||||||
|
RegisterDownloaderFactory(&CodeCommitDownloaderFactory{}) |
||||||
|
} |
||||||
|
|
||||||
|
// CodeCommitDownloaderFactory defines a codecommit downloader factory
|
||||||
|
type CodeCommitDownloaderFactory struct{} |
||||||
|
|
||||||
|
// New returns a Downloader related to this factory according MigrateOptions
|
||||||
|
func (c *CodeCommitDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { |
||||||
|
u, err := url.Parse(opts.CloneAddr) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
hostElems := strings.Split(u.Host, ".") |
||||||
|
if len(hostElems) != 4 { |
||||||
|
return nil, fmt.Errorf("cannot get the region from clone URL") |
||||||
|
} |
||||||
|
region := hostElems[1] |
||||||
|
|
||||||
|
pathElems := strings.Split(u.Path, "/") |
||||||
|
if len(pathElems) == 0 { |
||||||
|
return nil, fmt.Errorf("cannot get the repo name from clone URL") |
||||||
|
} |
||||||
|
repoName := pathElems[len(pathElems)-1] |
||||||
|
|
||||||
|
baseURL := u.Scheme + "://" + u.Host |
||||||
|
|
||||||
|
return NewCodeCommitDownloader(ctx, repoName, baseURL, opts.AWSAccessKeyID, opts.AWSSecretAccessKey, region), nil |
||||||
|
} |
||||||
|
|
||||||
|
// GitServiceType returns the type of git service
|
||||||
|
func (c *CodeCommitDownloaderFactory) GitServiceType() structs.GitServiceType { |
||||||
|
return structs.CodeCommitService |
||||||
|
} |
||||||
|
|
||||||
|
func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader { |
||||||
|
downloader := CodeCommitDownloader{ |
||||||
|
ctx: ctx, |
||||||
|
repoName: repoName, |
||||||
|
baseURL: baseURL, |
||||||
|
codeCommitClient: codecommit.New(codecommit.Options{ |
||||||
|
Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""), |
||||||
|
Region: region, |
||||||
|
}), |
||||||
|
} |
||||||
|
|
||||||
|
return &downloader |
||||||
|
} |
||||||
|
|
||||||
|
// CodeCommitDownloader implements a downloader for AWS CodeCommit
|
||||||
|
type CodeCommitDownloader struct { |
||||||
|
base.NullDownloader |
||||||
|
ctx context.Context |
||||||
|
codeCommitClient *codecommit.Client |
||||||
|
repoName string |
||||||
|
baseURL string |
||||||
|
allPullRequestIDs []string |
||||||
|
} |
||||||
|
|
||||||
|
// SetContext set context
|
||||||
|
func (c *CodeCommitDownloader) SetContext(ctx context.Context) { |
||||||
|
c.ctx = ctx |
||||||
|
} |
||||||
|
|
||||||
|
// GetRepoInfo returns a repository information
|
||||||
|
func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) { |
||||||
|
output, err := c.codeCommitClient.GetRepository(c.ctx, &codecommit.GetRepositoryInput{ |
||||||
|
RepositoryName: aws.String(c.repoName), |
||||||
|
}) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
repoMeta := output.RepositoryMetadata |
||||||
|
|
||||||
|
repo := &base.Repository{ |
||||||
|
Name: *repoMeta.RepositoryName, |
||||||
|
Owner: *repoMeta.AccountId, |
||||||
|
IsPrivate: true, // CodeCommit repos are always private
|
||||||
|
CloneURL: *repoMeta.CloneUrlHttp, |
||||||
|
} |
||||||
|
if repoMeta.DefaultBranch != nil { |
||||||
|
repo.DefaultBranch = *repoMeta.DefaultBranch |
||||||
|
} |
||||||
|
if repoMeta.RepositoryDescription != nil { |
||||||
|
repo.DefaultBranch = *repoMeta.RepositoryDescription |
||||||
|
} |
||||||
|
return repo, nil |
||||||
|
} |
||||||
|
|
||||||
|
// GetComments returns comments of an issue or PR
|
||||||
|
func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { |
||||||
|
var ( |
||||||
|
nextToken *string |
||||||
|
comments []*base.Comment |
||||||
|
) |
||||||
|
|
||||||
|
for { |
||||||
|
resp, err := c.codeCommitClient.GetCommentsForPullRequest(c.ctx, &codecommit.GetCommentsForPullRequestInput{ |
||||||
|
NextToken: nextToken, |
||||||
|
PullRequestId: aws.String(strconv.FormatInt(commentable.GetForeignIndex(), 10)), |
||||||
|
}) |
||||||
|
if err != nil { |
||||||
|
return nil, false, err |
||||||
|
} |
||||||
|
|
||||||
|
for _, prComment := range resp.CommentsForPullRequestData { |
||||||
|
for _, ccComment := range prComment.Comments { |
||||||
|
comment := &base.Comment{ |
||||||
|
IssueIndex: commentable.GetForeignIndex(), |
||||||
|
PosterName: c.getUsernameFromARN(*ccComment.AuthorArn), |
||||||
|
Content: *ccComment.Content, |
||||||
|
Created: *ccComment.CreationDate, |
||||||
|
Updated: *ccComment.LastModifiedDate, |
||||||
|
} |
||||||
|
comments = append(comments, comment) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
nextToken = resp.NextToken |
||||||
|
if nextToken == nil { |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return comments, true, nil |
||||||
|
} |
||||||
|
|
||||||
|
// GetPullRequests returns pull requests according page and perPage
|
||||||
|
func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { |
||||||
|
allPullRequestIDs, err := c.getAllPullRequestIDs() |
||||||
|
if err != nil { |
||||||
|
return nil, false, err |
||||||
|
} |
||||||
|
|
||||||
|
startIndex := (page - 1) * perPage |
||||||
|
endIndex := page * perPage |
||||||
|
if endIndex > len(allPullRequestIDs) { |
||||||
|
endIndex = len(allPullRequestIDs) |
||||||
|
} |
||||||
|
batch := allPullRequestIDs[startIndex:endIndex] |
||||||
|
|
||||||
|
prs := make([]*base.PullRequest, 0, len(batch)) |
||||||
|
for _, id := range batch { |
||||||
|
output, err := c.codeCommitClient.GetPullRequest(c.ctx, &codecommit.GetPullRequestInput{ |
||||||
|
PullRequestId: aws.String(id), |
||||||
|
}) |
||||||
|
if err != nil { |
||||||
|
return nil, false, err |
||||||
|
} |
||||||
|
orig := output.PullRequest |
||||||
|
number, err := strconv.ParseInt(*orig.PullRequestId, 10, 64) |
||||||
|
if err != nil { |
||||||
|
log.Error("CodeCommit pull request id is not a number: %s", *orig.PullRequestId) |
||||||
|
continue |
||||||
|
} |
||||||
|
if len(orig.PullRequestTargets) == 0 { |
||||||
|
log.Error("CodeCommit pull request does not contain targets", *orig.PullRequestId) |
||||||
|
continue |
||||||
|
} |
||||||
|
target := orig.PullRequestTargets[0] |
||||||
|
pr := &base.PullRequest{ |
||||||
|
Number: number, |
||||||
|
Title: *orig.Title, |
||||||
|
PosterName: c.getUsernameFromARN(*orig.AuthorArn), |
||||||
|
Content: *orig.Description, |
||||||
|
State: "open", |
||||||
|
Created: *orig.CreationDate, |
||||||
|
Updated: *orig.LastActivityDate, |
||||||
|
Merged: target.MergeMetadata.IsMerged, |
||||||
|
Head: base.PullRequestBranch{ |
||||||
|
Ref: strings.TrimPrefix(*target.SourceReference, git_module.BranchPrefix), |
||||||
|
SHA: *target.SourceCommit, |
||||||
|
RepoName: c.repoName, |
||||||
|
}, |
||||||
|
Base: base.PullRequestBranch{ |
||||||
|
Ref: strings.TrimPrefix(*target.DestinationReference, git_module.BranchPrefix), |
||||||
|
SHA: *target.DestinationCommit, |
||||||
|
RepoName: c.repoName, |
||||||
|
}, |
||||||
|
ForeignIndex: number, |
||||||
|
} |
||||||
|
|
||||||
|
if orig.PullRequestStatus == types.PullRequestStatusEnumClosed { |
||||||
|
pr.State = "closed" |
||||||
|
pr.Closed = orig.LastActivityDate |
||||||
|
} |
||||||
|
|
||||||
|
_ = CheckAndEnsureSafePR(pr, c.baseURL, c) |
||||||
|
prs = append(prs, pr) |
||||||
|
} |
||||||
|
|
||||||
|
return prs, len(prs) < perPage, nil |
||||||
|
} |
||||||
|
|
||||||
|
// FormatCloneURL add authentication into remote URLs
|
||||||
|
func (c *CodeCommitDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) { |
||||||
|
u, err := url.Parse(remoteAddr) |
||||||
|
if err != nil { |
||||||
|
return "", err |
||||||
|
} |
||||||
|
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) |
||||||
|
return u.String(), nil |
||||||
|
} |
||||||
|
|
||||||
|
func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) { |
||||||
|
if len(c.allPullRequestIDs) > 0 { |
||||||
|
return c.allPullRequestIDs, nil |
||||||
|
} |
||||||
|
|
||||||
|
var ( |
||||||
|
nextToken *string |
||||||
|
prIDs []string |
||||||
|
) |
||||||
|
|
||||||
|
for { |
||||||
|
output, err := c.codeCommitClient.ListPullRequests(c.ctx, &codecommit.ListPullRequestsInput{ |
||||||
|
RepositoryName: aws.String(c.repoName), |
||||||
|
NextToken: nextToken, |
||||||
|
}) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
prIDs = append(prIDs, output.PullRequestIds...) |
||||||
|
nextToken = output.NextToken |
||||||
|
if nextToken == nil { |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
c.allPullRequestIDs = prIDs |
||||||
|
return c.allPullRequestIDs, nil |
||||||
|
} |
||||||
|
|
||||||
|
func (c *CodeCommitDownloader) getUsernameFromARN(arn string) string { |
||||||
|
parts := strings.Split(arn, "/") |
||||||
|
if len(parts) > 0 { |
||||||
|
return parts[len(parts)-1] |
||||||
|
} |
||||||
|
return "" |
||||||
|
} |
||||||
@ -0,0 +1,117 @@ |
|||||||
|
{{template "base/head" .}} |
||||||
|
<div role="main" aria-label="{{.Title}}" class="page-content repository new migrate"> |
||||||
|
<div class="ui middle very relaxed page grid"> |
||||||
|
<div class="column"> |
||||||
|
<form class="ui form" action="{{.Link}}" method="post"> |
||||||
|
{{template "base/disable_form_autofill"}} |
||||||
|
{{.CsrfTokenHtml}} |
||||||
|
<h3 class="ui top attached header"> |
||||||
|
{{ctx.Locale.Tr "repo.migrate.migrate" .service.Title}} |
||||||
|
<input id="service_type" type="hidden" name="service" value="{{.service}}"> |
||||||
|
</h3> |
||||||
|
<div class="ui attached segment"> |
||||||
|
{{template "base/alert" .}} |
||||||
|
<div class="inline required field {{if .Err_CloneAddr}}error{{end}}"> |
||||||
|
<label for="clone_addr">{{ctx.Locale.Tr "repo.migrate.clone_address"}}</label> |
||||||
|
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required> |
||||||
|
<span class="help"> |
||||||
|
{{ctx.Locale.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{ctx.Locale.Tr "repo.migrate.clone_local_path"}}{{end}} |
||||||
|
</span> |
||||||
|
</div> |
||||||
|
|
||||||
|
<div class="inline required field {{if .Err_Auth}}error{{end}}"> |
||||||
|
<label for="aws_access_key_id">{{ctx.Locale.Tr "repo.migrate.codecommit.aws_access_key_id"}}</label> |
||||||
|
<input id="aws_access_key_id" name="aws_access_key_id" value="{{.aws_access_key_id}}" required> |
||||||
|
</div> |
||||||
|
<div class="inline required field {{if .Err_Auth}}error{{end}}"> |
||||||
|
<label for="aws_secret_access_key">{{ctx.Locale.Tr "repo.migrate.codecommit.aws_secret_access_key"}}</label> |
||||||
|
<input id="aws_secret_access_key" name="aws_secret_access_key" type="password" value="{{.aws_secret_access_key}}" required> |
||||||
|
</div> |
||||||
|
<div class="inline required field {{if .Err_Auth}}error{{end}}"> |
||||||
|
<label for="auth_username">{{ctx.Locale.Tr "repo.migrate.codecommit.https_git_credentials_username"}}</label> |
||||||
|
<input id="auth_username" name="auth_username" value="{{.auth_username}}" required> |
||||||
|
</div> |
||||||
|
<div class="inline required field {{if .Err_Auth}}error{{end}}"> |
||||||
|
<label for="auth_password">{{ctx.Locale.Tr "repo.migrate.codecommit.https_git_credentials_password"}}</label> |
||||||
|
<input id="auth_password" name="auth_password" type="password" value="{{.auth_password}}" required> |
||||||
|
</div> |
||||||
|
|
||||||
|
{{if not .DisableNewPullMirrors}} |
||||||
|
<div class="inline field"> |
||||||
|
<label>{{ctx.Locale.Tr "repo.migrate_options"}}</label> |
||||||
|
<div class="ui checkbox"> |
||||||
|
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}> |
||||||
|
<label>{{ctx.Locale.Tr "repo.migrate_options_mirror_helper"}}</label> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
{{end}} |
||||||
|
|
||||||
|
<div id="migrate_items"> |
||||||
|
<div class="inline field"> |
||||||
|
<label>{{ctx.Locale.Tr "repo.migrate_items"}}</label> |
||||||
|
<div class="ui checkbox"> |
||||||
|
<input name="pull_requests" type="checkbox" {{if .pull_requests}}checked{{end}}> |
||||||
|
<label>{{ctx.Locale.Tr "repo.migrate_items_pullrequests"}}</label> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
|
||||||
|
<div class="divider"></div> |
||||||
|
|
||||||
|
<div class="inline required field {{if .Err_Owner}}error{{end}}"> |
||||||
|
<label>{{ctx.Locale.Tr "repo.owner"}}</label> |
||||||
|
<div class="ui selection owner dropdown"> |
||||||
|
<input type="hidden" id="uid" name="uid" value="{{.ContextUser.ID}}" required> |
||||||
|
<span class="text truncated-item-container" title="{{.ContextUser.Name}}"> |
||||||
|
{{ctx.AvatarUtils.Avatar .ContextUser 28 "mini"}} |
||||||
|
<span class="truncated-item-name">{{.ContextUser.ShortName 40}}</span> |
||||||
|
</span> |
||||||
|
{{svg "octicon-triangle-down" 14 "dropdown icon"}} |
||||||
|
<div class="menu" title="{{.SignedUser.Name}}"> |
||||||
|
<div class="item truncated-item-container" data-value="{{.SignedUser.ID}}"> |
||||||
|
{{ctx.AvatarUtils.Avatar .SignedUser 28 "mini"}} |
||||||
|
<span class="truncated-item-name">{{.SignedUser.ShortName 40}}</span> |
||||||
|
</div> |
||||||
|
{{range .Orgs}} |
||||||
|
<div class="item truncated-item-container" data-value="{{.ID}}" title="{{.Name}}"> |
||||||
|
{{ctx.AvatarUtils.Avatar . 28 "mini"}} |
||||||
|
<span class="truncated-item-name">{{.ShortName 40}}</span> |
||||||
|
</div> |
||||||
|
{{end}} |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
|
||||||
|
<div class="inline required field {{if .Err_RepoName}}error{{end}}"> |
||||||
|
<label for="repo_name">{{ctx.Locale.Tr "repo.repo_name"}}</label> |
||||||
|
<input id="repo_name" name="repo_name" value="{{.repo_name}}" required maxlength="100"> |
||||||
|
</div> |
||||||
|
<div class="inline field"> |
||||||
|
<label>{{ctx.Locale.Tr "repo.visibility"}}</label> |
||||||
|
<div class="ui checkbox"> |
||||||
|
{{if .IsForcedPrivate}} |
||||||
|
<input name="private" type="checkbox" checked disabled> |
||||||
|
<label>{{ctx.Locale.Tr "repo.visibility_helper_forced"}}</label> |
||||||
|
{{else}} |
||||||
|
<input name="private" type="checkbox" {{if .private}}checked{{end}}> |
||||||
|
<label>{{ctx.Locale.Tr "repo.visibility_helper"}}</label> |
||||||
|
{{end}} |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
<div class="inline field {{if .Err_Description}}error{{end}}"> |
||||||
|
<label for="description">{{ctx.Locale.Tr "repo.repo_desc"}}</label> |
||||||
|
<textarea id="description" name="description" maxlength="2048">{{.description}}</textarea> |
||||||
|
</div> |
||||||
|
|
||||||
|
<div class="inline field"> |
||||||
|
<label></label> |
||||||
|
<button class="ui primary button"> |
||||||
|
{{ctx.Locale.Tr "repo.migrate_repo"}} |
||||||
|
</button> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</form> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
{{template "base/footer" .}} |
||||||
|
After Width: | Height: | Size: 2.1 KiB |
Loading…
Reference in new issue