mirror of https://github.com/go-gitea/gitea
Support migration from AWS CodeCommit (#31981)
This PR adds support for migrating repos from [AWS CodeCommit](https://docs.aws.amazon.com/codecommit/latest/userguide/welcome.html). The access key ID and secret access key are required to get repository information and pull requests. And [HTTPS Git credentials](https://docs.aws.amazon.com/codecommit/latest/userguide/setting-up-gc.html) are required to clone the repository. <img src="https://github.com/user-attachments/assets/82ecb2d0-8d43-42b0-b5af-f5347a13b9d0" width="680" /> The AWS CodeCommit icon is from [AWS Architecture Icons](https://aws.amazon.com/architecture/icons/). <img src="https://github.com/user-attachments/assets/3c44d21f-d753-40f5-9eae-5d3589e0d50d" width="320" />pull/32025/head^2
Zettat123
2 months ago
committed by
GitHub
parent
d9a7748cdc
commit
def1c9670b
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 1.9 KiB |
@ -0,0 +1,269 @@ |
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"net/url" |
||||
"strconv" |
||||
"strings" |
||||
|
||||
git_module "code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/log" |
||||
base "code.gitea.io/gitea/modules/migration" |
||||
"code.gitea.io/gitea/modules/structs" |
||||
|
||||
"github.com/aws/aws-sdk-go-v2/credentials" |
||||
"github.com/aws/aws-sdk-go-v2/service/codecommit" |
||||
"github.com/aws/aws-sdk-go-v2/service/codecommit/types" |
||||
"github.com/aws/aws-sdk-go/aws" |
||||
) |
||||
|
||||
var ( |
||||
_ base.Downloader = &CodeCommitDownloader{} |
||||
_ base.DownloaderFactory = &CodeCommitDownloaderFactory{} |
||||
) |
||||
|
||||
func init() { |
||||
RegisterDownloaderFactory(&CodeCommitDownloaderFactory{}) |
||||
} |
||||
|
||||
// CodeCommitDownloaderFactory defines a codecommit downloader factory
|
||||
type CodeCommitDownloaderFactory struct{} |
||||
|
||||
// New returns a Downloader related to this factory according MigrateOptions
|
||||
func (c *CodeCommitDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { |
||||
u, err := url.Parse(opts.CloneAddr) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
hostElems := strings.Split(u.Host, ".") |
||||
if len(hostElems) != 4 { |
||||
return nil, fmt.Errorf("cannot get the region from clone URL") |
||||
} |
||||
region := hostElems[1] |
||||
|
||||
pathElems := strings.Split(u.Path, "/") |
||||
if len(pathElems) == 0 { |
||||
return nil, fmt.Errorf("cannot get the repo name from clone URL") |
||||
} |
||||
repoName := pathElems[len(pathElems)-1] |
||||
|
||||
baseURL := u.Scheme + "://" + u.Host |
||||
|
||||
return NewCodeCommitDownloader(ctx, repoName, baseURL, opts.AWSAccessKeyID, opts.AWSSecretAccessKey, region), nil |
||||
} |
||||
|
||||
// GitServiceType returns the type of git service
|
||||
func (c *CodeCommitDownloaderFactory) GitServiceType() structs.GitServiceType { |
||||
return structs.CodeCommitService |
||||
} |
||||
|
||||
func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader { |
||||
downloader := CodeCommitDownloader{ |
||||
ctx: ctx, |
||||
repoName: repoName, |
||||
baseURL: baseURL, |
||||
codeCommitClient: codecommit.New(codecommit.Options{ |
||||
Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""), |
||||
Region: region, |
||||
}), |
||||
} |
||||
|
||||
return &downloader |
||||
} |
||||
|
||||
// CodeCommitDownloader implements a downloader for AWS CodeCommit
|
||||
type CodeCommitDownloader struct { |
||||
base.NullDownloader |
||||
ctx context.Context |
||||
codeCommitClient *codecommit.Client |
||||
repoName string |
||||
baseURL string |
||||
allPullRequestIDs []string |
||||
} |
||||
|
||||
// SetContext set context
|
||||
func (c *CodeCommitDownloader) SetContext(ctx context.Context) { |
||||
c.ctx = ctx |
||||
} |
||||
|
||||
// GetRepoInfo returns a repository information
|
||||
func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) { |
||||
output, err := c.codeCommitClient.GetRepository(c.ctx, &codecommit.GetRepositoryInput{ |
||||
RepositoryName: aws.String(c.repoName), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
repoMeta := output.RepositoryMetadata |
||||
|
||||
repo := &base.Repository{ |
||||
Name: *repoMeta.RepositoryName, |
||||
Owner: *repoMeta.AccountId, |
||||
IsPrivate: true, // CodeCommit repos are always private
|
||||
CloneURL: *repoMeta.CloneUrlHttp, |
||||
} |
||||
if repoMeta.DefaultBranch != nil { |
||||
repo.DefaultBranch = *repoMeta.DefaultBranch |
||||
} |
||||
if repoMeta.RepositoryDescription != nil { |
||||
repo.DefaultBranch = *repoMeta.RepositoryDescription |
||||
} |
||||
return repo, nil |
||||
} |
||||
|
||||
// GetComments returns comments of an issue or PR
|
||||
func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { |
||||
var ( |
||||
nextToken *string |
||||
comments []*base.Comment |
||||
) |
||||
|
||||
for { |
||||
resp, err := c.codeCommitClient.GetCommentsForPullRequest(c.ctx, &codecommit.GetCommentsForPullRequestInput{ |
||||
NextToken: nextToken, |
||||
PullRequestId: aws.String(strconv.FormatInt(commentable.GetForeignIndex(), 10)), |
||||
}) |
||||
if err != nil { |
||||
return nil, false, err |
||||
} |
||||
|
||||
for _, prComment := range resp.CommentsForPullRequestData { |
||||
for _, ccComment := range prComment.Comments { |
||||
comment := &base.Comment{ |
||||
IssueIndex: commentable.GetForeignIndex(), |
||||
PosterName: c.getUsernameFromARN(*ccComment.AuthorArn), |
||||
Content: *ccComment.Content, |
||||
Created: *ccComment.CreationDate, |
||||
Updated: *ccComment.LastModifiedDate, |
||||
} |
||||
comments = append(comments, comment) |
||||
} |
||||
} |
||||
|
||||
nextToken = resp.NextToken |
||||
if nextToken == nil { |
||||
break |
||||
} |
||||
} |
||||
|
||||
return comments, true, nil |
||||
} |
||||
|
||||
// GetPullRequests returns pull requests according page and perPage
|
||||
func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { |
||||
allPullRequestIDs, err := c.getAllPullRequestIDs() |
||||
if err != nil { |
||||
return nil, false, err |
||||
} |
||||
|
||||
startIndex := (page - 1) * perPage |
||||
endIndex := page * perPage |
||||
if endIndex > len(allPullRequestIDs) { |
||||
endIndex = len(allPullRequestIDs) |
||||
} |
||||
batch := allPullRequestIDs[startIndex:endIndex] |
||||
|
||||
prs := make([]*base.PullRequest, 0, len(batch)) |
||||
for _, id := range batch { |
||||
output, err := c.codeCommitClient.GetPullRequest(c.ctx, &codecommit.GetPullRequestInput{ |
||||
PullRequestId: aws.String(id), |
||||
}) |
||||
if err != nil { |
||||
return nil, false, err |
||||
} |
||||
orig := output.PullRequest |
||||
number, err := strconv.ParseInt(*orig.PullRequestId, 10, 64) |
||||
if err != nil { |
||||
log.Error("CodeCommit pull request id is not a number: %s", *orig.PullRequestId) |
||||
continue |
||||
} |
||||
if len(orig.PullRequestTargets) == 0 { |
||||
log.Error("CodeCommit pull request does not contain targets", *orig.PullRequestId) |
||||
continue |
||||
} |
||||
target := orig.PullRequestTargets[0] |
||||
pr := &base.PullRequest{ |
||||
Number: number, |
||||
Title: *orig.Title, |
||||
PosterName: c.getUsernameFromARN(*orig.AuthorArn), |
||||
Content: *orig.Description, |
||||
State: "open", |
||||
Created: *orig.CreationDate, |
||||
Updated: *orig.LastActivityDate, |
||||
Merged: target.MergeMetadata.IsMerged, |
||||
Head: base.PullRequestBranch{ |
||||
Ref: strings.TrimPrefix(*target.SourceReference, git_module.BranchPrefix), |
||||
SHA: *target.SourceCommit, |
||||
RepoName: c.repoName, |
||||
}, |
||||
Base: base.PullRequestBranch{ |
||||
Ref: strings.TrimPrefix(*target.DestinationReference, git_module.BranchPrefix), |
||||
SHA: *target.DestinationCommit, |
||||
RepoName: c.repoName, |
||||
}, |
||||
ForeignIndex: number, |
||||
} |
||||
|
||||
if orig.PullRequestStatus == types.PullRequestStatusEnumClosed { |
||||
pr.State = "closed" |
||||
pr.Closed = orig.LastActivityDate |
||||
} |
||||
|
||||
_ = CheckAndEnsureSafePR(pr, c.baseURL, c) |
||||
prs = append(prs, pr) |
||||
} |
||||
|
||||
return prs, len(prs) < perPage, nil |
||||
} |
||||
|
||||
// FormatCloneURL add authentication into remote URLs
|
||||
func (c *CodeCommitDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) { |
||||
u, err := url.Parse(remoteAddr) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) |
||||
return u.String(), nil |
||||
} |
||||
|
||||
func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) { |
||||
if len(c.allPullRequestIDs) > 0 { |
||||
return c.allPullRequestIDs, nil |
||||
} |
||||
|
||||
var ( |
||||
nextToken *string |
||||
prIDs []string |
||||
) |
||||
|
||||
for { |
||||
output, err := c.codeCommitClient.ListPullRequests(c.ctx, &codecommit.ListPullRequestsInput{ |
||||
RepositoryName: aws.String(c.repoName), |
||||
NextToken: nextToken, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
prIDs = append(prIDs, output.PullRequestIds...) |
||||
nextToken = output.NextToken |
||||
if nextToken == nil { |
||||
break |
||||
} |
||||
} |
||||
|
||||
c.allPullRequestIDs = prIDs |
||||
return c.allPullRequestIDs, nil |
||||
} |
||||
|
||||
func (c *CodeCommitDownloader) getUsernameFromARN(arn string) string { |
||||
parts := strings.Split(arn, "/") |
||||
if len(parts) > 0 { |
||||
return parts[len(parts)-1] |
||||
} |
||||
return "" |
||||
} |
||||
@ -0,0 +1,117 @@ |
||||
{{template "base/head" .}} |
||||
<div role="main" aria-label="{{.Title}}" class="page-content repository new migrate"> |
||||
<div class="ui middle very relaxed page grid"> |
||||
<div class="column"> |
||||
<form class="ui form" action="{{.Link}}" method="post"> |
||||
{{template "base/disable_form_autofill"}} |
||||
{{.CsrfTokenHtml}} |
||||
<h3 class="ui top attached header"> |
||||
{{ctx.Locale.Tr "repo.migrate.migrate" .service.Title}} |
||||
<input id="service_type" type="hidden" name="service" value="{{.service}}"> |
||||
</h3> |
||||
<div class="ui attached segment"> |
||||
{{template "base/alert" .}} |
||||
<div class="inline required field {{if .Err_CloneAddr}}error{{end}}"> |
||||
<label for="clone_addr">{{ctx.Locale.Tr "repo.migrate.clone_address"}}</label> |
||||
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required> |
||||
<span class="help"> |
||||
{{ctx.Locale.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{ctx.Locale.Tr "repo.migrate.clone_local_path"}}{{end}} |
||||
</span> |
||||
</div> |
||||
|
||||
<div class="inline required field {{if .Err_Auth}}error{{end}}"> |
||||
<label for="aws_access_key_id">{{ctx.Locale.Tr "repo.migrate.codecommit.aws_access_key_id"}}</label> |
||||
<input id="aws_access_key_id" name="aws_access_key_id" value="{{.aws_access_key_id}}" required> |
||||
</div> |
||||
<div class="inline required field {{if .Err_Auth}}error{{end}}"> |
||||
<label for="aws_secret_access_key">{{ctx.Locale.Tr "repo.migrate.codecommit.aws_secret_access_key"}}</label> |
||||
<input id="aws_secret_access_key" name="aws_secret_access_key" type="password" value="{{.aws_secret_access_key}}" required> |
||||
</div> |
||||
<div class="inline required field {{if .Err_Auth}}error{{end}}"> |
||||
<label for="auth_username">{{ctx.Locale.Tr "repo.migrate.codecommit.https_git_credentials_username"}}</label> |
||||
<input id="auth_username" name="auth_username" value="{{.auth_username}}" required> |
||||
</div> |
||||
<div class="inline required field {{if .Err_Auth}}error{{end}}"> |
||||
<label for="auth_password">{{ctx.Locale.Tr "repo.migrate.codecommit.https_git_credentials_password"}}</label> |
||||
<input id="auth_password" name="auth_password" type="password" value="{{.auth_password}}" required> |
||||
</div> |
||||
|
||||
{{if not .DisableNewPullMirrors}} |
||||
<div class="inline field"> |
||||
<label>{{ctx.Locale.Tr "repo.migrate_options"}}</label> |
||||
<div class="ui checkbox"> |
||||
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}> |
||||
<label>{{ctx.Locale.Tr "repo.migrate_options_mirror_helper"}}</label> |
||||
</div> |
||||
</div> |
||||
{{end}} |
||||
|
||||
<div id="migrate_items"> |
||||
<div class="inline field"> |
||||
<label>{{ctx.Locale.Tr "repo.migrate_items"}}</label> |
||||
<div class="ui checkbox"> |
||||
<input name="pull_requests" type="checkbox" {{if .pull_requests}}checked{{end}}> |
||||
<label>{{ctx.Locale.Tr "repo.migrate_items_pullrequests"}}</label> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
|
||||
<div class="divider"></div> |
||||
|
||||
<div class="inline required field {{if .Err_Owner}}error{{end}}"> |
||||
<label>{{ctx.Locale.Tr "repo.owner"}}</label> |
||||
<div class="ui selection owner dropdown"> |
||||
<input type="hidden" id="uid" name="uid" value="{{.ContextUser.ID}}" required> |
||||
<span class="text truncated-item-container" title="{{.ContextUser.Name}}"> |
||||
{{ctx.AvatarUtils.Avatar .ContextUser 28 "mini"}} |
||||
<span class="truncated-item-name">{{.ContextUser.ShortName 40}}</span> |
||||
</span> |
||||
{{svg "octicon-triangle-down" 14 "dropdown icon"}} |
||||
<div class="menu" title="{{.SignedUser.Name}}"> |
||||
<div class="item truncated-item-container" data-value="{{.SignedUser.ID}}"> |
||||
{{ctx.AvatarUtils.Avatar .SignedUser 28 "mini"}} |
||||
<span class="truncated-item-name">{{.SignedUser.ShortName 40}}</span> |
||||
</div> |
||||
{{range .Orgs}} |
||||
<div class="item truncated-item-container" data-value="{{.ID}}" title="{{.Name}}"> |
||||
{{ctx.AvatarUtils.Avatar . 28 "mini"}} |
||||
<span class="truncated-item-name">{{.ShortName 40}}</span> |
||||
</div> |
||||
{{end}} |
||||
</div> |
||||
</div> |
||||
</div> |
||||
|
||||
<div class="inline required field {{if .Err_RepoName}}error{{end}}"> |
||||
<label for="repo_name">{{ctx.Locale.Tr "repo.repo_name"}}</label> |
||||
<input id="repo_name" name="repo_name" value="{{.repo_name}}" required maxlength="100"> |
||||
</div> |
||||
<div class="inline field"> |
||||
<label>{{ctx.Locale.Tr "repo.visibility"}}</label> |
||||
<div class="ui checkbox"> |
||||
{{if .IsForcedPrivate}} |
||||
<input name="private" type="checkbox" checked disabled> |
||||
<label>{{ctx.Locale.Tr "repo.visibility_helper_forced"}}</label> |
||||
{{else}} |
||||
<input name="private" type="checkbox" {{if .private}}checked{{end}}> |
||||
<label>{{ctx.Locale.Tr "repo.visibility_helper"}}</label> |
||||
{{end}} |
||||
</div> |
||||
</div> |
||||
<div class="inline field {{if .Err_Description}}error{{end}}"> |
||||
<label for="description">{{ctx.Locale.Tr "repo.repo_desc"}}</label> |
||||
<textarea id="description" name="description" maxlength="2048">{{.description}}</textarea> |
||||
</div> |
||||
|
||||
<div class="inline field"> |
||||
<label></label> |
||||
<button class="ui primary button"> |
||||
{{ctx.Locale.Tr "repo.migrate_repo"}} |
||||
</button> |
||||
</div> |
||||
</div> |
||||
</form> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
{{template "base/footer" .}} |
||||
|
After Width: | Height: | Size: 2.1 KiB |
Loading…
Reference in new issue