From f2a3abc683ad4b2177b7c7c6160a2c0b4316120a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 13 Oct 2019 21:23:14 +0800 Subject: [PATCH] Move migrating repository from frontend to backend (#6200) * move migrating to backend * add loading image when migrating and fix tests * fix format * fix lint * add redis task queue support and improve docs * add redis vendor * fix vet * add database migrations and fix app.ini sample * add comments for task section on app.ini.sample * Update models/migrations/v84.go Co-Authored-By: lunny * Update models/repo.go Co-Authored-By: lunny * move migrating to backend * add loading image when migrating and fix tests * fix fmt * add redis task queue support and improve docs * fix fixtures * fix fixtures * fix duplicate function on index.js * fix tests * rename repository statuses * check if repository is being create when SSH request * fix lint * fix template * some improvements * fix template * unified migrate options * fix lint * fix loading page * refactor * When gitea restart, don't restart the running tasks because we may have servel gitea instances, that may break the migration * fix js * Update models/repo.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * fix tests * rename ErrTaskIsNotExist to ErrTaskDoesNotExist * delete release after add one on tests to make it run happy * fix tests * fix tests * improve codes * fix lint * fix lint * fix migrations --- .gitignore | 1 + custom/conf/app.ini.sample | 9 + .../doc/advanced/config-cheat-sheet.en-us.md | 7 + .../doc/advanced/config-cheat-sheet.zh-cn.md | 7 + models/fixtures/repository.yml | 43 +++- models/migrations/migrations.go | 2 + models/migrations/v99.go | 34 +++ models/models.go | 1 + models/repo.go | 83 +++--- models/task.go | 240 ++++++++++++++++++ modules/context/repo.go | 35 +-- modules/migrations/base/options.go | 21 +- modules/migrations/gitea.go | 36 ++- modules/migrations/gitea_test.go | 7 +- modules/migrations/github.go | 4 +- modules/migrations/migrate.go | 12 +- modules/setting/setting.go | 1 + modules/setting/task.go | 25 ++ modules/structs/repo.go | 16 +- modules/structs/task.go | 34 +++ modules/task/migrate.go | 120 +++++++++ modules/task/queue.go | 14 + modules/task/queue_channel.go | 48 ++++ modules/task/queue_redis.go | 130 ++++++++++ modules/task/task.go | 66 +++++ options/locale/locale_en-US.ini | 2 + public/img/loading.png | Bin 0 -> 18713 bytes public/js/index.js | 36 +++ routers/api/v1/repo/repo.go | 4 +- routers/init.go | 4 + routers/private/serv.go | 9 + routers/repo/repo.go | 103 +++++--- routers/repo/view.go | 30 +++ routers/routes/routes.go | 2 + services/mirror/mirror_test.go | 29 ++- templates/repo/header.tmpl | 144 +++++------ templates/repo/migrating.tmpl | 31 +++ 37 files changed, 1180 insertions(+), 210 deletions(-) create mode 100644 models/migrations/v99.go create mode 100644 models/task.go create mode 100644 modules/setting/task.go create mode 100644 modules/structs/task.go create mode 100644 modules/task/migrate.go create mode 100644 modules/task/queue.go create mode 100644 modules/task/queue_channel.go create mode 100644 modules/task/queue_redis.go create mode 100644 modules/task/task.go create mode 100644 public/img/loading.png create mode 100644 templates/repo/migrating.tmpl diff --git a/.gitignore b/.gitignore index fa6cbb454b5..773b4726c0a 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,4 @@ prime/ *.snap *.snap-build *_source.tar.bz2 +.DS_Store \ No newline at end of file diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 9bfddc97e8f..dd14089d2b0 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -808,3 +808,12 @@ IS_INPUT_FILE = false ENABLED = false ; If you want to add authorization, specify a token here TOKEN = + +[task] +; Task queue type, could be `channel` or `redis`. +QUEUE_TYPE = channel +; Task queue length, available only when `QUEUE_TYPE` is `channel`. +QUEUE_LENGTH = 1000 +; Task queue connction string, available only when `QUEUE_TYPE` is `redis`. +; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`. +QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" \ No newline at end of file diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 198cff6f049..ed34be032bb 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -514,9 +514,16 @@ Two special environment variables are passed to the render command: - `GITEA_PREFIX_RAW`, which contains the current URL prefix in the `raw` path tree. To be used as prefix for image paths. ## Time (`time`) + - `FORMAT`: Time format to diplay on UI. i.e. RFC1123 or 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: Default location of time on the UI, so that we can display correct user's time on UI. i.e. Shanghai/Asia +## Task (`task`) + +- `QUEUE_TYPE`: **channel**: Task queue type, could be `channel` or `redis`. +- `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`. +- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`. + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 541d66f4e9b..01ba821a47a 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -241,9 +241,16 @@ IS_INPUT_FILE = false - IS_INPUT_FILE: 输入方式是最后一个参数为文件路径还是从标准输入读取。 ## Time (`time`) + - `FORMAT`: 显示在界面上的时间格式。比如： RFC1123 或者 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: 默认显示在界面上的时区，默认为本地时区。比如： Asia/Shanghai +## Task (`task`) + +- `QUEUE_TYPE`: **channel**: 任务队列类型，可以为 `channel` 或 `redis`。 +- `QUEUE_LENGTH`: **1000**: 任务队列长度，当 `QUEUE_TYPE` 为 `channel` 时有效。 +- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串，当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码，则可以 `addrs=127.0.0.1:6379 password=123 db=0`。 + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 diff --git a/models/fixtures/repository.yml b/models/fixtures/repository.yml index 2e38c5e1dd6..cf7d24c6cdb 100644 --- a/models/fixtures/repository.yml +++ b/models/fixtures/repository.yml @@ -11,6 +11,7 @@ num_milestones: 3 num_closed_milestones: 1 num_watches: 3 + status: 0 - id: 2 @@ -24,6 +25,7 @@ num_closed_pulls: 0 num_stars: 1 close_issues_via_commit_in_any_branch: true + status: 0 - id: 3 @@ -36,6 +38,7 @@ num_pulls: 0 num_closed_pulls: 0 num_watches: 0 + status: 0 - id: 4 @@ -48,6 +51,7 @@ num_pulls: 0 num_closed_pulls: 0 num_stars: 1 + status: 0 - id: 5 @@ -61,6 +65,7 @@ num_closed_pulls: 0 num_watches: 0 is_mirror: true + status: 0 - id: 6 @@ -73,6 +78,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 7 @@ -85,6 +91,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 8 @@ -97,6 +104,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 9 @@ -109,6 +117,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 10 @@ -122,6 +131,7 @@ num_closed_pulls: 0 is_mirror: false num_forks: 1 + status: 0 - id: 11 @@ -135,6 +145,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 12 @@ -147,6 +158,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 13 @@ -159,6 +171,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 14 @@ -172,6 +185,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 15 @@ -179,6 +193,7 @@ lower_name: repo15 name: repo15 is_empty: true + status: 0 - id: 16 @@ -191,6 +206,7 @@ num_pulls: 0 num_closed_pulls: 0 num_watches: 0 + status: 0 - id: 17 @@ -205,6 +221,7 @@ num_watches: 0 is_mirror: false is_fork: false + status: 0 - id: 18 @@ -218,6 +235,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 19 @@ -231,6 +249,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 20 @@ -244,6 +263,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 21 @@ -257,6 +277,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 22 @@ -270,6 +291,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 23 @@ -283,6 +305,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 24 @@ -296,6 +319,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 25 @@ -310,6 +334,7 @@ num_watches: 0 is_mirror: true is_fork: false + status: 0 - id: 26 @@ -324,6 +349,7 @@ num_watches: 0 is_mirror: true is_fork: false + status: 0 - id: 27 @@ -339,6 +365,7 @@ is_mirror: true num_forks: 1 is_fork: false + status: 0 - id: 28 @@ -354,6 +381,7 @@ is_mirror: true num_forks: 1 is_fork: false + status: 0 - id: 29 @@ -368,6 +396,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: true + status: 0 - id: 30 @@ -382,6 +411,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: true + status: 0 - id: 31 @@ -392,6 +422,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 32 # org public repo @@ -403,6 +434,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 33 @@ -410,6 +442,7 @@ lower_name: utf8 name: utf8 is_private: false + status: 0 - id: 34 @@ -421,6 +454,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 35 @@ -432,6 +466,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 36 @@ -443,6 +478,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 37 @@ -454,6 +490,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 38 @@ -465,6 +502,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 39 @@ -476,6 +514,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 40 @@ -487,6 +526,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 41 @@ -519,4 +559,5 @@ num_stars: 0 num_forks: 0 num_issues: 0 - is_mirror: false \ No newline at end of file + is_mirror: false + status: 0 diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index e14437a04b3..ef5cd377a6c 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -252,6 +252,8 @@ var migrations = []Migration{ NewMigration("add repo_admin_change_team_access to user", addRepoAdminChangeTeamAccessColumnForUser), // v98 -> v99 NewMigration("add original author name and id on migrated release", addOriginalAuthorOnMigratedReleases), + // v99 -> v100 + NewMigration("add task table and status column for repository table", addTaskTable), } // Migrate database to current version diff --git a/models/migrations/v99.go b/models/migrations/v99.go new file mode 100644 index 00000000000..3eb287af6c9 --- /dev/null +++ b/models/migrations/v99.go @@ -0,0 +1,34 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/go-xorm/xorm" +) + +func addTaskTable(x *xorm.Engine) error { + type Task struct { + ID int64 + DoerID int64 `xorm:"index"` // operator + OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero + RepoID int64 `xorm:"index"` + Type structs.TaskType + Status structs.TaskStatus `xorm:"index"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + PayloadContent string `xorm:"TEXT"` + Errors string `xorm:"TEXT"` // if task failed, saved the error reason + Created timeutil.TimeStamp `xorm:"created"` + } + + type Repository struct { + Status int `xorm:"NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(Task), new(Repository)) +} diff --git a/models/models.go b/models/models.go index e802a35a777..ea550cb839f 100644 --- a/models/models.go +++ b/models/models.go @@ -112,6 +112,7 @@ func init() { new(OAuth2Application), new(OAuth2AuthorizationCode), new(OAuth2Grant), + new(Task), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo.go b/models/repo.go index 8db527477b0..23b1c2ef52c 100644 --- a/models/repo.go +++ b/models/repo.go @@ -126,6 +126,15 @@ func NewRepoContext() { RemoveAllWithNotice("Clean up repository temporary data", filepath.Join(setting.AppDataPath, "tmp")) } +// RepositoryStatus defines the status of repository +type RepositoryStatus int + +// all kinds of RepositoryStatus +const ( + RepositoryReady RepositoryStatus = iota // a normal repository + RepositoryBeingMigrated // repository is migrating +) + // Repository represents a git repository. type Repository struct { ID int64 `xorm:"pk autoincr"` @@ -156,9 +165,9 @@ type Repository struct { IsPrivate bool `xorm:"INDEX"` IsEmpty bool `xorm:"INDEX"` IsArchived bool `xorm:"INDEX"` - - IsMirror bool `xorm:"INDEX"` - *Mirror `xorm:"-"` + IsMirror bool `xorm:"INDEX"` + *Mirror `xorm:"-"` + Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` ExternalMetas map[string]string `xorm:"-"` Units []*RepoUnit `xorm:"-"` @@ -197,6 +206,16 @@ func (repo *Repository) ColorFormat(s fmt.State) { repo.Name) } +// IsBeingMigrated indicates that repository is being migtated +func (repo *Repository) IsBeingMigrated() bool { + return repo.Status == RepositoryBeingMigrated +} + +// IsBeingCreated indicates that repository is being migrated or forked +func (repo *Repository) IsBeingCreated() bool { + return repo.IsBeingMigrated() +} + // AfterLoad is invoked from XORM after setting the values of all fields of this object. func (repo *Repository) AfterLoad() { // FIXME: use models migration to solve all at once. @@ -884,18 +903,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) { return repo.cloneLink(x, false) } -// MigrateRepoOptions contains the repository migrate options -type MigrateRepoOptions struct { - Name string - Description string - OriginalURL string - IsPrivate bool - IsMirror bool - RemoteAddr string - Wiki bool // include wiki repository - SyncReleasesWithTags bool // sync releases from tags -} - /* GitHub, GitLab, Gogs: *.wiki.git BitBucket: *.git/wiki @@ -915,20 +922,28 @@ func wikiRemoteURL(remote string) string { return "" } -// MigrateRepository migrates an existing repository from other project hosting. -func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) { - repo, err := CreateRepository(doer, u, CreateRepoOptions{ - Name: opts.Name, - Description: opts.Description, - OriginalURL: opts.OriginalURL, - IsPrivate: opts.IsPrivate, - IsMirror: opts.IsMirror, - }) +// CheckCreateRepository check if could created a repository +func CheckCreateRepository(doer, u *User, name string) error { + if !doer.CanCreateRepo() { + return ErrReachLimitOfRepo{u.MaxRepoCreation} + } + + if err := IsUsableRepoName(name); err != nil { + return err + } + + has, err := isRepositoryExist(x, u, name) if err != nil { - return nil, err + return fmt.Errorf("IsRepositoryExist: %v", err) + } else if has { + return ErrRepoAlreadyExist{u.Name, name} } + return nil +} - repoPath := RepoPath(u.Name, opts.Name) +// MigrateRepositoryGitData starts migrating git related data after created migrating repository +func MigrateRepositoryGitData(doer, u *User, repo *Repository, opts api.MigrateRepoOption) (*Repository, error) { + repoPath := RepoPath(u.Name, opts.RepoName) if u.IsOrganization() { t, err := u.GetOwnerTeam() @@ -942,11 +957,12 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - if err := os.RemoveAll(repoPath); err != nil { + var err error + if err = os.RemoveAll(repoPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) } - if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ + if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -955,8 +971,8 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err } if opts.Wiki { - wikiPath := WikiPath(u.Name, opts.Name) - wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) + wikiPath := WikiPath(u.Name, opts.RepoName) + wikiRemotePath := wikiRemoteURL(opts.CloneAddr) if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) @@ -986,7 +1002,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err return repo, fmt.Errorf("git.IsEmpty: %v", err) } - if opts.SyncReleasesWithTags && !repo.IsEmpty { + if !opts.Releases && !repo.IsEmpty { // Try to get HEAD branch and set it as default branch. headBranch, err := gitRepo.GetHEADBranch() if err != nil { @@ -1005,7 +1021,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err log.Error("Failed to update size for repository: %v", err) } - if opts.IsMirror { + if opts.Mirror { if _, err = x.InsertOne(&Mirror{ RepoID: repo.ID, Interval: setting.Mirror.DefaultInterval, @@ -1143,6 +1159,7 @@ type CreateRepoOptions struct { IsPrivate bool IsMirror bool AutoInit bool + Status RepositoryStatus } func getRepoInitFile(tp, name string) ([]byte, error) { @@ -1410,6 +1427,7 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err IsPrivate: opts.IsPrivate, IsFsckEnabled: !opts.IsMirror, CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, + Status: opts.Status, } sess := x.NewSession() @@ -1856,6 +1874,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { &CommitStatus{RepoID: repoID}, &RepoIndexerStatus{RepoID: repoID}, &Comment{RefRepoID: repoID}, + &Task{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %v", err) } diff --git a/models/task.go b/models/task.go new file mode 100644 index 00000000000..cb878d387c1 --- /dev/null +++ b/models/task.go @@ -0,0 +1,240 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "encoding/json" + "fmt" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +// Task represents a task +type Task struct { + ID int64 + DoerID int64 `xorm:"index"` // operator + Doer *User `xorm:"-"` + OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero + Owner *User `xorm:"-"` + RepoID int64 `xorm:"index"` + Repo *Repository `xorm:"-"` + Type structs.TaskType + Status structs.TaskStatus `xorm:"index"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + PayloadContent string `xorm:"TEXT"` + Errors string `xorm:"TEXT"` // if task failed, saved the error reason + Created timeutil.TimeStamp `xorm:"created"` +} + +// LoadRepo loads repository of the task +func (task *Task) LoadRepo() error { + return task.loadRepo(x) +} + +func (task *Task) loadRepo(e Engine) error { + if task.Repo != nil { + return nil + } + var repo Repository + has, err := e.ID(task.RepoID).Get(&repo) + if err != nil { + return err + } else if !has { + return ErrRepoNotExist{ + ID: task.RepoID, + } + } + task.Repo = &repo + return nil +} + +// LoadDoer loads do user +func (task *Task) LoadDoer() error { + if task.Doer != nil { + return nil + } + + var doer User + has, err := x.ID(task.DoerID).Get(&doer) + if err != nil { + return err + } else if !has { + return ErrUserNotExist{ + UID: task.DoerID, + } + } + task.Doer = &doer + + return nil +} + +// LoadOwner loads owner user +func (task *Task) LoadOwner() error { + if task.Owner != nil { + return nil + } + + var owner User + has, err := x.ID(task.OwnerID).Get(&owner) + if err != nil { + return err + } else if !has { + return ErrUserNotExist{ + UID: task.OwnerID, + } + } + task.Owner = &owner + + return nil +} + +// UpdateCols updates some columns +func (task *Task) UpdateCols(cols ...string) error { + _, err := x.ID(task.ID).Cols(cols...).Update(task) + return err +} + +// MigrateConfig returns task config when migrate repository +func (task *Task) MigrateConfig() (*structs.MigrateRepoOption, error) { + if task.Type == structs.TaskTypeMigrateRepo { + var opts structs.MigrateRepoOption + err := json.Unmarshal([]byte(task.PayloadContent), &opts) + if err != nil { + return nil, err + } + return &opts, nil + } + return nil, fmt.Errorf("Task type is %s, not Migrate Repo", task.Type.Name()) +} + +// ErrTaskDoesNotExist represents a "TaskDoesNotExist" kind of error. +type ErrTaskDoesNotExist struct { + ID int64 + RepoID int64 + Type structs.TaskType +} + +// IsErrTaskDoesNotExist checks if an error is a ErrTaskIsNotExist. +func IsErrTaskDoesNotExist(err error) bool { + _, ok := err.(ErrTaskDoesNotExist) + return ok +} + +func (err ErrTaskDoesNotExist) Error() string { + return fmt.Sprintf("task is not exist [id: %d, repo_id: %d, type: %d]", + err.ID, err.RepoID, err.Type) +} + +// GetMigratingTask returns the migrating task by repo's id +func GetMigratingTask(repoID int64) (*Task, error) { + var task = Task{ + RepoID: repoID, + Type: structs.TaskTypeMigrateRepo, + } + has, err := x.Get(&task) + if err != nil { + return nil, err + } else if !has { + return nil, ErrTaskDoesNotExist{0, repoID, task.Type} + } + return &task, nil +} + +// FindTaskOptions find all tasks +type FindTaskOptions struct { + Status int +} + +// ToConds generates conditions for database operation. +func (opts FindTaskOptions) ToConds() builder.Cond { + var cond = builder.NewCond() + if opts.Status >= 0 { + cond = cond.And(builder.Eq{"status": opts.Status}) + } + return cond +} + +// FindTasks find all tasks +func FindTasks(opts FindTaskOptions) ([]*Task, error) { + var tasks = make([]*Task, 0, 10) + err := x.Where(opts.ToConds()).Find(&tasks) + return tasks, err +} + +func createTask(e Engine, task *Task) error { + _, err := e.Insert(task) + return err +} + +// CreateMigrateTask creates a migrate task +func CreateMigrateTask(doer, u *User, opts base.MigrateOptions) (*Task, error) { + bs, err := json.Marshal(&opts) + if err != nil { + return nil, err + } + + var task = Task{ + DoerID: doer.ID, + OwnerID: u.ID, + Type: structs.TaskTypeMigrateRepo, + Status: structs.TaskStatusQueue, + PayloadContent: string(bs), + } + + if err := createTask(x, &task); err != nil { + return nil, err + } + + repo, err := CreateRepository(doer, u, CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + OriginalURL: opts.CloneAddr, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: RepositoryBeingMigrated, + }) + if err != nil { + task.EndTime = timeutil.TimeStampNow() + task.Status = structs.TaskStatusFailed + err2 := task.UpdateCols("end_time", "status") + if err2 != nil { + log.Error("UpdateCols Failed: %v", err2.Error()) + } + return nil, err + } + + task.RepoID = repo.ID + if err = task.UpdateCols("repo_id"); err != nil { + return nil, err + } + + return &task, nil +} + +// FinishMigrateTask updates database when migrate task finished +func FinishMigrateTask(task *Task) error { + task.Status = structs.TaskStatusFinished + task.EndTime = timeutil.TimeStampNow() + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil { + return err + } + task.Repo.Status = RepositoryReady + if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil { + return err + } + + return sess.Commit() +} diff --git a/modules/context/repo.go b/modules/context/repo.go index 3caf583f836..f4af19a0e83 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -146,6 +146,9 @@ func (r *Repository) FileExists(path string, branch string) (bool, error) { // GetEditorconfig returns the .editorconfig definition if found in the // HEAD of the default repo branch. func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) { + if r.GitRepo == nil { + return nil, nil + } commit, err := r.GitRepo.GetBranchCommit(r.Repository.DefaultBranch) if err != nil { return nil, err @@ -358,12 +361,6 @@ func RepoAssignment() macaron.Handler { return } - gitRepo, err := git.OpenRepository(models.RepoPath(userName, repoName)) - if err != nil { - ctx.ServerError("RepoAssignment Invalid repo "+models.RepoPath(userName, repoName), err) - return - } - ctx.Repo.GitRepo = gitRepo ctx.Repo.RepoLink = repo.Link() ctx.Data["RepoLink"] = ctx.Repo.RepoLink ctx.Data["RepoRelPath"] = ctx.Repo.Owner.Name + "/" + ctx.Repo.Repository.Name @@ -373,13 +370,6 @@ func RepoAssignment() macaron.Handler { ctx.Data["RepoExternalIssuesLink"] = unit.ExternalTrackerConfig().ExternalTrackerURL } - tags, err := ctx.Repo.GitRepo.GetTags() - if err != nil { - ctx.ServerError("GetTags", err) - return - } - ctx.Data["Tags"] = tags - count, err := models.GetReleaseCountByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{ IncludeDrafts: false, IncludeTags: true, @@ -425,12 +415,25 @@ func RepoAssignment() macaron.Handler { } // repo is empty and display enable - if ctx.Repo.Repository.IsEmpty { + if ctx.Repo.Repository.IsEmpty || ctx.Repo.Repository.IsBeingCreated() { ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch return } - ctx.Data["TagName"] = ctx.Repo.TagName + gitRepo, err := git.OpenRepository(models.RepoPath(userName, repoName)) + if err != nil { + ctx.ServerError("RepoAssignment Invalid repo "+models.RepoPath(userName, repoName), err) + return + } + ctx.Repo.GitRepo = gitRepo + + tags, err := ctx.Repo.GitRepo.GetTags() + if err != nil { + ctx.ServerError("GetTags", err) + return + } + ctx.Data["Tags"] = tags + brs, err := ctx.Repo.GitRepo.GetBranches() if err != nil { ctx.ServerError("GetBranches", err) @@ -439,6 +442,8 @@ func RepoAssignment() macaron.Handler { ctx.Data["Branches"] = brs ctx.Data["BranchesCount"] = len(brs) + ctx.Data["TagName"] = ctx.Repo.TagName + // If not branch selected, try default one. // If default branch doesn't exists, fall back to some other branch. if len(ctx.Repo.BranchName) == 0 { diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index ba7fdc68156..2d180b61d95 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -5,22 +5,7 @@ package base -// MigrateOptions defines the way a repository gets migrated -type MigrateOptions struct { - RemoteURL string - AuthUsername string - AuthPassword string - Name string - Description string - OriginalURL string +import "code.gitea.io/gitea/modules/structs" - Wiki bool - Issues bool - Milestones bool - Labels bool - Releases bool - Comments bool - PullRequests bool - Private bool - Mirror bool -} +// MigrateOptions defines the way a repository gets migrated +type MigrateOptions = structs.MigrateRepoOption diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index 1edac47a6ec..ab3b0b9f694 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" gouuid "github.com/satori/go.uuid" @@ -90,16 +91,33 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate remoteAddr = u.String() } - r, err := models.MigrateRepository(g.doer, owner, models.MigrateRepoOptions{ - Name: g.repoName, - Description: repo.Description, - OriginalURL: repo.OriginalURL, - IsMirror: repo.IsMirror, - RemoteAddr: remoteAddr, - IsPrivate: repo.IsPrivate, - Wiki: opts.Wiki, - SyncReleasesWithTags: !opts.Releases, // if didn't get releases, then sync them from tags + var r *models.Repository + if opts.MigrateToRepoID <= 0 { + r, err = models.CreateRepository(g.doer, owner, models.CreateRepoOptions{ + Name: g.repoName, + Description: repo.Description, + OriginalURL: repo.OriginalURL, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: models.RepositoryBeingMigrated, + }) + } else { + r, err = models.GetRepositoryByID(opts.MigrateToRepoID) + } + if err != nil { + return err + } + + r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{ + RepoName: g.repoName, + Description: repo.Description, + Mirror: repo.IsMirror, + CloneAddr: remoteAddr, + Private: repo.IsPrivate, + Wiki: opts.Wiki, + Releases: opts.Releases, // if didn't get releases, then sync them from tags }) + g.repo = r if err != nil { return err diff --git a/modules/migrations/gitea_test.go b/modules/migrations/gitea_test.go index 88a3a6d2189..73c119a15de 100644 --- a/modules/migrations/gitea_test.go +++ b/modules/migrations/gitea_test.go @@ -10,6 +10,7 @@ import ( "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" @@ -29,9 +30,9 @@ func TestGiteaUploadRepo(t *testing.T) { uploader = NewGiteaLocalUploader(user, user.Name, repoName) ) - err := migrateRepository(downloader, uploader, MigrateOptions{ - RemoteURL: "https://github.com/go-xorm/builder", - Name: repoName, + err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{ + CloneAddr: "https://github.com/go-xorm/builder", + RepoName: repoName, AuthUsername: "", Wiki: true, diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 754f98941c1..1c5d96c03d4 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -34,7 +34,7 @@ type GithubDownloaderV3Factory struct { // Match returns ture if the migration remote URL matched this downloader factory func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) { - u, err := url.Parse(opts.RemoteURL) + u, err := url.Parse(opts.CloneAddr) if err != nil { return false, err } @@ -44,7 +44,7 @@ func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error // New returns a Downloader related to this factory according MigrateOptions func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) { - u, err := url.Parse(opts.RemoteURL) + u, err := url.Parse(opts.CloneAddr) if err != nil { return nil, err } diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 27782cb9403..3f5c0d1118c 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -6,6 +6,8 @@ package migrations import ( + "fmt" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" @@ -27,7 +29,7 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { var ( downloader base.Downloader - uploader = NewGiteaLocalUploader(doer, ownerName, opts.Name) + uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) ) for _, factory := range factories { @@ -50,14 +52,18 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt opts.Comments = false opts.Issues = false opts.PullRequests = false - downloader = NewPlainGitDownloader(ownerName, opts.Name, opts.RemoteURL) - log.Trace("Will migrate from git: %s", opts.RemoteURL) + downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) + log.Trace("Will migrate from git: %s", opts.CloneAddr) } if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) } + + if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.CloneAddr, err)); err2 != nil { + log.Error("create respotiry notice failed: ", err2) + } return nil, err } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5e476854b22..8c61bdbb771 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1043,4 +1043,5 @@ func NewServices() { newNotifyMailService() newWebhookService() newIndexerService() + newTaskService() } diff --git a/modules/setting/task.go b/modules/setting/task.go new file mode 100644 index 00000000000..97704d4a4da --- /dev/null +++ b/modules/setting/task.go @@ -0,0 +1,25 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +var ( + // Task settings + Task = struct { + QueueType string + QueueLength int + QueueConnStr string + }{ + QueueType: ChannelQueueType, + QueueLength: 1000, + QueueConnStr: "addrs=127.0.0.1:6379 db=0", + } +) + +func newTaskService() { + sec := Cfg.Section("task") + Task.QueueType = sec.Key("QUEUE_TYPE").MustString(ChannelQueueType) + Task.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) + Task.QueueConnStr = sec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0") +} diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 87396d6ce99..57f1768a0b9 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -162,8 +162,16 @@ type MigrateRepoOption struct { // required: true UID int `json:"uid" binding:"Required"` // required: true - RepoName string `json:"repo_name" binding:"Required"` - Mirror bool `json:"mirror"` - Private bool `json:"private"` - Description string `json:"description"` + RepoName string `json:"repo_name" binding:"Required"` + Mirror bool `json:"mirror"` + Private bool `json:"private"` + Description string `json:"description"` + Wiki bool + Issues bool + Milestones bool + Labels bool + Releases bool + Comments bool + PullRequests bool + MigrateToRepoID int64 } diff --git a/modules/structs/task.go b/modules/structs/task.go new file mode 100644 index 00000000000..e83d0437cef --- /dev/null +++ b/modules/structs/task.go @@ -0,0 +1,34 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package structs + +// TaskType defines task type +type TaskType int + +// all kinds of task types +const ( + TaskTypeMigrateRepo TaskType = iota // migrate repository from external or local disk +) + +// Name returns the task type name +func (taskType TaskType) Name() string { + switch taskType { + case TaskTypeMigrateRepo: + return "Migrate Repository" + } + return "" +} + +// TaskStatus defines task status +type TaskStatus int + +// enumerate all the kinds of task status +const ( + TaskStatusQueue TaskStatus = iota // 0 task is queue + TaskStatusRunning // 1 task is running + TaskStatusStopped // 2 task is stopped + TaskStatusFailed // 3 task is failed + TaskStatusFinished // 4 task is finished +) diff --git a/modules/task/migrate.go b/modules/task/migrate.go new file mode 100644 index 00000000000..5d15a506d79 --- /dev/null +++ b/modules/task/migrate.go @@ -0,0 +1,120 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +func handleCreateError(owner *models.User, err error, name string) error { + switch { + case models.IsErrReachLimitOfRepo(err): + return fmt.Errorf("You have already reached your limit of %d repositories", owner.MaxCreationLimit()) + case models.IsErrRepoAlreadyExist(err): + return errors.New("The repository name is already used") + case models.IsErrNameReserved(err): + return fmt.Errorf("The repository name '%s' is reserved", err.(models.ErrNameReserved).Name) + case models.IsErrNamePatternNotAllowed(err): + return fmt.Errorf("The pattern '%s' is not allowed in a repository name", err.(models.ErrNamePatternNotAllowed).Pattern) + default: + return err + } +} + +func runMigrateTask(t *models.Task) (err error) { + defer func() { + if e := recover(); e != nil { + var buf bytes.Buffer + fmt.Fprintf(&buf, "Handler crashed with error: %v", log.Stack(2)) + + err = errors.New(buf.String()) + } + + if err == nil { + err = models.FinishMigrateTask(t) + if err == nil { + notification.NotifyMigrateRepository(t.Doer, t.Owner, t.Repo) + return + } + + log.Error("FinishMigrateTask failed: %s", err.Error()) + } + + t.EndTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusFailed + t.Errors = err.Error() + if err := t.UpdateCols("status", "errors", "end_time"); err != nil { + log.Error("Task UpdateCols failed: %s", err.Error()) + } + + if t.Repo != nil { + if errDelete := models.DeleteRepository(t.Doer, t.OwnerID, t.Repo.ID); errDelete != nil { + log.Error("DeleteRepository: %v", errDelete) + } + } + }() + + if err := t.LoadRepo(); err != nil { + return err + } + + // if repository is ready, then just finsih the task + if t.Repo.Status == models.RepositoryReady { + return nil + } + + if err := t.LoadDoer(); err != nil { + return err + } + if err := t.LoadOwner(); err != nil { + return err + } + t.StartTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusRunning + if err := t.UpdateCols("start_time", "status"); err != nil { + return err + } + + var opts *structs.MigrateRepoOption + opts, err = t.MigrateConfig() + if err != nil { + return err + } + + opts.MigrateToRepoID = t.RepoID + repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) + if err == nil { + notification.NotifyMigrateRepository(t.Doer, t.Owner, repo) + + log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) + return nil + } + + if models.IsErrRepoAlreadyExist(err) { + return errors.New("The repository name is already used") + } + + // remoteAddr may contain credentials, so we sanitize it + err = util.URLSanitizedError(err, opts.CloneAddr) + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "could not read Username") { + return fmt.Errorf("Authentication failed: %v", err.Error()) + } else if strings.Contains(err.Error(), "fatal:") { + return fmt.Errorf("Migration failed: %v", err.Error()) + } + + return handleCreateError(t.Owner, err, "MigratePost") +} diff --git a/modules/task/queue.go b/modules/task/queue.go new file mode 100644 index 00000000000..ddee0b3d462 --- /dev/null +++ b/modules/task/queue.go @@ -0,0 +1,14 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import "code.gitea.io/gitea/models" + +// Queue defines an interface to run task queue +type Queue interface { + Run() error + Push(*models.Task) error + Stop() +} diff --git a/modules/task/queue_channel.go b/modules/task/queue_channel.go new file mode 100644 index 00000000000..da541f47551 --- /dev/null +++ b/modules/task/queue_channel.go @@ -0,0 +1,48 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +var ( + _ Queue = &ChannelQueue{} +) + +// ChannelQueue implements +type ChannelQueue struct { + queue chan *models.Task +} + +// NewChannelQueue create a memory channel queue +func NewChannelQueue(queueLen int) *ChannelQueue { + return &ChannelQueue{ + queue: make(chan *models.Task, queueLen), + } +} + +// Run starts to run the queue +func (c *ChannelQueue) Run() error { + for task := range c.queue { + err := Run(task) + if err != nil { + log.Error("Run task failed: %s", err.Error()) + } + } + return nil +} + +// Push will push the task ID to queue +func (c *ChannelQueue) Push(task *models.Task) error { + c.queue <- task + return nil +} + +// Stop stop the queue +func (c *ChannelQueue) Stop() { + close(c.queue) +} diff --git a/modules/task/queue_redis.go b/modules/task/queue_redis.go new file mode 100644 index 00000000000..127de0cdbf1 --- /dev/null +++ b/modules/task/queue_redis.go @@ -0,0 +1,130 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "encoding/json" + "errors" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + + "github.com/go-redis/redis" +) + +var ( + _ Queue = &RedisQueue{} +) + +type redisClient interface { + RPush(key string, args ...interface{}) *redis.IntCmd + LPop(key string) *redis.StringCmd + Ping() *redis.StatusCmd +} + +// RedisQueue redis queue +type RedisQueue struct { + client redisClient + queueName string + closeChan chan bool +} + +func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) { + fields := strings.Fields(connStr) + for _, f := range fields { + items := strings.SplitN(f, "=", 2) + if len(items) < 2 { + continue + } + switch strings.ToLower(items[0]) { + case "addrs": + addrs = items[1] + case "password": + password = items[1] + case "db": + dbIdx, err = strconv.Atoi(items[1]) + if err != nil { + return + } + } + } + return +} + +// NewRedisQueue creates single redis or cluster redis queue +func NewRedisQueue(addrs string, password string, dbIdx int) (*RedisQueue, error) { + dbs := strings.Split(addrs, ",") + var queue = RedisQueue{ + queueName: "task_queue", + closeChan: make(chan bool), + } + if len(dbs) == 0 { + return nil, errors.New("no redis host found") + } else if len(dbs) == 1 { + queue.client = redis.NewClient(&redis.Options{ + Addr: strings.TrimSpace(dbs[0]), // use default Addr + Password: password, // no password set + DB: dbIdx, // use default DB + }) + } else { + // cluster will ignore db + queue.client = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: dbs, + Password: password, + }) + } + if err := queue.client.Ping().Err(); err != nil { + return nil, err + } + return &queue, nil +} + +// Run starts to run the queue +func (r *RedisQueue) Run() error { + for { + select { + case <-r.closeChan: + return nil + case <-time.After(time.Millisecond * 100): + } + + bs, err := r.client.LPop(r.queueName).Bytes() + if err != nil { + if err != redis.Nil { + log.Error("LPop failed: %v", err) + } + time.Sleep(time.Millisecond * 100) + continue + } + + var task models.Task + err = json.Unmarshal(bs, &task) + if err != nil { + log.Error("Unmarshal task failed: %s", err.Error()) + } else { + err = Run(&task) + if err != nil { + log.Error("Run task failed: %s", err.Error()) + } + } + } +} + +// Push implements Queue +func (r *RedisQueue) Push(task *models.Task) error { + bs, err := json.Marshal(task) + if err != nil { + return err + } + return r.client.RPush(r.queueName, bs).Err() +} + +// Stop stop the queue +func (r *RedisQueue) Stop() { + r.closeChan <- true +} diff --git a/modules/task/task.go b/modules/task/task.go new file mode 100644 index 00000000000..64744afe7a4 --- /dev/null +++ b/modules/task/task.go @@ -0,0 +1,66 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" +) + +// taskQueue is a global queue of tasks +var taskQueue Queue + +// Run a task +func Run(t *models.Task) error { + switch t.Type { + case structs.TaskTypeMigrateRepo: + return runMigrateTask(t) + default: + return fmt.Errorf("Unknow task type: %d", t.Type) + } +} + +// Init will start the service to get all unfinished tasks and run them +func Init() error { + switch setting.Task.QueueType { + case setting.ChannelQueueType: + taskQueue = NewChannelQueue(setting.Task.QueueLength) + case setting.RedisQueueType: + var err error + addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr) + if err != nil { + return err + } + taskQueue, err = NewRedisQueue(addrs, pass, idx) + if err != nil { + return err + } + default: + return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType) + } + + go func() { + if err := taskQueue.Run(); err != nil { + log.Error("taskQueue.Run end failed: %v", err) + } + }() + + return nil +} + +// MigrateRepository add migration repository to task +func MigrateRepository(doer, u *models.User, opts base.MigrateOptions) error { + task, err := models.CreateMigrateTask(doer, u, opts) + if err != nil { + return err + } + + return taskQueue.Push(task) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ca09b6120d7..e6c5839a645 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -633,6 +633,8 @@ migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'g migrate.migrate_items_options = When migrating from github, input a username and migration options will be displayed. migrated_from = Migrated from %[2]s migrated_from_fake = Migrated From %[1]s +migrate.migrating = Migrating from %s ... +migrate.migrating_failed = Migrating from %s failed. mirror_from = mirror of forked_from = forked from diff --git a/public/img/loading.png b/public/img/loading.png new file mode 100644 index 0000000000000000000000000000000000000000..aac702cfd6d010abb22f4ebd8f011b606075ef62 GIT binary patch literal 18713 zcmZsCWl$VJ*EQ}A!3i2%7I$}8+%@RpE(sRgHRuA1ySoN=Sr)foA-H=81j3i+mHPg? zQ#I3f?%cUEbGo~x&Z*n6n(B(!7~~jmaB$elN^;t8aPYu?HVh5%-_fdn`ac)YEUk6_ z<3#$m!v)TO=KeKnnm}Fof30#Ri$|@lU4^nunVfbmFA&V3kiqCssp?v-VUWjXTp(mx zC<3YgT9?XdWOF)JYnv8|s)5-J3ng^&gp3P>&5Gr$%YmQ@4V6q5%TguPELOQRhPSu3 ze`(L}Pt0>j$F_U_o$@D&_9~oA_2E1C{O{XnytUPJ;Eu4FH2PP6Ns||Z+qv^M8h@T_ zoJ%c=4^GI;?QS=%C#%1l{ufIQ`frT>ng0#oKjXhQ`2XNmphfTxHQY>PIceR`-_O5q zB;^9|LM$n~_?SXlNwnHoP80OoY}#<=9)lacN7|qnVcBy5-u3^oDl@J9wJ1bLl7@tz7BN2m;Gg05wcOzYk8k^m7E$v7S~6a!Ogu%03&pZqBnwd9!uri9l!MTL zlsM?!1|)Pp23Mf2O&hJU2{(Uudj-gu?8Muh)x)w-T^%ujQ|bhjvsH;_>^D6}Pq2dd zr82CcS_FT0&$S`jqDm|}jl#R^uaf@J3`(Rr0BVi!WIzP;YTwJZ*-o3b#eJbHinbA(uqK*XDgJ;cfw zWVM|udK4Owkt?4E_Ew_*rEDZ7{PSiYB{GyHEkKd(Q3`I;kY3a!zB_J~$&Y@b4O-J`+Ip4Xz zl!ef&zqNd&2J>aphM1R+R1%$cg4}LVjBU@8Q7a z8^Ht{bdK8;BLo%tb?uE2VgRn!5QI}6gN!mUT8eO4;Wb)ae&1UC2Ttstbf=J14x&78 z$v82AJ28cEyRMg265!}l7YdPcmDQuqC4M6>^TByfugTI(7VB1I-w;{=$TjSkj@n<_c5-Me2q zT?r2X2h{wUgoq#r7nIx9u^*}M_!x}&aO+q>Uh^KwIueXuyX4JaGKJu2w(2cuANE-v)~ zgQ?9)ux&h*`Yjw|=ZVH(4_P51t)5AG?N3A|#z<7aI02A$+VbnRj4}6^h7f*MHV2fz zlz9NyKK1}YmGM{ZUpT8CjFo%%P7GRd|2$5^o@@ND9lDnFM~EU!N!Mjz38Or`|AN&J z<1yP<{2nWVj>MjmKOU zgZkjX?OP^yu$bZkY1;e6C5(#jfFuK7(Ed{QrN=^P*%bJq!EO0k=x&R)&qt=%{aki_ ziK1m0WC7VU7wYt}neQR7_~igq{7x;miuiN?I{qb*6}4ZCIY%jb0SprIV|YVU|Gxds zD*BPO8kd`KuhyYZIo5=qDLS&7oYclVW2{!vqQ3u!7@Ydy$dQi7(O83HGdHi5bCU=~GNw>+dS_jE+ni|80$d-t&#cLkRd=qZ5X8y|Pm}v&t^peC6 zVgH7T(Ya?xKzO3Iz+p~AGP-|0FN~UMy=@eh9btZ8WjJqc+cq#s|E+rsUJhmW>rxqS z>Q#mfgiCQRJ;3{A=8ZoNJBohr-#=RMm1X(tgxb$wF;@Ujj{D<>xe)EDXmi#@v*a-} z=a0_lIC#i^-`rUFTqM8H2a{D0$_ZtY_8p^nE2M)$Sh?Vg7mJ^KI;N z2pa=*H8Cn4o12tLDw_zUKJVL6*a@zOZ`-S|)+HU(%UT_S(NZTFD8~LlgJ+-usaWFr zsC!5(1~A+7wj~A>TV){R`als=6t2iA@gR-cR1Q$ZKRoN-aM{9o z_P<-`K^&@OLz`92EGYglBW=O(8BaF)3SoVBI8TNO$EmvX8E0)dYEn4|-Jyq)gEX!h zkasv-yVGkJ&0F;h!Em#191fVlLi_akI5L-)uqC&*8mRvIB0Nus$LP!lMSIasuQLr( zJKMJQ`nmXpG!Wt&niow1f@kf&#s01QK13YZT38_OiBEuBqg>owlEgkFj*=Xk&G-18 zd9naX#`=fPhR)2`E37SiJ*c0FYP}m#cgJ)BX=tg%Gd7w0j8*O6b;MpM|DN8uEn-p% zydaK9%4Ia0g{d$EqL<0iXKrI$&e6foDMb7ZAA9!X`2yxsU`^-DK&?EzmP3*)f$mIg zeMi59&jo8c)!xFrlfa23o*GJ^L}e1YJG(dL$wRGlhY#n`vmEneiCX8YDnKv&_XeGE z`KKvntI&QK_OB<_n!YdRR&5ZMtVYxYd$AgH5Aoq9>VruGtVNIf z;4C@OjQs{J1!BLIkoa{LE#>I?gkY>D5YXJD2s)5%WseU!Bx18d<$v-?eGrt(4g(A( zaQdF=zs`*eL2D8=Q7TLE`jLlpnMvUjifcw$-e4x+0IPq~JA~C*7sO>)#)eWN+V1Ij zimccc)n%KSZ1r^a-OxJ`X8yLdYGA>a4m25rdEoBmvk+?@FaXu1l$AeBj2x@i&Q+06 z;)@T$^m|BYLv7074|@nqjeEv zK0Vx2!9w#6I@qE|d2HDxG@@#jQ-=h>o8nk?kbLP^VK>TvGQVUMO0v$mIbX?3(nR_{ zwX=Mcc{=)Oj^Ssjc9xeye;+T8kh@=vY-U=dbTmZA8S2)B!=s0wvBtY!U{7KvgaqHA zFkE$Mfh(;G7jq_PW@mwyM8#wH$B6G`nsKM@m%f}>KfvO_oBG_7sBuF`m*cX9xGfXL z^y%+C9zJ}K{HexmdKhffI?R6I(XFG~zA-P;@#w)5BVl+)_$%gsx2|ew<2C9bm~d47 zZx%tGoZ;yZ#3$Wj^-UxnJD7h0t!XUkc-Xf<~`_=)Ge zf|gA(ppy9XeKOatjnfpNB_x^Lj@D`bf_59Ws5pRl*PXt+s$IwcN zNfOI;an2#qcs`1`u6H678VVjo9dZclU*^Z|bBH;Ae$EX@sRC3W_yU{|Bo{+YYB!Ie zq;fUVk;2FJhz$ywrC#Hl4=})xF6P?9V=uuXP6L@5_&RbUU$(c~J>Um6=B2mF(y{NS z>5P29Ol1k$1IIs-ka{rE6f(UmZ`n5qBv8@`hea56CU~WIJfpSjJUxaijCeD)^$w9q zO7cvb)^gfG+{Dx5rX>NJ9X%KEi-I29{gmRp^pXa9mpmsWN{qpq#<0*%kp^X%tdX3nD})Ji!xW5 zD}IHt^6E$ZqRPN;=1p6zpLuE1qGlZOxTM{|!shwuiLwK&3xAHOa%JrF#Tnz}n;V)n zaea-gzi0djLC&K+u_>n)+RQb-m=2P9WoAB(G5j%OyVSjT6xh%dyMyBVjnysKjOB@V z4y5syN7+qs!8c>oDeZn`xK+SZGJb0ZF1Ni*`04Qy#(`>y7gIb+^asx6zTxH$hO7M8 zFQJp8l(4cQH=UQ?7?uVPLP6kyNW1a=9jHK@?L$Z2?@#C=A9qQU91bilYe#RVZ$%cd z%5@YdK+Rk!hpUhws}+!!Jt!F|Z|sO8T3Ric5`Cs^JbNX*@n76n`O6tL+Vi3>p zeC6U}&%k5QULH1=EHBnh2jc9Rph$_9AYZ|04(P^KnI<0GhWcTH!P+Kjb{uW%ErBlK z#-`nbx;*~X^IJBiY3D#jf4Hvo@Sk<8Vb@uwFSdJ58&O3O4X&i^W;Mtk`~6?d1+ivk z3pSt};yQ@_WY=haIPbOA?|%@&XL_fDDb&eqM{AEGMBjBPnhAF|V}T9H=@02QFR)M& z72?%AEdHR(dhDCuj76RZMa6Jw%sTOthYwmz%>AxyVjoJgS;gc~GpKnAY+2{b`ZH=J zB3`@tZQZ%q1R>IVvWU_?d;d-k-Aw+s-Z{llyI2&P!efHqU8G6hcno+S>zRAC zZsY3N>(m%VKe@Qsi^|H#vd*K7aVQLiE4pIkwN|DRG|{Fs zyX92O`O8^`+1OKiJfQt@hQKRlj*x7?#(SQhx=D&p3s**5t#TC{ieU8Cc*|TR0ewp4 zj`MLpGckC_Ei(yqMNQAi4KQ3wAk=Ety6ISsJ?is#23d>yVGA^;MZYIqQBLvqGX5PC zd?*8?lc>YWP{yg>!W;`2V{7yKHkg(WWE+sJEh0DSrw?C-_TyRzbc6Jfe)_s8);_Z> zpOQP+ouP=K4CAj_Lh?s9_8c8;p`5nvQfT~~e8QXmf^pbK-=T#1(!a_#IbqSBi)PA3 z5nYKM56VDOt@${Kq8?{#I3ZgP^gPARTgGMUu>h3>%4uSFnuS_8-qB^Jd~0?05AvcD zCPzh-(?7?gK+D#wjZkj$dAzL&hb&T6KL}f+18WGGOI3ME4c5hicH*AtjcwB2Xr0<( zBV6jyEqLQJs|x`YFKGpCkb>P$E_Tlp8HyfC+zl1(2oxOeLI9IiG+Z(J)YX>th&KEc zGO(Q8|CG+yJ&~SykbHN_-Gq=v=3-BkcQobI<6Jo8Q9s0!+@ULLx9n=v7yZra1K@E< zh9??MOfspf>PvP_qunj08iP|-cXZDeuh6&BWZ%5lk`v+b$F*JF_e^34jWvOM8N=Ra zDX9sQPXR146&Y?hDi-fyfML&gF=1{H^$*P-LIzf|`;TeN(bcJzNuJy$eM`#evt3!PD`mN%QD?P@l6mr~Nh-a#PTfM<}mnUX=W z^_OGuyX_=pIHhsv4$qok*V46o2LzTcd`RWVuB(VVPlQn1PAK|MGNja@59-FuTpFP@ zz8?U%WFXYSIqeoLSAn^3ndzQlah+}W6XF!Q>F)5`J9IPV2qP_m78K2U>5lFs4A0in zhVOa=_=>hH~EgEHDd7Bb9vo-qshPE zAEBRF0A@5#6d;PN>5`0(_SOITaTJrEgM=5}sxSZ zqEN^;wnse9K>`jf@>TZbZIwhful@7qdd1)YHR<6Z^d6(vWESnbeUjwO+g`5uzD=0SrVGBWM`6Dg_p8e zO!fhMs249Z*Zjk*36%uY&Q(C_=pJWM@=ZbH(IUI$mU0e%K<*B=Pc-;>*0MUs%q;{u z+2iQ>yI=IN5N2b<_nto-{o7r@M+=eEFOFa0hA82<`U^}y!|8~tpXFL^Ux>Zj^sZmN zFF$Q@GEL8M^wkqIa@74K`*dhkK8I@XJ#@VSbw+Ao7+hsCe8GMv?cIsRAo!--kSV+v z{i+H6gIT1=N?*o`NMT+9Mb=0D__Je&eM zO59WRm>)eDVG#5^N<^aa{DW(ITn72h3OQY&^`#4*{vA`+P`>Y6?0_Ne&?L*Lj4;~~ zmUrrQfZf1*QLoiw)mdY^37Y_eJ|sv4H{A!FPP64~$?<~-_|_lSUl{9mJ2yoA7%=fU zwCTWKzS5|VG{+>Fhx)nT-nE`Jiw{~HdH`01%#We4~CMlfp+E^V66ri8{kqPRKJyEN5s;u25yqHu(Gg>4Sa^w%vo~MyL2|RFNNVmv(Vu4dSDt`f{2NH{ zTDi5Y{2&(y7F`kEJL!X=gPu9GPr=xD5-+!*3eqhO^dr09B$L}~P-?K*(AO-oIGvn< z1Inc*LoRqMPb`?HK3GWDXfuHRht|f)-7EgMlHMXCLVb7VSl_C(`hw;-&Q7EVMQCf- z6J8h`)0*pJbI^f=yj#~G>I+^(49(lXvvVITI*J?<6|O(eu!iSB< zeG90W+N5(;r-S=lv68B9*X}C?nsI*$jN^mWPX!Fb(`EjF2WzrZK!QHotQKS#b-;3QCszG3;Xxj%>7$A2w?RfXPOtamH_b5o;4z=!rasl(~W+UH5!u9Nx6LF5;1=N|>>s*&h?l-kOM!Z_P zkw)c7MTW_wCviscJxONg{)MI|$A3=tC-~0xB)i>qQI3tIl}kGH0bQ*|Os<->kpSmva=>-6ME{f%};g)A?A~m)J*(s4EEsxB>P+=5z3Vk zDP)A%jBAJQN0m7j?UjpoVvp~sJ^@bl2qdr7$pwbVrN?YjgSyYfi^T1dezS)7`TOI*GMaH1^a;TR1Tu3hNNhO<6 z-2GESxqaR#0Jk~WpyGd9uG-n~w$~v_tZCF=eUx<{e(LTNi-;N&Z~%t_6Q!svgUXex zc=9(@Fhiuk;8rCac+b7>V?+qfL>i|Pp_C>3}UI3JpTBLpq2%&_y} z%hZn=JyitSZaQ#omnFAd@P;&s)~jH zz8y;7w!{KUo@j+MXpvtDxZL}N{N?V8H`g&F5gjk90mK5w9;b|iyVR@}A+w)keAXx= zhC6W0_Llv7iS{>uk8AnS+3d0b_Pwl56R3l&Z3 z6UwPpqsuQ!c^rblhDyK|}2sDE;&@ppx?tU?0 zxqFr>kKg^bxa0IDFH=uzE^WXOE_WIME3gz<~Sl z9J+78qT>sV31i|+q7$9QI(=!Ai^4C(F&eYJNlrRE29R@idf)5B{5;R`A4L_o+q@5i zvwj8HfqZvV^uY9YjFHsTw^$#eK@O+OFk0)RhP%D#xOa!-dw0e+@ngAR%^E=&o#@Xn)J&EzO90n8rB|Yd>d@>?Xa89rVYOY$QWq|& z5jLVJtZ%J$iu4#&E>akSVeUkdl0BbUZSnBcck?>es?iV<$EKGMbAHoFV9m8#L8w&3 z+ezg!%w^mwEotpr$Y1vP()Vlivl`K1Mdv2se4-P#jkSGQ5GCWL zO~`DBRd6>IRzgn@M%uOp{@9Q5;KzHu^GM6ef}+lBh@XVSR^3#P)m%ZuXSOdl_lq8j z$$fuco@@-X5th@(z$6nN-WY({>&c>WnZ9tGB$+m@DaxnHV3(PApog6(#?WWC4s=i~ zsL$#^8H_u)KG*dA34xf6LWI{MF0f(9=9bPf;Ud`=Zj9f%nZc5?Lr$j3(p_P5@%Ka= z%+r!Q*pMgIt1F=5wh(XstNJ#SP1^Dct{hDVYU=T(!1}LGmF_Y z?gn97S5skBj26HFt1sH%EMsM!g?n{7 z1SQikOleDn!KXo$vDQuf z3)%DOM|Tlb-|3-t5kNNe1Q=yQ*>7RT(UUdmys?@Q!h{Xqo(aVFy(2V^<5$QLMt}Mg zYowws<6&K)fQF($PK#q`qmnMzAFmZF*;MXPl3fLlGl*f`-8OZ=yQK5CQJ2Fhr-q%6 z&(pKVAEj{uZ`T7@dwI331OD=H!p6M_do^A?-*ZnfA&=|gP9w8N{p3kredf<7tS!mB z$(yw1n}>G?&@lA|Z_vTv0mq$M_LG03PC>9PtXqu?j%J8A1?(+&J6AYKbuvf zUk)DEfUEw^Gv`VF)~kI>Fn6f*#k918x=Q42oenWDz$3RV+R>%^ufgRPc<0m1mHqy1 z(eRasP?`jg?78EDKKAFciy7O&x3@oWYpl!FT3AmK)WS$IoXImV7>q}FkMN0xp`TTx zSsPb|k?=6Tt$hZes@~bY%!STC<{M0Vtz7L|7?9RRcCD3NiX9Wgqo=wKZ0x_#MFiCm1x6zjDlxkL-KvfZ>q|HdGNn zCgoZuz!lweyx!tK)_z#3e+^T$FlA^1YA9rSJBk^2fHjP?^@+-^kv$NsMu zY`S2VoNj-*od|Ev<{BM^|J3XJuXy#({=bS)PFmS6{)tyb|07-@{S&Y9Fh4B-NTW6o z4q$5*0|S_}lRU=m+H*w!t~;!^H%Xo%D(aSU2fa+Jpkz@y(eBXXt51_ze8xrxd@9R> zg%%|75kxK5=BMtwo`O{R^6;iV5X0q!o7j+1pi(jO~p(M-&cq-8u#j8mH{Yno$IxWBVf zKj6W?CEIO3V(NgW@Grgo3)4a8&~@!WE z(j6arK6?y}a%`R4|Jicw7}q0N6aM`?t<(^(?Aoz&KMU7l6(vUHK06gEyOni-U(Lganl_2Hl1LBqPOR*gz7fQMpz2~6>R6B>J zAc7|TmxX4060tDY1sGN(?E$VAnUGDjTkf$Ar6g3KiW-EnB`PCL%DkkfI!U?NpaCHZ zV8YPmg~p$bP0MZrVuuhTZCz?DH`w$_1E0nBBZa5~s7x{UIO(3u{*(|9__7n~7h(B+ z2#PPJjOF3!IH@aG=p=}6_=t8W%(%yp={uQPT^g)0lK>&B3xUuvDzk4je=v{@UUij& z-GdV3g0vQ-1y`F0blV?rcN0R$)DqPR5@J2H4pr^>-i9{^y$Pmp2TLtK)D#AVFT_At zqJ}56Bti-Kw0#Uinomi+K%FjHgL3(RS@w)mU_glj0`NJbD^_(9ol*D$lzi;$4KR9X z6R^>gTsBr~sFKl|9Gy6kz1t`JX~obH+Du-qV?*I(ak?1i(;n}%o-ZWPq&=}BG0z9b zC=Y*Typm1C$hgb1@Oyg#3H0I&aYZ3490jhmlkEgsU?$5&qLGm^0; zvQ=+FLfE4HV9Mgro~}brP%U2{qfnXqY zI|ZEAFcK%-#kva<2h@D!n@AF;oxc_A?X;;g`;!ae z3Jo8=Z-B$b)TrT$d%#_^T23D`w`cD+@-KKo{K`aLkgTDE?qBzSyy5nXFU8#bX4qW9 z>KynuSy_Y12J*rNH9>QAn33FE(XvT*+K6g~*`i@GdMI8o9vrd@=~V1uJ>31K*pcdy00#-N3;A1YC$EV(|E+h8|3a7^%*3Ph#{^s^`cdghQf2ffIQ_gEmsm#V=$ z_YHm7fppmS8KoSDM6%sdjReBWP(suWsCv%im>e!yS}=~#{34nwbHdNXOnsU#jfDRM zRv|MjQ6DEVcwClh-z|+fuMeeQ5O>y^ZWU83tl&A^Ks(HuS3*@6QX{Qq z3C4=CaQ&G0&C$_%A?3i<9x4PI&ho94)P%kD`PIZpgVMW++@?Ln&Q(6 zbL1B9zk|1r1c#?p0p$Xa?Sx%122rJ)I3~{Y@g~7-9kM&|0s{0*N9&=>G0T09T}kv% z{*;lNiYkZh;9z;>vPdQ*^fFUS^|d!GhM9At7F@O|M2fq;Wjt@;9*UDe^k`wJipf+&5BfR6^sD5@OxdK@7yGSwy z#~5dSE7Wwr*0U+8g|y9)uA1;cZwAJx3wenGrl=b!zKS`FdKpq5XCHd|{)&pLd*~w2 z_o6NugvWOC@@?80B;CQwDoEOV6Hh<~rx*rRm0Q}M#Io*t{oE!rzDn>-no?qTw2UK@ z#8AV^)}-(WP%iw0vn`B~B`QF*nMUNm4-w;&v>~|goztxmFbnrEpQ0rm>Sv;5>t0mI zs0ySw$=*`Na{=1JJ6$Gwk&)N)(6ZM!_}!(G#ow8&e`XB~f)Fzv3IRn5?SG~Jd?68Z z0E!@^f8|C}|2l>`Exk4kzl)<52C7k`eg}AWBzh+IF(e=O+!E{odsm~D)(bp1MdLGT z_J71X(igz6#?TE(r<-cJ3-(Zu{g9*oUTFk`lnQb$ZzLGOwFd51Oy=z9iB!RdXuN?;h$XnFQLQ0}?5eT}%=q1a%0ntGXF%)r(8w zvXK*#bChn*5j9-^j?ylMDi`P-2i93f*X5~U)!cR;Sf1;#xk!kBgnz82rgJfoL|!@P zCgpQuyyC~<18JvYl0n+cMqW0B)~#p2zvu{N@JB8AtDie@KMmKBc_CX9+|hTniz5Df zg`=Y9nLDwtb43+Xd)ScuV11bru82TZZJJpwIp>9v!v}-d{rHvtiNfUd7n|!0SO`bT zCu8I@)3i`4(SauFa5>#9g8fl#JpR4-&&`FH6IJy`X*ru^u$Wx>8FubK*Y0GPTitfo z>gM^xU+&f*b$JpHCHcg`d-xWr5RhIWd$4%Z-Jqy6#6$#V;L{L9Ij1vLzkH)`)k8&p zUa#O6``@?8WLqFZVdOHtS1L6o%|loyNUxZpRLIZr3Zl}LDU%|K-4z;V<{&yK{B^;h zba0hz8nyB`foo(gi5veQDGv7Q7h{q^^vl3lgto|=JE1{rzZZ^-SK}4(`u8z8COJ-L za|kUuopUY%(usag!c+i;g1oi9O)7?*HfQVc-EG(RL?Jx3L*0G zCsG!YdzK;U+K(vuCd5X~V^-wA*0v9OKmVo z=drGjVAmW1d-v;%5DOCCq7Un8n z)sDU)i{=n7&qo@euVd#Me23R0S~dht+g8Fb={BJT-xtW03Udk`w^97mgd;01V(~=j zew*hf6vUP%h`A!gU#f$BEc+Nsf+kDxRQ!55*6dXFOpD$aUMp0N|*5jby3c54@Xk8 zur3rJu&pSs2LFUMVe*fXB+AmYL^u35{^NlkFM+xWN;jon0^zK!XlEE^VNrG{tq|tk zSa3eP2MbY&7_Af3(lG13Wu_%yz8iTe41r}nda>&JNsf45K2u+#(9_`P>d3j`a3P5} zi3u)GY4)$nj%e;6orFMlZmaBPZ58L_78@Y+i*wa{V7}44kvfqk7Z++|W0&St<6kOf zVH9_Jjt~=0b~3`5j*cYTw%*x14X%D$C9i6@cr`_lGl$i$DcK|5ukNCTc(SYs4sMaF zVhf{+)oZIsZeC_vbBCXL@i=%7**N5B{M3vV1GApeHJ7rk3Y(B-JPg1$pueR?J*^T|RD7+btUXQk& zf}`gjgLyO-JopBvP!u)k+4$on9T`Q1=!<-!aos$*h>0*=P7jm3mLLOWf3d7~SI1ls5vyPD% z<3!pgv2dG&xp}sKhWm<_6=vu@4D?LhW&8aOuA>j3E~M8xiTT-_JS6fZ%!Loemz&mQ z-=o*cgj;7PN7ln45kq4L0wyYyi#t`PWoVa<)G`wn8ZGD>PddI#?I?~px8 zDmC6)dyb*&@s}S_erznKf($FEG&#&C#*iJ+3zqNo%&om|@a0%{guEq5UP_vZi_=;Y zw;c|2#Pc3@biGwk@b>4UhSrA?jj>J2g`)mM{&mw zNZshMSVzb1W9LE-1PEI~>VL+Zh4>dKY`k@ZUY}x|eNWLEe3sQac_tZPV39u+Zot*5 z0?lrU(d_h}zyi}=^-VrId^@@zdg*O1$Ex>HYAr$})6sz}P@95w9aVl)tXz;soD0M> zKt1VGw_`OU{_*q0XVZ<#T)dj)(#^k(=nyT>ysK%$5BR?b&QOHUL>}I89`n^(ar36} zNfR`!_!qcfVDY=4*f~=`lFu_GOD8-jOgKL8zJ8S$8TNf;h~}+i=k#Qk+)w5c zsne+<9QK;UkD{ynA&y@2S6AQS?v0!I?EKr0dm@mVcFS`ZE7c=N_vZLb((N|mV-e-| zVk%(KkwtkY<4zog!Iw>`x&KsH{ja2j{C`VYNMDSy{z+OA|08Lk{C`PnQKo2<3}7K@ zUa34`FHI>cJ>ElA@%BcY4R-z5RPIp0jvtitK2N=3cAM*?t8FE7{gt2y)ohhMdX-2L z@3Hr3MT!3E<3?F({NFUfj-}d1N||e_2!QlUREo*TL(gCLO=p2`w*C9viVmwjTgFuv zS>F!I3nMUw<+uG9AWwl3x%fTX+qDe_Y&R6_(l6@EtQz&J4FxyHx99DCzD~gz=A3o9 zjvcSAHT^Y@;7Q*&e`AW zq1p_F9x=Z;_1~x>qM}C&C}K`{0<=eVmtl?FedY~r5<%Zx`&|Wcr`^77T`cQ)e;dV0 z_Ka%Y9bsf*IdoPFC*o7+&X|1X+aJ0u+RrjfN&{J*!q@c|W6}@_`w}c9CP|&Bu;-yV zcaUzPVV&Qh{@X1NH(Jb?3jWR7T;sc^J{t?>oc*`2-#NPkEIG(1;1iz$w~`OOsL7ADkdAWCaU-lr4zQ6D(lo3@ukX{wsu5fXQ6qu^x{3#73nBM*!aw9pZ|fSN{Ft+*e1bv4W|$k0j0m{%XY+D* zhOWP~D%}LlrJVS@M)`&8dHm9vA=jz9Ug>Ixx~v1gO}Mjj&=89vE(u5EBs=5Q>cy6tonkak(#5y# zlx~(jz}>%WshAxOZ59B_f$tYO(oqhN6xt`3kiB4f`e!cbA#7c zhm5#EV{A05gACKQWl>A5=o^L%i=^JeN@83&wh38D@jR8nV|Ew%>3*cs=Qc=$dyd$) z+T6iA6Li_GdRt%67+um6?f!-=L~-k!x7ryzQM%4CpHLLof7d!R1^IG~cSd`+f^TBW zv3d6XaG#n>jF(1u^Koy#X0fO-eQzIqO_)1GC9EyTRNUl^Ei|h`a@(z{II^mJz^VXWQ|$8+)1jBBL%ep!t7WrUSE8F=<9 znjUl7)EWCNo2Q}feFLKW4X%DhSKu0#>LG^BDJkV=Y$Qg9PE+?xT5P(2R2Dmh99LHZ z)VMJBo1gi}s}9p(7FQRaub_qetQG6pP&)D&;F&2{l~Js_fTJZbMc%29Iq<6${+P(g zP&#mp7a`jL;6N+YkNL?Wd*03+yfE3nSjL@w9oTwXj8BW5PH`0?xIvr6ng@S?OpTt$ zI0jsYp54xebool^XzW=m`hs68{oYv7sy_dl_#w`R)``&fQ)*9hi@)SEp{pcm9WoVU z`wlr=w$-yCny#h~t*_UEFQ2)yJ@XV;DRYFy?2BBF1Srkq)&~RqxU=2!ba(|0rZ&4M zF47;-bD0I{XGVvHQFPpIJe&9yf(x@tKx29({-M%@)b4Xjs5%V1KP2CP*vF;#mh>#V zdkWFvo;o_`?bH%rX-cTU?aaa6S%krcBWCvLa@V^6D!*_C1*UuI&dv;!J^ioJgrFS8 zuqW*vcoFLPf9`z|m29K{0j!*_oL~feAiYS@+klQ#&W3{jCM)|aH9uGf?XV9vZhtWs z%-|!lq*1{?!!_Fy{)7?sUs*q|Y3Lk*>K#tQl?ey=iV2@_TfKZFx%w&`=)lx^CMwY@ z=o)a+sM;YOB@_9Fi7Efw4Wkt0-a}4;cKuig%C5eoewSRv)#5=o|L!j%{I#=G<26}3 zq1)mUqqv+th$YzrQLi8z)J0khQSS^fiyFj1nb|Sc&P4k|E};+D0dke9!bV9FfkNye zaJ$5PU$DjK>S7Mjr&JGS8Jt5c zLYSoTx$M9GY&c7P zj$kllRrr$x8BkczxQ97J-|mJ9@!kg~^71H; z@^$^fH!|UBKhMQ&5kM4~CJ*58oj(J}S}7ds&Fi5%&oZgDn(KXxZDlkTRl|58EjWR44!OKjKxD z&(C#{-dIunGob^uF~`LT_Qhd#UL7B#9N#oin1gjOaw=<-5U?@+BuX?IJ^-@!TY7Y% zh0*?*DpANzpgcXRCDL&8{{<@u)c5Q%f;zwIJ zrMlrPHBlI0TSwEHFeiVUM_G)yOgSmMKh&V71M}&%@h~gg^~UvYM&FFH7D&`YC_})Pk!6Li_VJZB&jx&xF&r+fK{$ygf|6K=okEv>tpYQ9A7wQ~LuQ zX}se(c;H!k(cPZ6UkSE*DljH7RU&@p(FGU~==a+W15uCWwhOu%F^CUUt$z-;-NH1U37g4C|JJB}U4 zxhTMx*m0wY3&vayHIalgi2^D1&-pG;(Ftfq7?Z1F&-s`%h%S-(I48qkOzJqyom|44 z6UJny*vXWxk96SouUx@;fr?&Or3~1Ni72Pl*_$SA7KDQH0#&M}bp8=0J|&JQCut7y zc+>N~1gsaR=t+p0aT*iTH(JLzX`%_@xY42@L z4{@H?m~&j9l2|d*yLUNaF(z8cQ%hTww!bG%U;A3B*ttzE*Uo4WEM4rWa~KmvX4l$7 z&ANvoNiWV_Vd4zoEm$Uwn`APk5{$+~k*TjvVoq(G1MP2iMAXSvU?NlgnDw~U&pwJw zT~&Mg;MAQQ3HtOPI1vTAa(Rr2BC}%ct!7OFC%ypfiu}18=UQ>NAV-rFnfbuUYb%eO zB~G1Gq!;I5kk-#Y$mjaqJB-Oc$)w$h%$_r!(=d^pA!<83;58;bBVk*))onKtoZzXE z&yB#O%JC^_%pf#!jrYVdpw@BS_Ym&6^Etg{iuEuj`pH0Y8xu1QGitVNAO57`T;#~+ zXtkl#-H6K!#wk{;yy{VQQ$I_wUmiY14Or^msPSW;n3h8fNV|SM^aB6`{8{+QEVoy*n07E@O(HIQdq9aF0d*9Y3=$T>}cPF)u1c zwLYer95{!M*33)eN3~F}zqSgM;#teNF z>_zD%25~Gn$6H1Fgv*#eFvu)UHmu2kb0pO|%Wn~rF=pt|nz$rpAU3u3kkE%W{0Opnk&d3DI--VBhw)75t5CcwDV~VfE zkBhccCjy;i93L6l`8l=9r7AKy15Q3;3UQw7{FF>59`0Y5!WkUEn17Nj$%wf-72-Jjr%6O{1W24+(Dkm@$Q@S(kq@~dM8%d z6*uYELFsA6!=h4+*LSHF&o;{*=t1c98)Yrmw_Zr@|1CF7Afm2@X0)0x6ocU% zp;P7*L1E%uNx~2Ds>ebL)eA4Q@Bc=_4%!V?#>Qxqevj~RlKdz6IlCJlLZTPzsiu?PezWgML<=NzEKeTlam7>L>MIz^?<2^1#sEI(N62&l4D2W+0Aax5x<=F2H` z%g?L=0kOP^4ui<4xbQ>eVLBDwGTT3NQgtn|0Vu9H1H`+%ha==)biO7`7!2a@;%p22 kJJzct@F0X_`M)OL0qtIKw2(NPegFUf07*qoM6N<$f_ez0CjbBd literal 0 HcmV?d00001 diff --git a/public/js/index.js b/public/js/index.js index 8a85ad91579..3b15ad8f188 100644 --- a/public/js/index.js +++ b/public/js/index.js @@ -241,6 +241,41 @@ function updateIssuesMeta(url, action, issueIds, elementId) { }) } +function initRepoStatusChecker() { + const migrating = $("#repo_migrating"); + $('#repo_migrating_failed').hide(); + if (migrating) { + const repo_name = migrating.attr('repo'); + if (typeof repo_name === 'undefined') { + return + } + $.ajax({ + type: "GET", + url: suburl +"/"+repo_name+"/status", + data: { + "_csrf": csrf, + }, + complete: function(xhr) { + if (xhr.status == 200) { + if (xhr.responseJSON) { + if (xhr.responseJSON["status"] == 0) { + location.reload(); + return + } + + setTimeout(function () { + initRepoStatusChecker() + }, 2000); + return + } + } + $('#repo_migrating_progress').hide(); + $('#repo_migrating_failed').show(); + } + }) + } +} + function initReactionSelector(parent) { let reactions = ''; if (!parent) { @@ -2219,6 +2254,7 @@ $(document).ready(function () { initIssueList(); initWipTitle(); initPullRequestReview(); + initRepoStatusChecker(); // Repo clone url. if ($('#repo-clone-url').length > 0) { diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index d8b06862a5e..08c0635bc31 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -398,8 +398,8 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { } var opts = migrations.MigrateOptions{ - RemoteURL: remoteAddr, - Name: form.RepoName, + CloneAddr: remoteAddr, + RepoName: form.RepoName, Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, diff --git a/routers/init.go b/routers/init.go index 1efddcfaa6c..c37bbeb6b08 100644 --- a/routers/init.go +++ b/routers/init.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/markup/external" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" + "code.gitea.io/gitea/modules/task" "code.gitea.io/gitea/services/mailer" mirror_service "code.gitea.io/gitea/services/mirror" @@ -102,6 +103,9 @@ func GlobalInit() { mirror_service.InitSyncMirrors() models.InitDeliverHooks() models.InitTestPullRequests() + if err := task.Init(); err != nil { + log.Fatal("Failed to initialize task scheduler: %v", err) + } } if setting.EnableSQLite3 { log.Info("SQLite3 Supported") diff --git a/routers/private/serv.go b/routers/private/serv.go index 71c0f6ea2c4..c4508b4cb5e 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -119,6 +119,15 @@ func ServCommand(ctx *macaron.Context) { repo.OwnerName = ownerName results.RepoID = repo.ID + if repo.IsBeingCreated() { + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": "Repository is being created, you could retry after it finished", + }) + return + } + // We can shortcut at this point if the repo is a mirror if mode > models.AccessModeRead && repo.IsMirror { ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ diff --git a/routers/repo/repo.go b/routers/repo/repo.go index b67384d7219..bfd0c771b05 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/task" "code.gitea.io/gitea/modules/util" "github.com/unknwon/com" @@ -133,8 +134,6 @@ func Create(ctx *context.Context) { func handleCreateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form interface{}) { switch { - case migrations.IsRateLimitError(err): - ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form) case models.IsErrReachLimitOfRepo(err): ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form) case models.IsErrRepoAlreadyExist(err): @@ -221,6 +220,40 @@ func Migrate(ctx *context.Context) { ctx.HTML(200, tplMigrate) } +func handleMigrateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form *auth.MigrateRepoForm) { + switch { + case migrations.IsRateLimitError(err): + ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form) + case migrations.IsTwoFactorAuthError(err): + ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form) + case models.IsErrReachLimitOfRepo(err): + ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form) + case models.IsErrRepoAlreadyExist(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tpl, form) + case models.IsErrNameReserved(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tpl, form) + case models.IsErrNamePatternNotAllowed(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form) + default: + remoteAddr, _ := form.ParseRemoteAddr(owner) + err = util.URLSanitizedError(err, remoteAddr) + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "Bad credentials") || + strings.Contains(err.Error(), "could not read Username") { + ctx.Data["Err_Auth"] = true + ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tpl, form) + } else if strings.Contains(err.Error(), "fatal:") { + ctx.Data["Err_CloneAddr"] = true + ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tpl, form) + } else { + ctx.ServerError(name, err) + } + } +} + // MigratePost response for migrating from external git repository func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { ctx.Data["Title"] = ctx.Tr("new_migrate") @@ -258,8 +291,8 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { } var opts = migrations.MigrateOptions{ - RemoteURL: remoteAddr, - Name: form.RepoName, + CloneAddr: remoteAddr, + RepoName: form.RepoName, Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, @@ -282,47 +315,19 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { opts.Releases = false } - repo, err := migrations.MigrateRepository(ctx.User, ctxUser.Name, opts) - if err == nil { - notification.NotifyCreateRepository(ctx.User, ctxUser, repo) - - log.Trace("Repository migrated [%d]: %s/%s successfully", repo.ID, ctxUser.Name, form.RepoName) - ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + form.RepoName) + err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName) + if err != nil { + handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form) return } - switch { - case models.IsErrReachLimitOfRepo(err): - ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", ctxUser.MaxCreationLimit()), tplMigrate, &form) - case models.IsErrNameReserved(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tplMigrate, &form) - case models.IsErrRepoAlreadyExist(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tplMigrate, &form) - case models.IsErrNamePatternNotAllowed(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tplMigrate, &form) - case migrations.IsRateLimitError(err): - ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tplMigrate, &form) - case migrations.IsTwoFactorAuthError(err): - ctx.Data["Err_Auth"] = true - ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tplMigrate, &form) - default: - // remoteAddr may contain credentials, so we sanitize it - err = util.URLSanitizedError(err, remoteAddr) - if strings.Contains(err.Error(), "Authentication failed") || - strings.Contains(err.Error(), "Bad credentials") || - strings.Contains(err.Error(), "could not read Username") { - ctx.Data["Err_Auth"] = true - ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tplMigrate, &form) - } else if strings.Contains(err.Error(), "fatal:") { - ctx.Data["Err_CloneAddr"] = true - ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tplMigrate, &form) - } else { - ctx.ServerError("MigratePost", err) - } + err = task.MigrateRepository(ctx.User, ctxUser, opts) + if err == nil { + ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + opts.RepoName) + return } + + handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form) } // Action response for actions to a repository @@ -460,3 +465,19 @@ func Download(ctx *context.Context) { ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } + +// Status returns repository's status +func Status(ctx *context.Context) { + task, err := models.GetMigratingTask(ctx.Repo.Repository.ID) + if err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err, + }) + return + } + + ctx.JSON(200, map[string]interface{}{ + "status": ctx.Repo.Repository.Status, + "err": task.Errors, + }) +} diff --git a/routers/repo/view.go b/routers/repo/view.go index 1967b511ca4..c4e6a69220a 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -11,6 +11,7 @@ import ( "fmt" gotemplate "html/template" "io/ioutil" + "net/url" "path" "strings" @@ -31,6 +32,7 @@ const ( tplRepoHome base.TplName = "repo/home" tplWatchers base.TplName = "repo/watchers" tplForks base.TplName = "repo/forks" + tplMigrating base.TplName = "repo/migrating" ) func renderDirectory(ctx *context.Context, treeLink string) { @@ -356,9 +358,37 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } } +func safeURL(address string) string { + u, err := url.Parse(address) + if err != nil { + return address + } + u.User = nil + return u.String() +} + // Home render repository home page func Home(ctx *context.Context) { if len(ctx.Repo.Units) > 0 { + if ctx.Repo.Repository.IsBeingCreated() { + task, err := models.GetMigratingTask(ctx.Repo.Repository.ID) + if err != nil { + ctx.ServerError("models.GetMigratingTask", err) + return + } + cfg, err := task.MigrateConfig() + if err != nil { + ctx.ServerError("task.MigrateConfig", err) + return + } + + ctx.Data["Repo"] = ctx.Repo + ctx.Data["MigrateTask"] = task + ctx.Data["CloneAddr"] = safeURL(cfg.CloneAddr) + ctx.HTML(200, tplMigrating) + return + } + var firstUnit *models.Unit for _, repoUnit := range ctx.Repo.Units { if repoUnit.Type == models.UnitTypeCode { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 11f2029226a..8dfcdb9c9b6 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -845,6 +845,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/archive/*", repo.MustBeNotEmpty, reqRepoCodeReader, repo.Download) + m.Get("/status", reqRepoCodeReader, repo.Status) + m.Group("/branches", func() { m.Get("", repo.Branches) }, repo.MustBeNotEmpty, context.RepoRef(), reqRepoCodeReader) diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go index 76bd4c72f76..9ad11b72656 100644 --- a/services/mirror/mirror_test.go +++ b/services/mirror/mirror_test.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/structs" release_service "code.gitea.io/gitea/services/release" "github.com/stretchr/testify/assert" @@ -26,16 +27,26 @@ func TestRelease_MirrorDelete(t *testing.T) { repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) repoPath := models.RepoPath(user.Name, repo.Name) - migrationOptions := models.MigrateRepoOptions{ - Name: "test_mirror", - Description: "Test mirror", - IsPrivate: false, - IsMirror: true, - RemoteAddr: repoPath, - Wiki: true, - SyncReleasesWithTags: true, + opts := structs.MigrateRepoOption{ + RepoName: "test_mirror", + Description: "Test mirror", + Private: false, + Mirror: true, + CloneAddr: repoPath, + Wiki: true, + Releases: false, } - mirror, err := models.MigrateRepository(user, user, migrationOptions) + + mirrorRepo, err := models.CreateRepository(user, user, models.CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: models.RepositoryBeingMigrated, + }) + assert.NoError(t, err) + + mirror, err := models.MigrateRepositoryGitData(user, user, mirrorRepo, opts) assert.NoError(t, err) gitRepo, err := git.OpenRepository(repoPath) diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index fc7f1b660ca..9fb3e32899a 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -16,93 +16,95 @@ {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{MirrorAddress $.Mirror}}
{{end}} {{if .IsFork}}
{{$.i18n.Tr "repo.forked_from"}} {{SubStr .BaseRepo.RelLink 1 -1}}
{{end}} -
-
- - {{if $.IsWatchingRepo}}{{$.i18n.Tr "repo.unwatch"}}{{else}}{{$.i18n.Tr "repo.watch"}}{{end}} - - - {{.NumWatches}} - -
-
- - {{if $.IsStaringRepo}}{{$.i18n.Tr "repo.unstar"}}{{else}}{{$.i18n.Tr "repo.star"}}{{end}} - - - {{.NumStars}} - -
- {{if and (not .IsEmpty) ($.Permission.CanRead $.UnitTypeCode)}} -
- - {{$.i18n.Tr "repo.fork"}} + {{if not .IsBeingCreated}} +
+
+ + {{if $.IsWatchingRepo}}{{$.i18n.Tr "repo.unwatch"}}{{else}}{{$.i18n.Tr "repo.watch"}}{{end}} - - {{.NumForks}} + + {{.NumWatches}}
- {{end}} -
+
+ + {{if $.IsStaringRepo}}{{$.i18n.Tr "repo.unstar"}}{{else}}{{$.i18n.Tr "repo.star"}}{{end}} + + + {{.NumStars}} + +
+ {{if and (not .IsEmpty) ($.Permission.CanRead $.UnitTypeCode)}} +
+ + {{$.i18n.Tr "repo.fork"}} + + + {{.NumForks}} + +
+ {{end}} +
+ {{end}}
{{end}} -
-
- {{if .Permission.CanRead $.UnitTypeCode}} - - {{.i18n.Tr "repo.code"}} - - {{end}} - - {{if .Permission.CanRead $.UnitTypeIssues}} - - {{.i18n.Tr "repo.issues"}} {{.Repository.NumOpenIssues}} + {{if not .Repository.IsBeingCreated}} +
+ {{if .Permission.CanRead $.UnitTypeCode}} + + {{.i18n.Tr "repo.code"}} - {{end}} - - {{if .Permission.CanRead $.UnitTypeExternalTracker}} - - {{.i18n.Tr "repo.issues"}} - - {{end}} + {{end}} - {{if and .Repository.CanEnablePulls (.Permission.CanRead $.UnitTypePullRequests)}} - - {{.i18n.Tr "repo.pulls"}} {{.Repository.NumOpenPulls}} - - {{end}} + {{if .Permission.CanRead $.UnitTypeIssues}} + + {{.i18n.Tr "repo.issues"}} {{.Repository.NumOpenIssues}} + + {{end}} - {{if and (.Permission.CanRead $.UnitTypeReleases) (not .IsEmptyRepo) }} - - {{.i18n.Tr "repo.releases"}} {{.Repository.NumReleases}} - - {{end}} + {{if .Permission.CanRead $.UnitTypeExternalTracker}} + + {{.i18n.Tr "repo.issues"}} + + {{end}} - {{if or (.Permission.CanRead $.UnitTypeWiki) (.Permission.CanRead $.UnitTypeExternalWiki)}} - - {{.i18n.Tr "repo.wiki"}} - - {{end}} + {{if and .Repository.CanEnablePulls (.Permission.CanRead $.UnitTypePullRequests)}} + + {{.i18n.Tr "repo.pulls"}} {{.Repository.NumOpenPulls}} + + {{end}} - {{if and (.Permission.CanReadAny $.UnitTypePullRequests $.UnitTypeIssues $.UnitTypeReleases) (not .IsEmptyRepo)}} - - {{.i18n.Tr "repo.activity"}} + {{if and (.Permission.CanRead $.UnitTypeReleases) (not .IsEmptyRepo) }} + + {{.i18n.Tr "repo.releases"}} {{.Repository.NumReleases}} - {{end}} + {{end}} - {{template "custom/extra_tabs" .}} + {{if or (.Permission.CanRead $.UnitTypeWiki) (.Permission.CanRead $.UnitTypeExternalWiki)}} + + {{.i18n.Tr "repo.wiki"}} + + {{end}} - {{if .Permission.IsAdmin}} -
- - {{.i18n.Tr "repo.settings"}} + {{if and (.Permission.CanReadAny $.UnitTypePullRequests $.UnitTypeIssues $.UnitTypeReleases) (not .IsEmptyRepo)}} + + {{.i18n.Tr "repo.activity"}} -
- {{end}} -
+ {{end}} + + {{template "custom/extra_tabs" .}} + + {{if .Permission.IsAdmin}} +
+ + {{.i18n.Tr "repo.settings"}} + +
+ {{end}} +
+ {{end}}
- diff --git a/templates/repo/migrating.tmpl b/templates/repo/migrating.tmpl new file mode 100644 index 00000000000..34031d5653e --- /dev/null +++ b/templates/repo/migrating.tmpl @@ -0,0 +1,31 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
+
+
+ {{template "base/alert" .}} +
+
+
+
+ +
+
+
+
+
+
+

{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}

+
+
+

{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}

+
+
+
+
+
+
+
+
+{{template "base/footer" .}}