mirror
/
gitea
mirror of https://github.com/go-gitea/gitea
2
0
Fork 0
Code Issues Releases Wiki Activity

Add migrate repo archiver and packages storage support on command line (#20757)

Browse Source 
* Add migrate repo archiver and packages storage support on command line

* Fix typo

* Use stdCtx

* Use packageblob and fix command description

* Add migrate packages unit tests

* Fix comment year

* Fix the migrate storage command line description

* Update cmd/migrate_storage.go

Co-authored-by: zeripath <art27@cantab.net>

* Update cmd/migrate_storage.go

Co-authored-by: zeripath <art27@cantab.net>

* Update cmd/migrate_storage.go

Co-authored-by: zeripath <art27@cantab.net>

* Fix test

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: zeripath <art27@cantab.net>
pull/20776/head^2
Lunny Xiao 2 years ago committed by GitHub
parent 86c85c19b6
commit 1f146090ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 187 additions and 134 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 23
      cmd/main_test.go
  2. 83
      cmd/migrate_storage.go
  3. 74
      cmd/migrate_storage_test.go
  4. 34
      models/db/iterate.go
  5. 23
      models/git/lfs.go
  6. 22
      models/repo/attachment.go
  7. 24
      models/repo/repo_list.go
  8. 23
      models/user/search.go
  9. 10
      modules/packages/content_store.go
  10. 3
      routers/private/mail.go
  11. 2
      services/repository/avatar.go

23
cmd/main_test.go
Unescape View File

@ -0,0 +1,23 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"testing"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/setting"
)
func init() {
setting.SetCustomPathAndConf("", "", "")
setting.LoadForTest()
}
func TestMain(m *testing.M) {
unittest.MainTest(m, &unittest.TestOptions{
GiteaRootPath: "..",
})
}

83
cmd/migrate_storage.go
Unescape View File

@ -12,9 +12,11 @@ import (
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
"code.gitea.io/gitea/models/migrations"
packages_model "code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
@ -25,13 +27,13 @@ import (
var CmdMigrateStorage = cli.Command{
Name: "migrate-storage",
Usage: "Migrate the storage",
Description: "This is a command for migrating storage.",
Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
Action: runMigrateStorage,
Flags: []cli.Flag{
cli.StringFlag{
Name: "type, t",
Value: "",
Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'",
},
cli.StringFlag{
Name: "storage, s",
@ -80,34 +82,53 @@ var CmdMigrateStorage = cli.Command{
},
}
func migrateAttachments(dstStorage storage.ObjectStorage) error {
return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error {
func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
return db.IterateObjects(ctx, func(attach *repo_model.Attachment) error {
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
return err
})
}
func migrateLFS(dstStorage storage.ObjectStorage) error {
return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error {
func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
return db.IterateObjects(ctx, func(mo *git_model.LFSMetaObject) error {
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
return err
})
}
func migrateAvatars(dstStorage storage.ObjectStorage) error {
return user_model.IterateUser(func(user *user_model.User) error {
func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
return db.IterateObjects(ctx, func(user *user_model.User) error {
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
return err
})
}
func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
return repo_model.IterateRepository(func(repo *repo_model.Repository) error {
func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
return db.IterateObjects(ctx, func(repo *repo_model.Repository) error {
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
return err
})
}
func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error {
p, err := archiver.RelativePath()
if err != nil {
return err
}
_, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
return err
})
}
func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
return db.IterateObjects(ctx, func(pb *packages_model.PackageBlob) error {
p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
_, err := storage.Copy(dstStorage, p, storage.Packages, p)
return err
})
}
func runMigrateStorage(ctx *cli.Context) error {
stdCtx, cancel := installSignals()
defer cancel()
@ -127,8 +148,6 @@ func runMigrateStorage(ctx *cli.Context) error {
return err
}
goCtx := context.Background()
if err := storage.Init(); err != nil {
return err
}
@ -145,13 +164,13 @@ func runMigrateStorage(ctx *cli.Context) error {
return nil
}
dstStorage, err = storage.NewLocalStorage(
goCtx,
stdCtx,
storage.LocalStorageConfig{
Path: p,
})
case string(storage.MinioStorageType):
dstStorage, err = storage.NewMinioStorage(
goCtx,
stdCtx,
storage.MinioStorageConfig{
Endpoint: ctx.String("minio-endpoint"),
AccessKeyID: ctx.String("minio-access-key-id"),
@ -162,35 +181,29 @@ func runMigrateStorage(ctx *cli.Context) error {
UseSSL: ctx.Bool("minio-use-ssl"),
})
default:
return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage"))
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
}
if err != nil {
return err
}
migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
"attachments": migrateAttachments,
"lfs": migrateLFS,
"avatars": migrateAvatars,
"repo-avatars": migrateRepoAvatars,
"repo-archivers": migrateRepoArchivers,
"packages": migratePackages,
}
tp := strings.ToLower(ctx.String("type"))
switch tp {
case "attachments":
if err := migrateAttachments(dstStorage); err != nil {
return err
}
case "lfs":
if err := migrateLFS(dstStorage); err != nil {
return err
}
case "avatars":
if err := migrateAvatars(dstStorage); err != nil {
return err
}
case "repo-avatars":
if err := migrateRepoAvatars(dstStorage); err != nil {
if m, ok := migratedMethods[tp]; ok {
if err := m(stdCtx, dstStorage); err != nil {
return err
}
default:
return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
log.Info("%s files have successfully been copied to the new storage.", tp)
return nil
}
log.Warn("All files have been copied to the new placement but old files are still on the original placement.")
return nil
return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
}

74
cmd/migrate_storage_test.go
Unescape View File

@ -0,0 +1,74 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"context"
"os"
"strings"
"testing"
"code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/storage"
packages_service "code.gitea.io/gitea/services/packages"
"github.com/stretchr/testify/assert"
)
func TestMigratePackages(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
assert.NoError(t, err)
defer buf.Close()
v, f, err := packages_service.CreatePackageAndAddFile(&packages_service.PackageCreationInfo{
PackageInfo: packages_service.PackageInfo{
Owner: creator,
PackageType: packages.TypeGeneric,
Name: "test",
Version: "1.0.0",
},
Creator: creator,
SemverCompatible: true,
VersionProperties: map[string]string{},
}, &packages_service.PackageFileCreationInfo{
PackageFileInfo: packages_service.PackageFileInfo{
Filename: "a.go",
},
Data: buf,
IsLead: true,
})
assert.NoError(t, err)
assert.NotNil(t, v)
assert.NotNil(t, f)
ctx := context.Background()
p, err := os.MkdirTemp(os.TempDir(), "migrated_packages")
assert.NoError(t, err)
dstStorage, err := storage.NewLocalStorage(
ctx,
storage.LocalStorageConfig{
Path: p,
})
assert.NoError(t, err)
err = migratePackages(ctx, dstStorage)
assert.NoError(t, err)
entries, err := os.ReadDir(p)
assert.NoError(t, err)
assert.EqualValues(t, 2, len(entries))
assert.EqualValues(t, "01", entries[0].Name())
assert.EqualValues(t, "tmp", entries[1].Name())
}

34
models/db/iterate.go
Unescape View File

@ -0,0 +1,34 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package db
import (
"context"
"code.gitea.io/gitea/modules/setting"
)
// IterateObjects iterate all the Bean object
func IterateObjects[Object any](ctx context.Context, f func(repo *Object) error) error {
var start int
batchSize := setting.Database.IterateBufferSize
sess := GetEngine(ctx)
for {
repos := make([]*Object, 0, batchSize)
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
return err
}
if len(repos) == 0 {
return nil
}
start += len(repos)
for _, repo := range repos {
if err := f(repo); err != nil {
return err
}
}
}
}

23
models/git/lfs.go
Unescape View File

@ -278,29 +278,6 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
return committer.Commit()
}
// IterateLFS iterates lfs object
func IterateLFS(f func(mo *LFSMetaObject) error) error {
var start int
const batchSize = 100
e := db.GetEngine(db.DefaultContext)
for {
mos := make([]*LFSMetaObject, 0, batchSize)
if err := e.Limit(batchSize, start).Find(&mos); err != nil {
return err
}
if len(mos) == 0 {
return nil
}
start += len(mos)
for _, mo := range mos {
if err := f(mo); err != nil {
return err
}
}
}
}
// CopyLFS copies LFS data from one repo to another
func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
var lfsObjects []*LFSMetaObject

22
models/repo/attachment.go
Unescape View File

@ -226,28 +226,6 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
return err
}
// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
func IterateAttachment(f func(attach *Attachment) error) error {
var start int
const batchSize = 100
for {
attachments := make([]*Attachment, 0, batchSize)
if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&attachments); err != nil {
return err
}
if len(attachments) == 0 {
return nil
}
start += len(attachments)
for _, attach := range attachments {
if err := f(attach); err != nil {
return err
}
}
}
}
// CountOrphanedAttachments returns the number of bad attachments
func CountOrphanedAttachments() (int64, error) {
return db.GetEngine(db.DefaultContext).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))").

24
models/repo/repo_list.go
Unescape View File

@ -15,36 +15,12 @@ import (
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"xorm.io/builder"
)
// IterateRepository iterate repositories
func IterateRepository(f func(repo *Repository) error) error {
var start int
batchSize := setting.Database.IterateBufferSize
sess := db.GetEngine(db.DefaultContext)
for {
repos := make([]*Repository, 0, batchSize)
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
return err
}
if len(repos) == 0 {
return nil
}
start += len(repos)
for _, repo := range repos {
if err := f(repo); err != nil {
return err
}
}
}
}
// FindReposMapByIDs find repos as map
func FindReposMapByIDs(repoIDs []int64, res map[int64]*Repository) error {
return db.GetEngine(db.DefaultContext).In("id", repoIDs).Find(&res)

23
models/user/search.go
Unescape View File

@ -9,7 +9,6 @@ import (
"strings"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
@ -125,28 +124,6 @@ func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) {
return users, count, sessQuery.Find(&users)
}
// IterateUser iterate users
func IterateUser(f func(user *User) error) error {
var start int
batchSize := setting.Database.IterateBufferSize
for {
users := make([]*User, 0, batchSize)
if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&users); err != nil {
return err
}
if len(users) == 0 {
return nil
}
start += len(users)
for _, user := range users {
if err := f(user); err != nil {
return err
}
}
}
}
// BuildCanSeeUserCondition creates a condition which can be used to restrict results to users/orgs the actor can see
func BuildCanSeeUserCondition(actor *User) builder.Cond {
if actor != nil {

10
modules/packages/content_store.go
Unescape View File

@ -27,21 +27,21 @@ func NewContentStore() *ContentStore {
// Get gets a package blob
func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
return s.store.Open(keyToRelativePath(key))
return s.store.Open(KeyToRelativePath(key))
}
// Save stores a package blob
func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
_, err := s.store.Save(keyToRelativePath(key), r, size)
_, err := s.store.Save(KeyToRelativePath(key), r, size)
return err
}
// Delete deletes a package blob
func (s *ContentStore) Delete(key BlobHash256Key) error {
return s.store.Delete(keyToRelativePath(key))
return s.store.Delete(KeyToRelativePath(key))
}
// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
func keyToRelativePath(key BlobHash256Key) string {
// KeyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
func KeyToRelativePath(key BlobHash256Key) string {
return path.Join(string(key)[0:2], string(key)[2:4], string(key))
}

3
routers/private/mail.go
Unescape View File

@ -9,6 +9,7 @@ import (
"net/http"
"strconv"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/json"
@ -59,7 +60,7 @@ func SendEmail(ctx *context.PrivateContext) {
}
}
} else {
err := user_model.IterateUser(func(user *user_model.User) error {
err := db.IterateObjects(ctx, func(user *user_model.User) error {
if len(user.Email) > 0 && user.IsActive {
emails = append(emails, user.Email)
}

2
services/repository/avatar.go
Unescape View File

@ -96,7 +96,7 @@ func DeleteAvatar(repo *repo_model.Repository) error {
// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
func RemoveRandomAvatars(ctx context.Context) error {
return repo_model.IterateRepository(func(repository *repo_model.Repository) error {
return db.IterateObjects(ctx, func(repository *repo_model.Repository) error {
select {
case <-ctx.Done():
return db.ErrCancelledf("before random avatars removed for %s", repository.FullName())

Write Preview
Loading…
Cancel
Save