mirror of https://github.com/go-gitea/gitea
Rework repository archive (#14723)
* Use storage to store archive files * Fix backend lint * Add archiver table on database * Finish archive download * Fix test * Add database migrations * Add status for archiver * Fix lint * Add queue * Add doctor to check and delete old archives * Improve archive queue * Fix tests * improve archive storage * Delete repo archives * Add missing fixture * fix fixture * Fix fixture * Fix test * Fix archiver cleaning * Fix bug * Add docs for repository archive storage * remove repo-archive configuration * Fix test * Fix test * Fix lint Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: techknowlogick <techknowlogick@gitea.io>pull/16238/head
Lunny Xiao
3 years ago
committed by
GitHub
parent
c9c7afda1a
commit
b223d36195
@ -0,0 +1 @@ |
||||
aacbdfe9e1c4b47f60abe81849045fa4e96f1d75 |
||||
@ -0,0 +1 @@ |
||||
[] # empty |
||||
@ -0,0 +1,22 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"xorm.io/xorm" |
||||
) |
||||
|
||||
func addRepoArchiver(x *xorm.Engine) error { |
||||
// RepoArchiver represents all archivers
|
||||
type RepoArchiver struct { |
||||
ID int64 `xorm:"pk autoincr"` |
||||
RepoID int64 `xorm:"index unique(s)"` |
||||
Type int `xorm:"unique(s)"` |
||||
Status int |
||||
CommitID string `xorm:"VARCHAR(40) unique(s)"` |
||||
CreatedUnix int64 `xorm:"INDEX NOT NULL created"` |
||||
} |
||||
return x.Sync2(new(RepoArchiver)) |
||||
} |
||||
@ -0,0 +1,86 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package models |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/timeutil" |
||||
) |
||||
|
||||
// RepoArchiverStatus represents repo archive status
|
||||
type RepoArchiverStatus int |
||||
|
||||
// enumerate all repo archive statuses
|
||||
const ( |
||||
RepoArchiverGenerating = iota // the archiver is generating
|
||||
RepoArchiverReady // it's ready
|
||||
) |
||||
|
||||
// RepoArchiver represents all archivers
|
||||
type RepoArchiver struct { |
||||
ID int64 `xorm:"pk autoincr"` |
||||
RepoID int64 `xorm:"index unique(s)"` |
||||
Repo *Repository `xorm:"-"` |
||||
Type git.ArchiveType `xorm:"unique(s)"` |
||||
Status RepoArchiverStatus |
||||
CommitID string `xorm:"VARCHAR(40) unique(s)"` |
||||
CreatedUnix timeutil.TimeStamp `xorm:"INDEX NOT NULL created"` |
||||
} |
||||
|
||||
// LoadRepo loads repository
|
||||
func (archiver *RepoArchiver) LoadRepo() (*Repository, error) { |
||||
if archiver.Repo != nil { |
||||
return archiver.Repo, nil |
||||
} |
||||
|
||||
var repo Repository |
||||
has, err := x.ID(archiver.RepoID).Get(&repo) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if !has { |
||||
return nil, ErrRepoNotExist{ |
||||
ID: archiver.RepoID, |
||||
} |
||||
} |
||||
return &repo, nil |
||||
} |
||||
|
||||
// RelativePath returns relative path
|
||||
func (archiver *RepoArchiver) RelativePath() (string, error) { |
||||
repo, err := archiver.LoadRepo() |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
|
||||
return fmt.Sprintf("%s/%s/%s.%s", repo.FullName(), archiver.CommitID[:2], archiver.CommitID, archiver.Type.String()), nil |
||||
} |
||||
|
||||
// GetRepoArchiver get an archiver
|
||||
func GetRepoArchiver(ctx DBContext, repoID int64, tp git.ArchiveType, commitID string) (*RepoArchiver, error) { |
||||
var archiver RepoArchiver |
||||
has, err := ctx.e.Where("repo_id=?", repoID).And("`type`=?", tp).And("commit_id=?", commitID).Get(&archiver) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if has { |
||||
return &archiver, nil |
||||
} |
||||
return nil, nil |
||||
} |
||||
|
||||
// AddRepoArchiver adds an archiver
|
||||
func AddRepoArchiver(ctx DBContext, archiver *RepoArchiver) error { |
||||
_, err := ctx.e.Insert(archiver) |
||||
return err |
||||
} |
||||
|
||||
// UpdateRepoArchiverStatus updates archiver's status
|
||||
func UpdateRepoArchiverStatus(ctx DBContext, archiver *RepoArchiver) error { |
||||
_, err := ctx.e.ID(archiver.ID).Cols("status").Update(archiver) |
||||
return err |
||||
} |
||||
@ -0,0 +1,59 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package doctor |
||||
|
||||
import ( |
||||
"os" |
||||
"path/filepath" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/util" |
||||
) |
||||
|
||||
func checkOldArchives(logger log.Logger, autofix bool) error { |
||||
numRepos := 0 |
||||
numReposUpdated := 0 |
||||
err := iterateRepositories(func(repo *models.Repository) error { |
||||
if repo.IsEmpty { |
||||
return nil |
||||
} |
||||
|
||||
p := filepath.Join(repo.RepoPath(), "archives") |
||||
isDir, err := util.IsDir(p) |
||||
if err != nil { |
||||
log.Warn("check if %s is directory failed: %v", p, err) |
||||
} |
||||
if isDir { |
||||
numRepos++ |
||||
if autofix { |
||||
if err := os.RemoveAll(p); err == nil { |
||||
numReposUpdated++ |
||||
} else { |
||||
log.Warn("remove %s failed: %v", p, err) |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
|
||||
if autofix { |
||||
logger.Info("%d / %d old archives in repository deleted", numReposUpdated, numRepos) |
||||
} else { |
||||
logger.Info("%d old archives in repository need to be deleted", numRepos) |
||||
} |
||||
|
||||
return err |
||||
} |
||||
|
||||
func init() { |
||||
Register(&Check{ |
||||
Title: "Check old archives", |
||||
Name: "check-old-archives", |
||||
IsDefault: false, |
||||
Run: checkOldArchives, |
||||
Priority: 7, |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue