mirror of https://github.com/go-gitea/gitea
Add doctor command for full GC of LFS (#21978)
The recent PR adding orphaned checks to the LFS storage is not sufficient to completely GC LFS, as it is possible for LFSMetaObjects to remain associated with repos but still need to be garbage collected. Imagine a situation where a branch is uploaded containing LFS files but that branch is later completely deleted. The LFSMetaObjects will remain associated with the Repository but the Repository will no longer contain any pointers to the object. This PR adds a second doctor command to perform a full GC. Signed-off-by: Andrew Thornton <art27@cantab.net>pull/22034/head^2
zeripath
2 years ago
committed by
GitHub
parent
3243dbe1a9
commit
651fe4bb7d
@ -0,0 +1,37 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package doctor |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/services/repository" |
||||
) |
||||
|
||||
func init() { |
||||
Register(&Check{ |
||||
Title: "Garbage collect LFS", |
||||
Name: "gc-lfs", |
||||
IsDefault: false, |
||||
Run: garbageCollectLFSCheck, |
||||
AbortIfFailed: false, |
||||
SkipDatabaseInitialization: false, |
||||
Priority: 1, |
||||
}) |
||||
} |
||||
|
||||
func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool) error { |
||||
if !setting.LFS.StartServer { |
||||
return fmt.Errorf("LFS support is disabled") |
||||
} |
||||
|
||||
if err := repository.GarbageCollectLFSMetaObjects(ctx, logger, autofix); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return checkStorage(&checkStorageOptions{LFS: true})(ctx, logger, autofix) |
||||
} |
||||
@ -0,0 +1,105 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package repository |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models/db" |
||||
git_model "code.gitea.io/gitea/models/git" |
||||
repo_model "code.gitea.io/gitea/models/repo" |
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/lfs" |
||||
"code.gitea.io/gitea/modules/log" |
||||
|
||||
"xorm.io/builder" |
||||
) |
||||
|
||||
func GarbageCollectLFSMetaObjects(ctx context.Context, logger log.Logger, autofix bool) error { |
||||
log.Trace("Doing: GarbageCollectLFSMetaObjects") |
||||
|
||||
if err := db.Iterate( |
||||
ctx, |
||||
builder.And(builder.Gt{"id": 0}), |
||||
func(ctx context.Context, repo *repo_model.Repository) error { |
||||
return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, logger, autofix) |
||||
}, |
||||
); err != nil { |
||||
return err |
||||
} |
||||
|
||||
log.Trace("Finished: GarbageCollectLFSMetaObjects") |
||||
return nil |
||||
} |
||||
|
||||
func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, logger log.Logger, autofix bool) error { |
||||
if logger != nil { |
||||
logger.Info("Checking %-v", repo) |
||||
} |
||||
total, orphaned, collected, deleted := 0, 0, 0, 0 |
||||
if logger != nil { |
||||
defer func() { |
||||
if orphaned == 0 { |
||||
logger.Info("Found %d total LFSMetaObjects in %-v", total, repo) |
||||
} else if !autofix { |
||||
logger.Info("Found %d/%d orphaned LFSMetaObjects in %-v", orphaned, total, repo) |
||||
} else { |
||||
logger.Info("Collected %d/%d orphaned/%d total LFSMetaObjects in %-v. %d removed from storage.", collected, orphaned, total, repo, deleted) |
||||
} |
||||
}() |
||||
} |
||||
|
||||
gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) |
||||
if err != nil { |
||||
log.Error("Unable to open git repository %-v: %v", repo, err) |
||||
return err |
||||
} |
||||
defer gitRepo.Close() |
||||
|
||||
store := lfs.NewContentStore() |
||||
|
||||
return git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject, count int64) error { |
||||
total++ |
||||
pointerSha := git.ComputeBlobHash([]byte(metaObject.Pointer.StringContent())) |
||||
|
||||
if gitRepo.IsObjectExist(pointerSha.String()) { |
||||
return nil |
||||
} |
||||
orphaned++ |
||||
|
||||
if !autofix { |
||||
return nil |
||||
} |
||||
// Non-existent pointer file
|
||||
_, err = git_model.RemoveLFSMetaObjectByOidFn(repo.ID, metaObject.Oid, func(count int64) error { |
||||
if count > 0 { |
||||
return nil |
||||
} |
||||
|
||||
if err := store.Delete(metaObject.RelativePath()); err != nil { |
||||
log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err) |
||||
} |
||||
deleted++ |
||||
return nil |
||||
}) |
||||
if err != nil { |
||||
return fmt.Errorf("unable to remove meta-object %s in %s: %w", metaObject.Oid, repo.FullName(), err) |
||||
} |
||||
collected++ |
||||
|
||||
return nil |
||||
}, &git_model.IterateLFSMetaObjectsForRepoOptions{ |
||||
// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
|
||||
// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
|
||||
// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
|
||||
// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
|
||||
// objects.
|
||||
//
|
||||
// It is likely that a week is potentially excessive but it should definitely be enough that any
|
||||
// unassociated LFS object is genuinely unassociated.
|
||||
OlderThan: time.Now().Add(-24 * 7 * time.Hour), |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue