mirror of https://github.com/go-gitea/gitea
Refactor "dump" sub-command (#30240)
Major changes: * Move some functions like "addReader" / "isSubDir" / "addRecursiveExclude" to a separate package, and add tests * Clarify the filename&dump type logic and add tests * Clarify the logger behavior and remove FIXME comments Co-authored-by: Giteabot <teabot@gitea.io>pull/30201/head^2
wxiaoguang
8 months ago
committed by
GitHub
parent
b28d3a4218
commit
654cfd1dfb
@ -0,0 +1,174 @@ |
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package dump |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"os" |
||||
"path" |
||||
"path/filepath" |
||||
"slices" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/timeutil" |
||||
|
||||
"github.com/mholt/archiver/v3" |
||||
) |
||||
|
||||
var SupportedOutputTypes = []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"} |
||||
|
||||
// PrepareFileNameAndType prepares the output file name and type, if the type is not supported, it returns an empty "outType"
|
||||
func PrepareFileNameAndType(argFile, argType string) (outFileName, outType string) { |
||||
if argFile == "" && argType == "" { |
||||
outType = SupportedOutputTypes[0] |
||||
outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType) |
||||
} else if argFile == "" { |
||||
outType = argType |
||||
outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType) |
||||
} else if argType == "" { |
||||
if filepath.Ext(outFileName) == "" { |
||||
outType = SupportedOutputTypes[0] |
||||
outFileName = argFile |
||||
} else { |
||||
for _, t := range SupportedOutputTypes { |
||||
if strings.HasSuffix(argFile, "."+t) { |
||||
outFileName = argFile |
||||
outType = t |
||||
} |
||||
} |
||||
} |
||||
} else { |
||||
outFileName, outType = argFile, argType |
||||
} |
||||
if !slices.Contains(SupportedOutputTypes, outType) { |
||||
return "", "" |
||||
} |
||||
return outFileName, outType |
||||
} |
||||
|
||||
func IsSubdir(upper, lower string) (bool, error) { |
||||
if relPath, err := filepath.Rel(upper, lower); err != nil { |
||||
return false, err |
||||
} else if relPath == "." || !strings.HasPrefix(relPath, ".") { |
||||
return true, nil |
||||
} |
||||
return false, nil |
||||
} |
||||
|
||||
type Dumper struct { |
||||
Writer archiver.Writer |
||||
Verbose bool |
||||
|
||||
globalExcludeAbsPaths []string |
||||
} |
||||
|
||||
func (dumper *Dumper) AddReader(r io.ReadCloser, info os.FileInfo, customName string) error { |
||||
if dumper.Verbose { |
||||
log.Info("Adding file %s", customName) |
||||
} |
||||
|
||||
return dumper.Writer.Write(archiver.File{ |
||||
FileInfo: archiver.FileInfo{ |
||||
FileInfo: info, |
||||
CustomName: customName, |
||||
}, |
||||
ReadCloser: r, |
||||
}) |
||||
} |
||||
|
||||
func (dumper *Dumper) AddFile(filePath, absPath string) error { |
||||
file, err := os.Open(absPath) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer file.Close() |
||||
fileInfo, err := file.Stat() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return dumper.AddReader(file, fileInfo, filePath) |
||||
} |
||||
|
||||
func (dumper *Dumper) normalizeFilePath(absPath string) string { |
||||
absPath = filepath.Clean(absPath) |
||||
if setting.IsWindows { |
||||
absPath = strings.ToLower(absPath) |
||||
} |
||||
return absPath |
||||
} |
||||
|
||||
func (dumper *Dumper) GlobalExcludeAbsPath(absPaths ...string) { |
||||
for _, absPath := range absPaths { |
||||
dumper.globalExcludeAbsPaths = append(dumper.globalExcludeAbsPaths, dumper.normalizeFilePath(absPath)) |
||||
} |
||||
} |
||||
|
||||
func (dumper *Dumper) shouldExclude(absPath string, excludes []string) bool { |
||||
norm := dumper.normalizeFilePath(absPath) |
||||
return slices.Contains(dumper.globalExcludeAbsPaths, norm) || slices.Contains(excludes, norm) |
||||
} |
||||
|
||||
func (dumper *Dumper) AddRecursiveExclude(insidePath, absPath string, excludes []string) error { |
||||
excludes = slices.Clone(excludes) |
||||
for i := range excludes { |
||||
excludes[i] = dumper.normalizeFilePath(excludes[i]) |
||||
} |
||||
return dumper.addFileOrDir(insidePath, absPath, excludes) |
||||
} |
||||
|
||||
func (dumper *Dumper) addFileOrDir(insidePath, absPath string, excludes []string) error { |
||||
absPath, err := filepath.Abs(absPath) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
dir, err := os.Open(absPath) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer dir.Close() |
||||
|
||||
files, err := dir.Readdir(0) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
for _, file := range files { |
||||
currentAbsPath := filepath.Join(absPath, file.Name()) |
||||
if dumper.shouldExclude(currentAbsPath, excludes) { |
||||
continue |
||||
} |
||||
|
||||
currentInsidePath := path.Join(insidePath, file.Name()) |
||||
if file.IsDir() { |
||||
if err := dumper.AddFile(currentInsidePath, currentAbsPath); err != nil { |
||||
return err |
||||
} |
||||
if err = dumper.addFileOrDir(currentInsidePath, currentAbsPath, excludes); err != nil { |
||||
return err |
||||
} |
||||
} else { |
||||
// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
|
||||
shouldAdd := file.Mode().IsRegular() |
||||
if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink { |
||||
target, err := filepath.EvalSymlinks(currentAbsPath) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
targetStat, err := os.Stat(target) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
shouldAdd = targetStat.Mode().IsRegular() |
||||
} |
||||
if shouldAdd { |
||||
if err = dumper.AddFile(currentInsidePath, currentAbsPath); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,113 @@ |
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package dump |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"os" |
||||
"path/filepath" |
||||
"sort" |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/timeutil" |
||||
|
||||
"github.com/mholt/archiver/v3" |
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestPrepareFileNameAndType(t *testing.T) { |
||||
defer timeutil.MockSet(time.Unix(1234, 0))() |
||||
test := func(argFile, argType, expFile, expType string) { |
||||
outFile, outType := PrepareFileNameAndType(argFile, argType) |
||||
assert.Equal(t, |
||||
fmt.Sprintf("outFile=%s, outType=%s", expFile, expType), |
||||
fmt.Sprintf("outFile=%s, outType=%s", outFile, outType), |
||||
fmt.Sprintf("argFile=%s, argType=%s", argFile, argType), |
||||
) |
||||
} |
||||
|
||||
test("", "", "gitea-dump-1234.zip", "zip") |
||||
test("", "tar.gz", "gitea-dump-1234.tar.gz", "tar.gz") |
||||
test("", "no-such", "", "") |
||||
|
||||
test("-", "", "-", "zip") |
||||
test("-", "tar.gz", "-", "tar.gz") |
||||
test("-", "no-such", "", "") |
||||
|
||||
test("a", "", "a", "zip") |
||||
test("a", "tar.gz", "a", "tar.gz") |
||||
test("a", "no-such", "", "") |
||||
|
||||
test("a.zip", "", "a.zip", "zip") |
||||
test("a.zip", "tar.gz", "a.zip", "tar.gz") |
||||
test("a.zip", "no-such", "", "") |
||||
|
||||
test("a.tar.gz", "", "a.tar.gz", "zip") |
||||
test("a.tar.gz", "tar.gz", "a.tar.gz", "tar.gz") |
||||
test("a.tar.gz", "no-such", "", "") |
||||
} |
||||
|
||||
func TestIsSubDir(t *testing.T) { |
||||
tmpDir := t.TempDir() |
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755) |
||||
|
||||
isSub, err := IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include")) |
||||
assert.NoError(t, err) |
||||
assert.True(t, isSub) |
||||
|
||||
isSub, err = IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include/sub")) |
||||
assert.NoError(t, err) |
||||
assert.True(t, isSub) |
||||
|
||||
isSub, err = IsSubdir(filepath.Join(tmpDir, "include/sub"), filepath.Join(tmpDir, "include")) |
||||
assert.NoError(t, err) |
||||
assert.False(t, isSub) |
||||
} |
||||
|
||||
type testWriter struct { |
||||
added []string |
||||
} |
||||
|
||||
func (t *testWriter) Create(out io.Writer) error { |
||||
return nil |
||||
} |
||||
|
||||
func (t *testWriter) Write(f archiver.File) error { |
||||
t.added = append(t.added, f.Name()) |
||||
return nil |
||||
} |
||||
|
||||
func (t *testWriter) Close() error { |
||||
return nil |
||||
} |
||||
|
||||
func TestDumper(t *testing.T) { |
||||
sortStrings := func(s []string) []string { |
||||
sort.Strings(s) |
||||
return s |
||||
} |
||||
tmpDir := t.TempDir() |
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude1"), 0o755) |
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude2"), 0o755) |
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755) |
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/a"), nil, 0o644) |
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/sub/b"), nil, 0o644) |
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude1/a-1"), nil, 0o644) |
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude2/a-2"), nil, 0o644) |
||||
|
||||
tw := &testWriter{} |
||||
d := &Dumper{Writer: tw} |
||||
d.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1")) |
||||
err := d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")}) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added)) |
||||
|
||||
tw = &testWriter{} |
||||
d = &Dumper{Writer: tw} |
||||
err = d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added)) |
||||
} |
||||
Loading…
Reference in new issue