diff --git a/cmd/manager_logging.go b/cmd/manager_logging.go index eb311d28926..0043ea1e52a 100644 --- a/cmd/manager_logging.go +++ b/cmd/manager_logging.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" + "github.com/urfave/cli" ) diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index ceeba4de3b5..42ccf88af83 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -112,7 +112,7 @@ func runPR() { unittest.LoadFixtures() util.RemoveAll(setting.RepoRootPath) util.RemoveAll(models.LocalCopyPath()) - util.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath) + unittest.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath) log.Printf("[PR] Setting up router\n") // routers.GlobalInit() diff --git a/go.mod b/go.mod index 0378ccd5e7b..da2f1e03dd8 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,6 @@ require ( github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 - github.com/unknwon/com v1.0.1 github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361 github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae github.com/unrolled/render v1.4.1 @@ -251,6 +250,7 @@ require ( github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/toqueteos/webbrowser v1.2.0 // indirect github.com/ulikunitz/xz v0.5.10 // indirect + github.com/unknwon/com v1.0.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.1 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect diff --git a/integrations/integration_test.go b/integrations/integration_test.go index c778fb80134..9e0445cae73 100644 --- a/integrations/integration_test.go +++ b/integrations/integration_test.go @@ -254,7 +254,7 @@ func prepareTestEnv(t testing.TB, skip ...int) func() { assert.NoError(t, unittest.LoadFixtures()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) @@ -550,7 +550,7 @@ func resetFixtures(t *testing.T) { assert.NoError(t, queue.GetManager().FlushAll(context.Background(), -1)) assert.NoError(t, unittest.LoadFixtures()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index 0518dd11798..6e55807c271 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/integrations" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" @@ -60,7 +61,7 @@ func initMigrationTest(t *testing.T) func() { assert.True(t, len(setting.RepoRootPath) != 0) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) diff --git a/models/migrations/migrations_test.go b/models/migrations/migrations_test.go index a17eba54e82..a1fd49a8b9f 100644 --- a/models/migrations/migrations_test.go +++ b/models/migrations/migrations_test.go @@ -203,7 +203,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En deferFn := PrintCurrentTest(t, ourSkip) assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { diff --git a/models/unittest/fscopy.go b/models/unittest/fscopy.go new file mode 100644 index 00000000000..ff815e729d4 --- /dev/null +++ b/models/unittest/fscopy.go @@ -0,0 +1,103 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package unittest + +import ( + "errors" + "io" + "os" + "path" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +// Copy copies file from source to target path. +func Copy(src, dest string) error { + // Gather file information to set back later. + si, err := os.Lstat(src) + if err != nil { + return err + } + + // Handle symbolic link. + if si.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(src) + if err != nil { + return err + } + // NOTE: os.Chmod and os.Chtimes don't recognize symbolic link, + // which will lead "no such file or directory" error. + return os.Symlink(target, dest) + } + + sr, err := os.Open(src) + if err != nil { + return err + } + defer sr.Close() + + dw, err := os.Create(dest) + if err != nil { + return err + } + defer dw.Close() + + if _, err = io.Copy(dw, sr); err != nil { + return err + } + + // Set back file information. + if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil { + return err + } + return os.Chmod(dest, si.Mode()) +} + +// CopyDir copy files recursively from source to target directory. +// +// The filter accepts a function that process the path info. +// and should return true for need to filter. +// +// It returns error when error occurs in underlying functions. +func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error { + // Check if target directory exists. + if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) { + return errors.New("file or directory already exists: " + destPath) + } + + err := os.MkdirAll(destPath, os.ModePerm) + if err != nil { + return err + } + + // Gather directory info. + infos, err := util.StatDir(srcPath, true) + if err != nil { + return err + } + + var filter func(filePath string) bool + if len(filters) > 0 { + filter = filters[0] + } + + for _, info := range infos { + if filter != nil && filter(info) { + continue + } + + curPath := path.Join(destPath, info) + if strings.HasSuffix(info, "/") { + err = os.MkdirAll(curPath, os.ModePerm) + } else { + err = Copy(path.Join(srcPath, info), curPath) + } + if err != nil { + return err + } + } + return nil +} diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index b6469fb309c..b6924d47068 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -104,7 +104,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string, fixtureFiles ...string) { if err = util.RemoveAll(repoRootPath); err != nil { fatalTestError("util.RemoveAll: %v\n", err) } - if err = util.CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { + if err = CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { fatalTestError("util.CopyDir: %v\n", err) } @@ -175,7 +175,7 @@ func PrepareTestEnv(t testing.TB) { assert.NoError(t, PrepareTestDatabase()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) metaPath := filepath.Join(giteaRoot, "integrations", "gitea-repositories-meta") - assert.NoError(t, util.CopyDir(metaPath, setting.RepoRootPath)) + assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) assert.NoError(t, err) diff --git a/modules/cache/cache_redis.go b/modules/cache/cache_redis.go index e4b9a70f63f..ff6c8d424c1 100644 --- a/modules/cache/cache_redis.go +++ b/modules/cache/cache_redis.go @@ -6,11 +6,11 @@ package cache import ( "fmt" + "strconv" "time" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/nosql" - "code.gitea.io/gitea/modules/util" "gitea.com/go-chi/cache" "github.com/go-redis/redis/v8" @@ -24,20 +24,37 @@ type RedisCacher struct { occupyMode bool } -// Put puts value into cache with key and expire time. +// toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally +func toStr(v interface{}) string { + if v == nil { + return "" + } + switch v := v.(type) { + case string: + return v + case []byte: + return string(v) + case int: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + default: + return fmt.Sprint(v) // as what the old com.ToStr does in most cases + } +} + +// Put puts value (string type) into cache with key and expire time. // If expired is 0, it lives forever. func (c *RedisCacher) Put(key string, val interface{}, expire int64) error { + // this function is not well-designed, it only puts string values into cache key = c.prefix + key if expire == 0 { - if err := c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), 0).Err(); err != nil { + if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil { return err } } else { - dur, err := time.ParseDuration(util.ToStr(expire) + "s") - if err != nil { - return err - } - if err = c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), dur).Err(); err != nil { + dur := time.Duration(expire) * time.Second + if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil { return err } } diff --git a/modules/context/csrf.go b/modules/context/csrf.go index 1fb992e2ae4..4fc92705048 100644 --- a/modules/context/csrf.go +++ b/modules/context/csrf.go @@ -22,8 +22,10 @@ import ( "encoding/base32" "fmt" "net/http" + "strconv" "time" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" @@ -215,9 +217,16 @@ func Csrfer(opt CsrfOptions, ctx *Context) CSRF { } x.ID = "0" - uid := ctx.Session.Get(opt.SessionKey) - if uid != nil { - x.ID = util.ToStr(uid) + uidAny := ctx.Session.Get(opt.SessionKey) + if uidAny != nil { + switch uidVal := uidAny.(type) { + case string: + x.ID = uidVal + case int64: + x.ID = strconv.FormatInt(uidVal, 10) + default: + log.Error("invalid uid type in session: %T", uidAny) + } } needsNew := false diff --git a/modules/nosql/manager.go b/modules/nosql/manager.go index dab30812ce7..93338fdc3f9 100644 --- a/modules/nosql/manager.go +++ b/modules/nosql/manager.go @@ -11,6 +11,7 @@ import ( "time" "code.gitea.io/gitea/modules/process" + "github.com/go-redis/redis/v8" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/modules/sync/unique_queue.go b/modules/sync/unique_queue.go index 414cc50f39a..df115d7c96c 100644 --- a/modules/sync/unique_queue.go +++ b/modules/sync/unique_queue.go @@ -5,8 +5,6 @@ package sync -import "code.gitea.io/gitea/modules/util" - // UniqueQueue is a queue which guarantees only one instance of same // identity is in the line. Instances with same identity will be // discarded if there is already one in the line. @@ -53,10 +51,10 @@ func (q *UniqueQueue) IsClosed() <-chan struct{} { } // IDs returns the current ids in the pool -func (q *UniqueQueue) IDs() []interface{} { +func (q *UniqueQueue) IDs() []string { q.table.lock.Lock() defer q.table.lock.Unlock() - ids := make([]interface{}, 0, len(q.table.pool)) + ids := make([]string, 0, len(q.table.pool)) for id := range q.table.pool { ids = append(ids, id) } @@ -70,20 +68,19 @@ func (q *UniqueQueue) Queue() <-chan string { // Exist returns true if there is an instance with given identity // exists in the queue. -func (q *UniqueQueue) Exist(id interface{}) bool { - return q.table.IsRunning(util.ToStr(id)) +func (q *UniqueQueue) Exist(id string) bool { + return q.table.IsRunning(id) } // AddFunc adds new instance to the queue with a custom runnable function, // the queue is blocked until the function exits. -func (q *UniqueQueue) AddFunc(id interface{}, fn func()) { - idStr := util.ToStr(id) +func (q *UniqueQueue) AddFunc(id string, fn func()) { q.table.lock.Lock() - if _, ok := q.table.pool[idStr]; ok { + if _, ok := q.table.pool[id]; ok { q.table.lock.Unlock() return } - q.table.pool[idStr] = struct{}{} + q.table.pool[id] = struct{}{} if fn != nil { fn() } @@ -91,17 +88,17 @@ func (q *UniqueQueue) AddFunc(id interface{}, fn func()) { select { case <-q.closed: return - case q.queue <- idStr: + case q.queue <- id: return } } // Add adds new instance to the queue. -func (q *UniqueQueue) Add(id interface{}) { +func (q *UniqueQueue) Add(id string) { q.AddFunc(id, nil) } // Remove removes instance from the queue. -func (q *UniqueQueue) Remove(id interface{}) { - q.table.Stop(util.ToStr(id)) +func (q *UniqueQueue) Remove(id string) { + q.table.Stop(id) } diff --git a/modules/util/legacy.go b/modules/util/legacy.go index c7da5415349..d319faad098 100644 --- a/modules/util/legacy.go +++ b/modules/util/legacy.go @@ -9,29 +9,37 @@ import ( "crypto/cipher" "crypto/rand" "errors" - - "github.com/unknwon/com" //nolint:depguard + "io" + "os" ) // CopyFile copies file from source to target path. func CopyFile(src, dest string) error { - return com.Copy(src, dest) -} + si, err := os.Lstat(src) + if err != nil { + return err + } -// CopyDir copy files recursively from source to target directory. -// It returns error when error occurs in underlying functions. -func CopyDir(srcPath, destPath string) error { - return com.CopyDir(srcPath, destPath) -} + sr, err := os.Open(src) + if err != nil { + return err + } + defer sr.Close() -// ToStr converts any interface to string. should be replaced. -func ToStr(value interface{}, args ...int) string { - return com.ToStr(value, args...) -} + dw, err := os.Create(dest) + if err != nil { + return err + } + defer dw.Close() -// ToSnakeCase converts a string to snake_case. should be replaced. -func ToSnakeCase(str string) string { - return com.ToSnakeCase(str) + if _, err = io.Copy(dw, sr); err != nil { + return err + } + + if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil { + return err + } + return os.Chmod(dest, si.Mode()) } // AESGCMEncrypt (from legacy package): encrypts plaintext with the given key using AES in GCM mode. should be replaced. diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go index cfda93d3ad3..c41f7a008c2 100644 --- a/modules/util/legacy_test.go +++ b/modules/util/legacy_test.go @@ -7,12 +7,38 @@ package util import ( "crypto/aes" "crypto/rand" + "fmt" + "os" "testing" + "time" "github.com/stretchr/testify/assert" - "github.com/unknwon/com" //nolint:depguard ) +func TestCopyFile(t *testing.T) { + testContent := []byte("hello") + + tmpDir := os.TempDir() + now := time.Now() + srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro()) + dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro()) + + _ = os.Remove(srcFile) + _ = os.Remove(dstFile) + defer func() { + _ = os.Remove(srcFile) + _ = os.Remove(dstFile) + }() + + err := os.WriteFile(srcFile, testContent, 0o777) + assert.NoError(t, err) + err = CopyFile(srcFile, dstFile) + assert.NoError(t, err) + dstContent, err := os.ReadFile(dstFile) + assert.NoError(t, err) + assert.Equal(t, testContent, dstContent) +} + func TestAESGCM(t *testing.T) { t.Parallel() @@ -29,9 +55,4 @@ func TestAESGCM(t *testing.T) { assert.NoError(t, err) assert.Equal(t, plaintext, decrypted) - - // at the moment, we make sure the result is the same as the legacy package, this assertion can be removed in next round refactoring - legacy, err := com.AESGCMDecrypt(key, ciphertext) - assert.NoError(t, err) - assert.Equal(t, legacy, plaintext) } diff --git a/modules/util/string.go b/modules/util/string.go new file mode 100644 index 00000000000..4301f75f99c --- /dev/null +++ b/modules/util/string.go @@ -0,0 +1,88 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import "github.com/yuin/goldmark/util" + +func isSnakeCaseUpper(c byte) bool { + return 'A' <= c && c <= 'Z' +} + +func isSnakeCaseLowerOrNumber(c byte) bool { + return 'a' <= c && c <= 'z' || '0' <= c && c <= '9' +} + +// ToSnakeCase convert the input string to snake_case format. +// +// Some samples. +// "FirstName" => "first_name" +// "HTTPServer" => "http_server" +// "NoHTTPS" => "no_https" +// "GO_PATH" => "go_path" +// "GO PATH" => "go_path" // space is converted to underscore. +// "GO-PATH" => "go_path" // hyphen is converted to underscore. +// +func ToSnakeCase(input string) string { + if len(input) == 0 { + return "" + } + + var res []byte + if len(input) == 1 { + c := input[0] + if isSnakeCaseUpper(c) { + res = []byte{c + 'a' - 'A'} + } else if isSnakeCaseLowerOrNumber(c) { + res = []byte{c} + } else { + res = []byte{'_'} + } + } else { + res = make([]byte, 0, len(input)*4/3) + pos := 0 + needSep := false + for pos < len(input) { + c := input[pos] + if c >= 0x80 { + res = append(res, c) + pos++ + continue + } + isUpper := isSnakeCaseUpper(c) + if isUpper || isSnakeCaseLowerOrNumber(c) { + end := pos + 1 + if isUpper { + // skip the following upper letters + for end < len(input) && isSnakeCaseUpper(input[end]) { + end++ + } + if end-pos > 1 && end < len(input) && isSnakeCaseLowerOrNumber(input[end]) { + end-- + } + } + // skip the following lower or number letters + for end < len(input) && (isSnakeCaseLowerOrNumber(input[end]) || input[end] >= 0x80) { + end++ + } + if needSep { + res = append(res, '_') + } + res = append(res, input[pos:end]...) + pos = end + needSep = true + } else { + res = append(res, '_') + pos++ + needSep = false + } + } + for i := 0; i < len(res); i++ { + if isSnakeCaseUpper(res[i]) { + res[i] += 'a' - 'A' + } + } + } + return util.BytesToReadOnlyString(res) +} diff --git a/modules/util/string_test.go b/modules/util/string_test.go new file mode 100644 index 00000000000..49de29ab67a --- /dev/null +++ b/modules/util/string_test.go @@ -0,0 +1,48 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToSnakeCase(t *testing.T) { + cases := map[string]string{ + // all old cases from the legacy package + "HTTPServer": "http_server", + "_camelCase": "_camel_case", + "NoHTTPS": "no_https", + "Wi_thF": "wi_th_f", + "_AnotherTES_TCaseP": "_another_tes_t_case_p", + "ALL": "all", + "_HELLO_WORLD_": "_hello_world_", + "HELLO_WORLD": "hello_world", + "HELLO____WORLD": "hello____world", + "TW": "tw", + "_C": "_c", + + " sentence case ": "__sentence_case__", + " Mixed-hyphen case _and SENTENCE_case and UPPER-case": "_mixed_hyphen_case__and_sentence_case_and_upper_case", + + // new cases + " ": "_", + "A": "a", + "A0": "a0", + "a0": "a0", + "Aa0": "aa0", + "啊": "啊", + "A啊": "a啊", + "Aa啊b": "aa啊b", + "A啊B": "a啊_b", + "Aa啊B": "aa啊_b", + "TheCase2": "the_case2", + "ObjIDs": "obj_i_ds", // the strange database column name which already exists + } + for input, expected := range cases { + assert.Equal(t, expected, ToSnakeCase(input)) + } +} diff --git a/services/webhook/webhook.go b/services/webhook/webhook.go index 557dd147bfe..a3efc7535fc 100644 --- a/services/webhook/webhook.go +++ b/services/webhook/webhook.go @@ -6,6 +6,7 @@ package webhook import ( "fmt" + "strconv" "strings" repo_model "code.gitea.io/gitea/models/repo" @@ -106,7 +107,7 @@ func PrepareWebhook(w *webhook_model.Webhook, repo *repo_model.Repository, event return err } - go hookQueue.Add(repo.ID) + go hookQueue.Add(strconv.FormatInt(repo.ID, 10)) return nil } @@ -187,7 +188,7 @@ func PrepareWebhooks(repo *repo_model.Repository, event webhook_model.HookEventT return err } - go hookQueue.Add(repo.ID) + go hookQueue.Add(strconv.FormatInt(repo.ID, 10)) return nil } @@ -239,7 +240,7 @@ func ReplayHookTask(w *webhook_model.Webhook, uuid string) error { return err } - go hookQueue.Add(t.RepoID) + go hookQueue.Add(strconv.FormatInt(t.RepoID, 10)) return nil }