Remove legacy `unknwon/com` package (#19298) Follows: #19284 * The `CopyDir` is only used inside test code * Rewrite `ToSnakeCase` with more test cases * The `RedisCacher` only put strings into cache, here we use internal `toStr` to replace the legacy `ToStr` * The `UniqueQueue` can use string as ID directly, no need to call `ToStr`

Changes

17 changed files (+353/-58)

• cmd/manager_logging.go
  • @@ -11,6 +11,7 @@ import (
    "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" + "github.com/urfave/cli" )
• contrib/pr/checkout.go
  • @@ -112,7 +112,7 @@ func runPR() {
    unittest.LoadFixtures() util.RemoveAll(setting.RepoRootPath) util.RemoveAll(models.LocalCopyPath()) - util.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath) + unittest.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath) log.Printf("[PR] Setting up router\n") // routers.GlobalInit()
• go.mod
  • @@ -78,7 +78,6 @@ require (
    github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 - github.com/unknwon/com v1.0.1 github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361 github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae github.com/unrolled/render v1.4.1
  • @@ -251,6 +250,7 @@ require (
    github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/toqueteos/webbrowser v1.2.0 // indirect github.com/ulikunitz/xz v0.5.10 // indirect + github.com/unknwon/com v1.0.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.1 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
• integrations/integration_test.go
  • @@ -254,7 +254,7 @@ func prepareTestEnv(t testing.TB, skip ...int) func() {
    assert.NoError(t, unittest.LoadFixtures()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
  • @@ -550,7 +550,7 @@ func resetFixtures(t *testing.T) {
    assert.NoError(t, queue.GetManager().FlushAll(context.Background(), -1)) assert.NoError(t, unittest.LoadFixtures()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
• integrations/migration-test/migration_test.go
  • @@ -21,6 +21,7 @@ import (
    "code.gitea.io/gitea/integrations" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git"
  • @@ -60,7 +61,7 @@ func initMigrationTest(t *testing.T) func() {
    assert.True(t, len(setting.RepoRootPath) != 0) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
• models/migrations/migrations_test.go
  • @@ -203,7 +203,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
    deferFn := PrintCurrentTest(t, ourSkip) assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil {
• models/unittest/fscopy.go (new)
  • @@ -0,0 +1,103 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package unittest + + import ( + "errors" + "io" + "os" + "path" + "strings" + + "code.gitea.io/gitea/modules/util" + ) + + // Copy copies file from source to target path. + func Copy(src, dest string) error { + // Gather file information to set back later. + si, err := os.Lstat(src) + if err != nil { + return err + } + + // Handle symbolic link. + if si.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(src) + if err != nil { + return err + } + // NOTE: os.Chmod and os.Chtimes don't recognize symbolic link, + // which will lead "no such file or directory" error. + return os.Symlink(target, dest) + } + + sr, err := os.Open(src) + if err != nil { + return err + } + defer sr.Close() + + dw, err := os.Create(dest) + if err != nil { + return err + } + defer dw.Close() + + if _, err = io.Copy(dw, sr); err != nil { + return err + } + + // Set back file information. + if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil { + return err + } + return os.Chmod(dest, si.Mode()) + } + + // CopyDir copy files recursively from source to target directory. + // + // The filter accepts a function that process the path info. + // and should return true for need to filter. + // + // It returns error when error occurs in underlying functions. + func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error { + // Check if target directory exists. + if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) { + return errors.New("file or directory already exists: " + destPath) + } + + err := os.MkdirAll(destPath, os.ModePerm) + if err != nil { + return err + } + + // Gather directory info. + infos, err := util.StatDir(srcPath, true) + if err != nil { + return err + } + + var filter func(filePath string) bool + if len(filters) > 0 { + filter = filters[0] + } + + for _, info := range infos { + if filter != nil && filter(info) { + continue + } + + curPath := path.Join(destPath, info) + if strings.HasSuffix(info, "/") { + err = os.MkdirAll(curPath, os.ModePerm) + } else { + err = Copy(path.Join(srcPath, info), curPath) + } + if err != nil { + return err + } + } + return nil + }
• models/unittest/testdb.go
  • @@ -104,7 +104,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string, fixtureFiles ...string) {
    if err = util.RemoveAll(repoRootPath); err != nil { fatalTestError("util.RemoveAll: %v\n", err) } - if err = util.CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { + if err = CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { fatalTestError("util.CopyDir: %v\n", err) }
  • @@ -175,7 +175,7 @@ func PrepareTestEnv(t testing.TB) {
    assert.NoError(t, PrepareTestDatabase()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) metaPath := filepath.Join(giteaRoot, "integrations", "gitea-repositories-meta") - assert.NoError(t, util.CopyDir(metaPath, setting.RepoRootPath)) + assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) assert.NoError(t, err)
• modules/cache/cache_redis.go
  • @@ -6,11 +6,11 @@ package cache
    import ( "fmt" + "strconv" "time" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/nosql" - "code.gitea.io/gitea/modules/util" "gitea.com/go-chi/cache" "github.com/go-redis/redis/v8"
  • @@ -24,20 +24,37 @@ type RedisCacher struct {
    occupyMode bool } - // Put puts value into cache with key and expire time. + // toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally + func toStr(v interface{}) string { + if v == nil { + return "" + } + switch v := v.(type) { + case string: + return v + case []byte: + return string(v) + case int: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + default: + return fmt.Sprint(v) // as what the old com.ToStr does in most cases + } + } + + // Put puts value (string type) into cache with key and expire time. // If expired is 0, it lives forever. func (c *RedisCacher) Put(key string, val interface{}, expire int64) error { + // this function is not well-designed, it only puts string values into cache key = c.prefix + key if expire == 0 { - if err := c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), 0).Err(); err != nil { + if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil { return err } } else { - dur, err := time.ParseDuration(util.ToStr(expire) + "s") - if err != nil { - return err - } - if err = c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), dur).Err(); err != nil { + dur := time.Duration(expire) * time.Second + if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil { return err } }
• modules/context/csrf.go
  • @@ -22,8 +22,10 @@ import (
    "encoding/base32" "fmt" "net/http" + "strconv" "time" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware"
  • @@ -215,9 +217,16 @@ func Csrfer(opt CsrfOptions, ctx *Context) CSRF {
    } x.ID = "0" - uid := ctx.Session.Get(opt.SessionKey) - if uid != nil { - x.ID = util.ToStr(uid) + uidAny := ctx.Session.Get(opt.SessionKey) + if uidAny != nil { + switch uidVal := uidAny.(type) { + case string: + x.ID = uidVal + case int64: + x.ID = strconv.FormatInt(uidVal, 10) + default: + log.Error("invalid uid type in session: %T", uidAny) + } } needsNew := false
• modules/nosql/manager.go
  • @@ -11,6 +11,7 @@ import (
    "time" "code.gitea.io/gitea/modules/process" + "github.com/go-redis/redis/v8" "github.com/syndtr/goleveldb/leveldb" )
• modules/sync/unique_queue.go
  • @@ -5,8 +5,6 @@
    package sync - import "code.gitea.io/gitea/modules/util" - // UniqueQueue is a queue which guarantees only one instance of same // identity is in the line. Instances with same identity will be // discarded if there is already one in the line.
  • @@ -53,10 +51,10 @@ func (q *UniqueQueue) IsClosed() <-chan struct{} {
    } // IDs returns the current ids in the pool - func (q *UniqueQueue) IDs() []interface{} { + func (q *UniqueQueue) IDs() []string { q.table.lock.Lock() defer q.table.lock.Unlock() - ids := make([]interface{}, 0, len(q.table.pool)) + ids := make([]string, 0, len(q.table.pool)) for id := range q.table.pool { ids = append(ids, id) }
  • @@ -70,20 +68,19 @@ func (q *UniqueQueue) Queue() <-chan string {
    // Exist returns true if there is an instance with given identity // exists in the queue. - func (q *UniqueQueue) Exist(id interface{}) bool { - return q.table.IsRunning(util.ToStr(id)) + func (q *UniqueQueue) Exist(id string) bool { + return q.table.IsRunning(id) } // AddFunc adds new instance to the queue with a custom runnable function, // the queue is blocked until the function exits. - func (q *UniqueQueue) AddFunc(id interface{}, fn func()) { - idStr := util.ToStr(id) + func (q *UniqueQueue) AddFunc(id string, fn func()) { q.table.lock.Lock() - if _, ok := q.table.pool[idStr]; ok { + if _, ok := q.table.pool[id]; ok { q.table.lock.Unlock() return } - q.table.pool[idStr] = struct{}{} + q.table.pool[id] = struct{}{} if fn != nil { fn() }
  • @@ -91,17 +88,17 @@ func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
    select { case <-q.closed: return - case q.queue <- idStr: + case q.queue <- id: return } } // Add adds new instance to the queue. - func (q *UniqueQueue) Add(id interface{}) { + func (q *UniqueQueue) Add(id string) { q.AddFunc(id, nil) } // Remove removes instance from the queue. - func (q *UniqueQueue) Remove(id interface{}) { - q.table.Stop(util.ToStr(id)) + func (q *UniqueQueue) Remove(id string) { + q.table.Stop(id) }
• modules/util/legacy.go
  • @@ -9,29 +9,37 @@ import (
    "crypto/cipher" "crypto/rand" "errors" - - "github.com/unknwon/com" //nolint:depguard + "io" + "os" ) // CopyFile copies file from source to target path. func CopyFile(src, dest string) error { - return com.Copy(src, dest) - } + si, err := os.Lstat(src) + if err != nil { + return err + } - // CopyDir copy files recursively from source to target directory. - // It returns error when error occurs in underlying functions. - func CopyDir(srcPath, destPath string) error { - return com.CopyDir(srcPath, destPath) - } + sr, err := os.Open(src) + if err != nil { + return err + } + defer sr.Close() - // ToStr converts any interface to string. should be replaced. - func ToStr(value interface{}, args ...int) string { - return com.ToStr(value, args...) - } + dw, err := os.Create(dest) + if err != nil { + return err + } + defer dw.Close() - // ToSnakeCase converts a string to snake_case. should be replaced. - func ToSnakeCase(str string) string { - return com.ToSnakeCase(str) + if _, err = io.Copy(dw, sr); err != nil { + return err + } + + if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil { + return err + } + return os.Chmod(dest, si.Mode()) } // AESGCMEncrypt (from legacy package): encrypts plaintext with the given key using AES in GCM mode. should be replaced.
• modules/util/legacy_test.go
  • @@ -7,12 +7,38 @@ package util
    import ( "crypto/aes" "crypto/rand" + "fmt" + "os" "testing" + "time" "github.com/stretchr/testify/assert" - "github.com/unknwon/com" //nolint:depguard ) + func TestCopyFile(t *testing.T) { + testContent := []byte("hello") + + tmpDir := os.TempDir() + now := time.Now() + srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro()) + dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro()) + + _ = os.Remove(srcFile) + _ = os.Remove(dstFile) + defer func() { + _ = os.Remove(srcFile) + _ = os.Remove(dstFile) + }() + + err := os.WriteFile(srcFile, testContent, 0o777) + assert.NoError(t, err) + err = CopyFile(srcFile, dstFile) + assert.NoError(t, err) + dstContent, err := os.ReadFile(dstFile) + assert.NoError(t, err) + assert.Equal(t, testContent, dstContent) + } + func TestAESGCM(t *testing.T) { t.Parallel()
  • @@ -29,9 +55,4 @@ func TestAESGCM(t *testing.T) {
    assert.NoError(t, err) assert.Equal(t, plaintext, decrypted) - - // at the moment, we make sure the result is the same as the legacy package, this assertion can be removed in next round refactoring - legacy, err := com.AESGCMDecrypt(key, ciphertext) - assert.NoError(t, err) - assert.Equal(t, legacy, plaintext) }
• modules/util/string.go (new)
  • @@ -0,0 +1,88 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package util + + import "github.com/yuin/goldmark/util" + + func isSnakeCaseUpper(c byte) bool { + return 'A' <= c && c <= 'Z' + } + + func isSnakeCaseLowerOrNumber(c byte) bool { + return 'a' <= c && c <= 'z' || '0' <= c && c <= '9' + } + + // ToSnakeCase convert the input string to snake_case format. + // + // Some samples. + // "FirstName" => "first_name" + // "HTTPServer" => "http_server" + // "NoHTTPS" => "no_https" + // "GO_PATH" => "go_path" + // "GO PATH" => "go_path" // space is converted to underscore. + // "GO-PATH" => "go_path" // hyphen is converted to underscore. + // + func ToSnakeCase(input string) string { + if len(input) == 0 { + return "" + } + + var res []byte + if len(input) == 1 { + c := input[0] + if isSnakeCaseUpper(c) { + res = []byte{c + 'a' - 'A'} + } else if isSnakeCaseLowerOrNumber(c) { + res = []byte{c} + } else { + res = []byte{'_'} + } + } else { + res = make([]byte, 0, len(input)*4/3) + pos := 0 + needSep := false + for pos < len(input) { + c := input[pos] + if c >= 0x80 { + res = append(res, c) + pos++ + continue + } + isUpper := isSnakeCaseUpper(c) + if isUpper || isSnakeCaseLowerOrNumber(c) { + end := pos + 1 + if isUpper { + // skip the following upper letters + for end < len(input) && isSnakeCaseUpper(input[end]) { + end++ + } + if end-pos > 1 && end < len(input) && isSnakeCaseLowerOrNumber(input[end]) { + end-- + } + } + // skip the following lower or number letters + for end < len(input) && (isSnakeCaseLowerOrNumber(input[end]) || input[end] >= 0x80) { + end++ + } + if needSep { + res = append(res, '_') + } + res = append(res, input[pos:end]...) + pos = end + needSep = true + } else { + res = append(res, '_') + pos++ + needSep = false + } + } + for i := 0; i < len(res); i++ { + if isSnakeCaseUpper(res[i]) { + res[i] += 'a' - 'A' + } + } + } + return util.BytesToReadOnlyString(res) + }
• modules/util/string_test.go (new)
  • @@ -0,0 +1,48 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package util + + import ( + "testing" + + "github.com/stretchr/testify/assert" + ) + + func TestToSnakeCase(t *testing.T) { + cases := map[string]string{ + // all old cases from the legacy package + "HTTPServer": "http_server", + "_camelCase": "_camel_case", + "NoHTTPS": "no_https", + "Wi_thF": "wi_th_f", + "_AnotherTES_TCaseP": "_another_tes_t_case_p", + "ALL": "all", + "_HELLO_WORLD_": "_hello_world_", + "HELLO_WORLD": "hello_world", + "HELLO____WORLD": "hello____world", + "TW": "tw", + "_C": "_c", + + " sentence case ": "__sentence_case__", + " Mixed-hyphen case _and SENTENCE_case and UPPER-case": "_mixed_hyphen_case__and_sentence_case_and_upper_case", + + // new cases + " ": "_", + "A": "a", + "A0": "a0", + "a0": "a0", + "Aa0": "aa0", + "啊": "啊", + "A啊": "a啊", + "Aa啊b": "aa啊b", + "A啊B": "a啊_b", + "Aa啊B": "aa啊_b", + "TheCase2": "the_case2", + "ObjIDs": "obj_i_ds", // the strange database column name which already exists + } + for input, expected := range cases { + assert.Equal(t, expected, ToSnakeCase(input)) + } + }
• services/webhook/webhook.go
  • @@ -6,6 +6,7 @@ package webhook
    import ( "fmt" + "strconv" "strings" repo_model "code.gitea.io/gitea/models/repo"
  • @@ -106,7 +107,7 @@ func PrepareWebhook(w *webhook_model.Webhook, repo *repo_model.Repository, event
    return err } - go hookQueue.Add(repo.ID) + go hookQueue.Add(strconv.FormatInt(repo.ID, 10)) return nil }
  • @@ -187,7 +188,7 @@ func PrepareWebhooks(repo *repo_model.Repository, event webhook_model.HookEventT
    return err } - go hookQueue.Add(repo.ID) + go hookQueue.Add(strconv.FormatInt(repo.ID, 10)) return nil }
  • @@ -239,7 +240,7 @@ func ReplayHookTask(w *webhook_model.Webhook, uuid string) error {
    return err } - go hookQueue.Add(t.RepoID) + go hookQueue.Add(strconv.FormatInt(t.RepoID, 10)) return nil }