From 5b7466053d993685939da8623fb78e94e4ee0797 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 16:00:26 +0800 Subject: [PATCH 01/10] Skip frontend ROOT_URL check on installation page, remove unnecessary global var (#19291) Skip `checkAppUrl` message on installation page because the ROOT_URL is not determined yet Move global var `supportedDbTypeNames` into `install.Init` as a local var --- routers/install/install.go | 16 +++++++--------- web_src/js/features/common-global.js | 3 +++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/routers/install/install.go b/routers/install/install.go index 164ce6840565f..ec1719439f53a 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -42,20 +42,18 @@ const ( tplPostInstall base.TplName = "post-install" ) -var supportedDbTypeNames []map[string]string // use a slice to keep order -func getDbTypeNames() []map[string]string { - if supportedDbTypeNames == nil { - for _, t := range setting.SupportedDatabaseTypes { - supportedDbTypeNames = append(supportedDbTypeNames, map[string]string{"type": t, "name": setting.DatabaseTypeNames[t]}) - } +// getSupportedDbTypeNames returns a slice for supported database types and names. The slice is used to keep the order +func getSupportedDbTypeNames() (dbTypeNames []map[string]string) { + for _, t := range setting.SupportedDatabaseTypes { + dbTypeNames = append(dbTypeNames, map[string]string{"type": t, "name": setting.DatabaseTypeNames[t]}) } - return supportedDbTypeNames + return dbTypeNames } // Init prepare for rendering installation page func Init(next http.Handler) http.Handler { rnd := templates.HTMLRenderer() - + dbTypeNames := getSupportedDbTypeNames() return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { if setting.InstallLock { resp.Header().Add("Refresh", "1; url="+setting.AppURL+"user/login") @@ -74,7 +72,7 @@ func Init(next http.Handler) http.Handler { "i18n": locale, "Title": locale.Tr("install.install"), "PageIsInstall": true, - "DbTypeNames": getDbTypeNames(), + "DbTypeNames": dbTypeNames, "AllLangs": translation.AllLangs(), "PageStartTime": startTime, diff --git a/web_src/js/features/common-global.js b/web_src/js/features/common-global.js index 60af4d0d67e0e..dda803ae8de31 100644 --- a/web_src/js/features/common-global.js +++ b/web_src/js/features/common-global.js @@ -358,6 +358,9 @@ export function checkAppUrl() { if (curUrl.startsWith(appUrl)) { return; } + if (document.querySelector('.page-content.install')) { + return; // no need to show the message on the installation page + } showGlobalErrorMessage(`Your ROOT_URL in app.ini is ${appUrl} but you are visiting ${curUrl} You should set ROOT_URL correctly, otherwise the web may not work correctly.`); } From 65f17bfc31f0f2659978d4d1d5ff825146c53a4d Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 1 Apr 2022 16:47:50 +0800 Subject: [PATCH 02/10] Refactor legacy `unknwon/com` package, improve golangci lint (#19284) The main purpose is to refactor the legacy `unknwon/com` package. 1. Remove most imports of `unknwon/com`, only `util/legacy.go` imports the legacy `unknwon/com` 2. Use golangci's depguard to process denied packages 3. Fix some incorrect values in golangci.yml, eg, the version should be quoted string `"1.18"` 4. Use correctly escaped content for `go-import` and `go-source` meta tags 5. Refactor `com.Expand` to our stable (and the same fast) `vars.Expand`, our `vars.Expand` can still return partially rendered content even if the template is not good (eg: key mistach). --- .golangci.yml | 14 ++++- integrations/goget_test.go | 3 +- models/migrations/migrations_test.go | 3 +- modules/cache/cache_redis.go | 8 +-- modules/context/context.go | 6 +- modules/context/csrf.go | 14 +++-- modules/context/repo.go | 10 ++- modules/json/json.go | 2 +- modules/markup/html.go | 11 +++- modules/repository/init.go | 10 ++- modules/setting/setting.go | 3 +- modules/sync/unique_queue.go | 10 ++- modules/templates/vars/vars.go | 93 ++++++++++++++++++++++++++++ modules/templates/vars/vars_test.go | 72 +++++++++++++++++++++ modules/util/copy.go | 20 ------ modules/util/io.go | 2 +- modules/util/legacy.go | 84 +++++++++++++++++++++++++ modules/util/legacy_test.go | 37 +++++++++++ modules/util/path.go | 27 +++++++- modules/web/middleware/binding.go | 6 +- routers/web/goget.go | 32 +++++----- routers/web/repo/issue.go | 11 +++- 22 files changed, 397 insertions(+), 81 deletions(-) create mode 100644 modules/templates/vars/vars.go create mode 100644 modules/templates/vars/vars_test.go delete mode 100644 modules/util/copy.go create mode 100644 modules/util/legacy.go create mode 100644 modules/util/legacy_test.go diff --git a/.golangci.yml b/.golangci.yml index 1794b4594dea8..8e31d0cbc4d6b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -18,6 +18,7 @@ linters: - ineffassign - revive - gofumpt + - depguard enable-all: false disable-all: true fast: false @@ -65,7 +66,15 @@ linters-settings: - name: modifies-value-receiver gofumpt: extra-rules: true - lang-version: 1.18 + lang-version: "1.18" + depguard: + # TODO: use depguard to replace import checks in gitea-vet + list-type: denylist + # Check the list against standard lib. + include-go-root: true + packages-with-error-message: + - encoding/json: "use gitea's modules/json instead of encoding/json" + - github.com/unknwon/com: "use gitea's util and replacements" issues: exclude-rules: @@ -153,5 +162,6 @@ issues: - path: models/user/openid.go linters: - golint - - linters: staticcheck + - linters: + - staticcheck text: "strings.Title is deprecated: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead." diff --git a/integrations/goget_test.go b/integrations/goget_test.go index 5dc9c5e0a8d17..504d869990fbe 100644 --- a/integrations/goget_test.go +++ b/integrations/goget_test.go @@ -29,8 +29,7 @@ func TestGoGet(t *testing.T) { go get --insecure %[1]s:%[2]s/blah/glah - -`, setting.Domain, setting.HTTPPort, setting.AppURL) +`, setting.Domain, setting.HTTPPort, setting.AppURL) assert.Equal(t, expected, resp.Body.String()) } diff --git a/models/migrations/migrations_test.go b/models/migrations/migrations_test.go index f798d501179f7..a17eba54e82be 100644 --- a/models/migrations/migrations_test.go +++ b/models/migrations/migrations_test.go @@ -24,7 +24,6 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" - "github.com/unknwon/com" "xorm.io/xorm" "xorm.io/xorm/names" ) @@ -204,7 +203,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En deferFn := PrintCurrentTest(t, ourSkip) assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, com.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), + assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { diff --git a/modules/cache/cache_redis.go b/modules/cache/cache_redis.go index 148725ae66015..e4b9a70f63f3a 100644 --- a/modules/cache/cache_redis.go +++ b/modules/cache/cache_redis.go @@ -10,10 +10,10 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/nosql" + "code.gitea.io/gitea/modules/util" "gitea.com/go-chi/cache" "github.com/go-redis/redis/v8" - "github.com/unknwon/com" ) // RedisCacher represents a redis cache adapter implementation. @@ -29,15 +29,15 @@ type RedisCacher struct { func (c *RedisCacher) Put(key string, val interface{}, expire int64) error { key = c.prefix + key if expire == 0 { - if err := c.c.Set(graceful.GetManager().HammerContext(), key, com.ToStr(val), 0).Err(); err != nil { + if err := c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), 0).Err(); err != nil { return err } } else { - dur, err := time.ParseDuration(com.ToStr(expire) + "s") + dur, err := time.ParseDuration(util.ToStr(expire) + "s") if err != nil { return err } - if err = c.c.Set(graceful.GetManager().HammerContext(), key, com.ToStr(val), dur).Err(); err != nil { + if err = c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), dur).Err(); err != nil { return err } } diff --git a/modules/context/context.go b/modules/context/context.go index 4905e1cb80e2f..f73b5f19c0c67 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -31,13 +31,13 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" "code.gitea.io/gitea/services/auth" "gitea.com/go-chi/cache" "gitea.com/go-chi/session" chi "github.com/go-chi/chi/v5" - "github.com/unknwon/com" "github.com/unrolled/render" "golang.org/x/crypto/pbkdf2" ) @@ -475,7 +475,7 @@ func (ctx *Context) CookieDecrypt(secret, val string) (string, bool) { } key := pbkdf2.Key([]byte(secret), []byte(secret), 1000, 16, sha256.New) - text, err = com.AESGCMDecrypt(key, text) + text, err = util.AESGCMDecrypt(key, text) return string(text), err == nil } @@ -489,7 +489,7 @@ func (ctx *Context) SetSuperSecureCookie(secret, name, value string, expiry int) // CookieEncrypt encrypts a given value using the provided secret func (ctx *Context) CookieEncrypt(secret, value string) string { key := pbkdf2.Key([]byte(secret), []byte(secret), 1000, 16, sha256.New) - text, err := com.AESGCMEncrypt(key, []byte(value)) + text, err := util.AESGCMEncrypt(key, []byte(value)) if err != nil { panic("error encrypting cookie: " + err.Error()) } diff --git a/modules/context/csrf.go b/modules/context/csrf.go index 99c223c884da3..1fb992e2ae409 100644 --- a/modules/context/csrf.go +++ b/modules/context/csrf.go @@ -19,13 +19,14 @@ package context import ( + "encoding/base32" + "fmt" "net/http" "time" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" - - "github.com/unknwon/com" ) // CSRF represents a CSRF service and is used to get the current token and validate a suspect token. @@ -162,7 +163,12 @@ func prepareOptions(options []CsrfOptions) CsrfOptions { // Defaults. if len(opt.Secret) == 0 { - opt.Secret = string(com.RandomCreateBytes(10)) + randBytes, err := util.CryptoRandomBytes(8) + if err != nil { + // this panic can be handled by the recover() in http handlers + panic(fmt.Errorf("failed to generate random bytes: %w", err)) + } + opt.Secret = base32.StdEncoding.EncodeToString(randBytes) } if len(opt.Header) == 0 { opt.Header = "X-CSRFToken" @@ -211,7 +217,7 @@ func Csrfer(opt CsrfOptions, ctx *Context) CSRF { x.ID = "0" uid := ctx.Session.Get(opt.SessionKey) if uid != nil { - x.ID = com.ToStr(uid) + x.ID = util.ToStr(uid) } needsNew := false diff --git a/modules/context/repo.go b/modules/context/repo.go index 5a9e38a1d9197..a7c9a982c42b2 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -8,6 +8,7 @@ package context import ( "context" "fmt" + "html" "io" "net/http" "net/url" @@ -29,7 +30,6 @@ import ( asymkey_service "code.gitea.io/gitea/services/asymkey" "github.com/editorconfig/editorconfig-core-go/v2" - "github.com/unknwon/com" ) // IssueTemplateDirCandidates issue templates directory @@ -308,11 +308,9 @@ func EarlyResponseForGoGetMeta(ctx *Context) { ctx.PlainText(http.StatusBadRequest, "invalid repository path") return } - ctx.PlainText(http.StatusOK, com.Expand(``, - map[string]string{ - "GoGetImport": ComposeGoGetImport(username, reponame), - "CloneLink": repo_model.ComposeHTTPSCloneURL(username, reponame), - })) + goImportContent := fmt.Sprintf("%s git %s", ComposeGoGetImport(username, reponame), repo_model.ComposeHTTPSCloneURL(username, reponame)) + htmlMeta := fmt.Sprintf(``, html.EscapeString(goImportContent)) + ctx.PlainText(http.StatusOK, htmlMeta) } // RedirectToRepo redirect to a differently-named repository diff --git a/modules/json/json.go b/modules/json/json.go index 3afa86023c6f8..4361262a2f241 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -8,7 +8,7 @@ package json import ( "bytes" "encoding/binary" - "encoding/json" + "encoding/json" //nolint:depguard "io" jsoniter "github.com/json-iterator/go" diff --git a/modules/markup/html.go b/modules/markup/html.go index 6673f52bf4d6b..c5d36e701fb01 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -21,9 +21,9 @@ import ( "code.gitea.io/gitea/modules/markup/common" "code.gitea.io/gitea/modules/references" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" - "github.com/unknwon/com" "golang.org/x/net/html" "golang.org/x/net/html/atom" "mvdan.cc/xurls/v2" @@ -838,7 +838,14 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End] if exttrack && !ref.IsPull { ctx.Metas["index"] = ref.Issue - link = createLink(com.Expand(ctx.Metas["format"], ctx.Metas), reftext, "ref-issue ref-external-issue") + + res, err := vars.Expand(ctx.Metas["format"], ctx.Metas) + if err != nil { + // here we could just log the error and continue the rendering + log.Error("unable to expand template vars for ref %s, err: %v", ref.Issue, err) + } + + link = createLink(res, reftext, "ref-issue ref-external-issue") } else { // Path determines the type of link that will be rendered. It's unknown at this point whether // the linked item is actually a PR or an issue. Luckily it's of no real consequence because diff --git a/modules/repository/init.go b/modules/repository/init.go index d5a67df5d150c..515992f97deb1 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -22,10 +22,9 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" - - "github.com/unknwon/com" ) var ( @@ -250,8 +249,13 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, "CloneURL.HTTPS": cloneLink.HTTPS, "OwnerName": repo.OwnerName, } + res, err := vars.Expand(string(data), match) + if err != nil { + // here we could just log the error and continue the rendering + log.Error("unable to expand template vars for repo README: %s, err: %v", opts.Readme, err) + } if err = os.WriteFile(filepath.Join(tmpDir, "README.md"), - []byte(com.Expand(string(data), match)), 0o644); err != nil { + []byte(res), 0o644); err != nil { return fmt.Errorf("write README.md: %v", err) } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 17a02bf5a1834..ed91382de37b6 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -27,7 +27,6 @@ import ( "code.gitea.io/gitea/modules/user" "code.gitea.io/gitea/modules/util" - "github.com/unknwon/com" gossh "golang.org/x/crypto/ssh" ini "gopkg.in/ini.v1" ) @@ -612,7 +611,7 @@ func loadFromConf(allowEmpty bool, extraConfig string) { Cfg.NameMapper = ini.SnackCase - homeDir, err := com.HomeDir() + homeDir, err := util.HomeDir() if err != nil { log.Fatal("Failed to get home directory: %v", err) } diff --git a/modules/sync/unique_queue.go b/modules/sync/unique_queue.go index d41726b5af930..414cc50f39a76 100644 --- a/modules/sync/unique_queue.go +++ b/modules/sync/unique_queue.go @@ -5,9 +5,7 @@ package sync -import ( - "github.com/unknwon/com" -) +import "code.gitea.io/gitea/modules/util" // UniqueQueue is a queue which guarantees only one instance of same // identity is in the line. Instances with same identity will be @@ -73,13 +71,13 @@ func (q *UniqueQueue) Queue() <-chan string { // Exist returns true if there is an instance with given identity // exists in the queue. func (q *UniqueQueue) Exist(id interface{}) bool { - return q.table.IsRunning(com.ToStr(id)) + return q.table.IsRunning(util.ToStr(id)) } // AddFunc adds new instance to the queue with a custom runnable function, // the queue is blocked until the function exits. func (q *UniqueQueue) AddFunc(id interface{}, fn func()) { - idStr := com.ToStr(id) + idStr := util.ToStr(id) q.table.lock.Lock() if _, ok := q.table.pool[idStr]; ok { q.table.lock.Unlock() @@ -105,5 +103,5 @@ func (q *UniqueQueue) Add(id interface{}) { // Remove removes instance from the queue. func (q *UniqueQueue) Remove(id interface{}) { - q.table.Stop(com.ToStr(id)) + q.table.Stop(util.ToStr(id)) } diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go new file mode 100644 index 0000000000000..a22ea4d777fca --- /dev/null +++ b/modules/templates/vars/vars.go @@ -0,0 +1,93 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package vars + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +// ErrWrongSyntax represents a wrong syntax with a template +type ErrWrongSyntax struct { + Template string +} + +func (err ErrWrongSyntax) Error() string { + return fmt.Sprintf("wrong syntax found in %s", err.Template) +} + +// ErrVarMissing represents an error that no matched variable +type ErrVarMissing struct { + Template string + Var string +} + +func (err ErrVarMissing) Error() string { + return fmt.Sprintf("the variable %s is missing for %s", err.Var, err.Template) +} + +// Expand replaces all variables like {var} by `vars` map, it always returns the expanded string regardless of errors +// if error occurs, the error part doesn't change and is returned as it is. +func Expand(template string, vars map[string]string) (string, error) { + // in the future, if necessary, we can introduce some escape-char, + // for example: it will use `#' as a reversed char, templates will use `{#{}` to do escape and output char '{'. + var buf strings.Builder + var err error + + posBegin := 0 + strLen := len(template) + for posBegin < strLen { + // find the next `{` + pos := strings.IndexByte(template[posBegin:], '{') + if pos == -1 { + buf.WriteString(template[posBegin:]) + break + } + + // copy texts between vars + buf.WriteString(template[posBegin : posBegin+pos]) + + // find the var between `{` and `}`/end + posBegin += pos + posEnd := posBegin + 1 + for posEnd < strLen { + if template[posEnd] == '}' { + posEnd++ + break + } // in the future, if we need to support escape chars, we can do: if (isEscapeChar) { posEnd+=2 } + posEnd++ + } + + // the var part, it can be "{", "{}", "{..." or or "{...}" + part := template[posBegin:posEnd] + posBegin = posEnd + if part == "{}" || part[len(part)-1] != '}' { + // treat "{}" or "{..." as error + err = ErrWrongSyntax{Template: template} + buf.WriteString(part) + } else { + // now we get a valid key "{...}" + key := part[1 : len(part)-1] + keyFirst, _ := utf8.DecodeRuneInString(key) + if unicode.IsSpace(keyFirst) || unicode.IsPunct(keyFirst) || unicode.IsControl(keyFirst) { + // the if key doesn't start with a letter, then we do not treat it as a var now + buf.WriteString(part) + } else { + // look up in the map + if val, ok := vars[key]; ok { + buf.WriteString(val) + } else { + // write the non-existing var as it is + buf.WriteString(part) + err = ErrVarMissing{Template: template, Var: key} + } + } + } + } + + return buf.String(), err +} diff --git a/modules/templates/vars/vars_test.go b/modules/templates/vars/vars_test.go new file mode 100644 index 0000000000000..1cd7669c0095e --- /dev/null +++ b/modules/templates/vars/vars_test.go @@ -0,0 +1,72 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package vars + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExpandVars(t *testing.T) { + kases := []struct { + tmpl string + data map[string]string + out string + error bool + }{ + { + tmpl: "{a}", + data: map[string]string{ + "a": "1", + }, + out: "1", + }, + { + tmpl: "expand {a}, {b} and {c}, with non-var { } {#}", + data: map[string]string{ + "a": "1", + "b": "2", + "c": "3", + }, + out: "expand 1, 2 and 3, with non-var { } {#}", + }, + { + tmpl: "中文内容 {一}, {二} 和 {三} 中文结尾", + data: map[string]string{ + "一": "11", + "二": "22", + "三": "33", + }, + out: "中文内容 11, 22 和 33 中文结尾", + }, + { + tmpl: "expand {{a}, {b} and {c}", + data: map[string]string{ + "a": "foo", + "b": "bar", + }, + out: "expand {{a}, bar and {c}", + error: true, + }, + { + tmpl: "expand } {} and {", + out: "expand } {} and {", + error: true, + }, + } + + for _, kase := range kases { + t.Run(kase.tmpl, func(t *testing.T) { + res, err := Expand(kase.tmpl, kase.data) + assert.EqualValues(t, kase.out, res) + if kase.error { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/modules/util/copy.go b/modules/util/copy.go deleted file mode 100644 index 46765849dc753..0000000000000 --- a/modules/util/copy.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package util - -import ( - "github.com/unknwon/com" -) - -// CopyFile copies file from source to target path. -func CopyFile(src, dest string) error { - return com.Copy(src, dest) -} - -// CopyDir copy files recursively from source to target directory. -// It returns error when error occurs in underlying functions. -func CopyDir(srcPath, destPath string) error { - return com.CopyDir(srcPath, destPath) -} diff --git a/modules/util/io.go b/modules/util/io.go index b467c0ac8a04a..0c677c359f5ea 100644 --- a/modules/util/io.go +++ b/modules/util/io.go @@ -9,7 +9,7 @@ import ( ) // ReadAtMost reads at most len(buf) bytes from r into buf. -// It returns the number of bytes copied. n is only less then len(buf) if r provides fewer bytes. +// It returns the number of bytes copied. n is only less than len(buf) if r provides fewer bytes. // If EOF occurs while reading, err will be nil. func ReadAtMost(r io.Reader, buf []byte) (n int, err error) { n, err = io.ReadFull(r, buf) diff --git a/modules/util/legacy.go b/modules/util/legacy.go new file mode 100644 index 0000000000000..c7da5415349a0 --- /dev/null +++ b/modules/util/legacy.go @@ -0,0 +1,84 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "errors" + + "github.com/unknwon/com" //nolint:depguard +) + +// CopyFile copies file from source to target path. +func CopyFile(src, dest string) error { + return com.Copy(src, dest) +} + +// CopyDir copy files recursively from source to target directory. +// It returns error when error occurs in underlying functions. +func CopyDir(srcPath, destPath string) error { + return com.CopyDir(srcPath, destPath) +} + +// ToStr converts any interface to string. should be replaced. +func ToStr(value interface{}, args ...int) string { + return com.ToStr(value, args...) +} + +// ToSnakeCase converts a string to snake_case. should be replaced. +func ToSnakeCase(str string) string { + return com.ToSnakeCase(str) +} + +// AESGCMEncrypt (from legacy package): encrypts plaintext with the given key using AES in GCM mode. should be replaced. +func AESGCMEncrypt(key, plaintext []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + + ciphertext := gcm.Seal(nil, nonce, plaintext, nil) + return append(nonce, ciphertext...), nil +} + +// AESGCMDecrypt (from legacy package): decrypts ciphertext with the given key using AES in GCM mode. should be replaced. +func AESGCMDecrypt(key, ciphertext []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + size := gcm.NonceSize() + if len(ciphertext)-size <= 0 { + return nil, errors.New("ciphertext is empty") + } + + nonce := ciphertext[:size] + ciphertext = ciphertext[size:] + + plainText, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, err + } + + return plainText, nil +} diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go new file mode 100644 index 0000000000000..cfda93d3ad34b --- /dev/null +++ b/modules/util/legacy_test.go @@ -0,0 +1,37 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "crypto/aes" + "crypto/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/unknwon/com" //nolint:depguard +) + +func TestAESGCM(t *testing.T) { + t.Parallel() + + key := make([]byte, aes.BlockSize) + _, err := rand.Read(key) + assert.NoError(t, err) + + plaintext := []byte("this will be encrypted") + + ciphertext, err := AESGCMEncrypt(key, plaintext) + assert.NoError(t, err) + + decrypted, err := AESGCMDecrypt(key, ciphertext) + assert.NoError(t, err) + + assert.Equal(t, plaintext, decrypted) + + // at the moment, we make sure the result is the same as the legacy package, this assertion can be removed in next round refactoring + legacy, err := com.AESGCMDecrypt(key, ciphertext) + assert.NoError(t, err) + assert.Equal(t, legacy, plaintext) +} diff --git a/modules/util/path.go b/modules/util/path.go index f4acf92ba933e..ed7cc62699446 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -154,6 +154,10 @@ func StatDir(rootPath string, includeDir ...bool) ([]string, error) { return statDir(rootPath, "", isIncludeDir, false, false) } +func isOSWindows() bool { + return runtime.GOOS == "windows" +} + // FileURLToPath extracts the path information from a file://... url. func FileURLToPath(u *url.URL) (string, error) { if u.Scheme != "file" { @@ -162,7 +166,7 @@ func FileURLToPath(u *url.URL) (string, error) { path := u.Path - if runtime.GOOS != "windows" { + if !isOSWindows() { return path, nil } @@ -173,3 +177,24 @@ func FileURLToPath(u *url.URL) (string, error) { } return path, nil } + +// HomeDir returns path of '~'(in Linux) on Windows, +// it returns error when the variable does not exist. +func HomeDir() (home string, err error) { + // TODO: some users run Gitea with mismatched uid and "HOME=xxx" (they set HOME=xxx by environment manually) + // so at the moment we can not use `user.Current().HomeDir` + if isOSWindows() { + home = os.Getenv("USERPROFILE") + if home == "" { + home = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + } + } else { + home = os.Getenv("HOME") + } + + if home == "" { + return "", errors.New("cannot get home directory") + } + + return home, nil +} diff --git a/modules/web/middleware/binding.go b/modules/web/middleware/binding.go index 9b0b1d7784c32..c9dc4a8f59a01 100644 --- a/modules/web/middleware/binding.go +++ b/modules/web/middleware/binding.go @@ -10,10 +10,10 @@ import ( "strings" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/validation" "gitea.com/go-chi/binding" - "github.com/unknwon/com" ) // Form form binding interface @@ -22,7 +22,7 @@ type Form interface { } func init() { - binding.SetNameMapper(com.ToSnakeCase) + binding.SetNameMapper(util.ToSnakeCase) } // AssignForm assign form values back to the template data. @@ -43,7 +43,7 @@ func AssignForm(form interface{}, data map[string]interface{}) { if fieldName == "-" { continue } else if len(fieldName) == 0 { - fieldName = com.ToSnakeCase(field.Name) + fieldName = util.ToSnakeCase(field.Name) } data[fieldName] = val.Field(i).Interface() diff --git a/routers/web/goget.go b/routers/web/goget.go index 4a31fcc2c51cc..a58739fe42a3c 100644 --- a/routers/web/goget.go +++ b/routers/web/goget.go @@ -5,6 +5,8 @@ package web import ( + "fmt" + "html" "net/http" "net/url" "path" @@ -14,8 +16,6 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - - "github.com/unknwon/com" ) func goGet(ctx *context.Context) { @@ -65,23 +65,23 @@ func goGet(ctx *context.Context) { if appURL.Scheme == string(setting.HTTP) { insecure = "--insecure " } - ctx.RespHeader().Set("Content-Type", "text/html") - ctx.Status(http.StatusOK) - _, _ = ctx.Write([]byte(com.Expand(` + + goGetImport := context.ComposeGoGetImport(ownerName, trimmedRepoName) + goImportContent := fmt.Sprintf("%s git %s", goGetImport, repo_model.ComposeHTTPSCloneURL(ownerName, repoName) /*CloneLink*/) + goSourceContent := fmt.Sprintf("%s _ %s %s", goGetImport, prefix+"{/dir}" /*GoDocDirectory*/, prefix+"{/dir}/{file}#L{line}" /*GoDocFile*/) + goGetCli := fmt.Sprintf("go get %s%s", insecure, goGetImport) + + res := fmt.Sprintf(` - - + + - go get {Insecure}{GoGetImport} + %s - -`, map[string]string{ - "GoGetImport": context.ComposeGoGetImport(ownerName, trimmedRepoName), - "CloneLink": repo_model.ComposeHTTPSCloneURL(ownerName, repoName), - "GoDocDirectory": prefix + "{/dir}", - "GoDocFile": prefix + "{/dir}/{file}#L{line}", - "Insecure": insecure, - }))) +`, html.EscapeString(goImportContent), html.EscapeString(goSourceContent), html.EscapeString(goGetCli)) + + ctx.RespHeader().Set("Content-Type", "text/html") + _, _ = ctx.Write([]byte(res)) } diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 486a63a9e105f..a1a7200ba41fa 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -35,6 +35,7 @@ import ( "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/upload" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" @@ -43,8 +44,6 @@ import ( "code.gitea.io/gitea/services/forms" issue_service "code.gitea.io/gitea/services/issue" pull_service "code.gitea.io/gitea/services/pull" - - "github.com/unknwon/com" ) const ( @@ -1113,7 +1112,13 @@ func ViewIssue(ctx *context.Context) { if extIssueUnit.ExternalTrackerConfig().ExternalTrackerStyle == markup.IssueNameStyleNumeric || extIssueUnit.ExternalTrackerConfig().ExternalTrackerStyle == "" { metas := ctx.Repo.Repository.ComposeMetas() metas["index"] = ctx.Params(":index") - ctx.Redirect(com.Expand(extIssueUnit.ExternalTrackerConfig().ExternalTrackerFormat, metas)) + res, err := vars.Expand(extIssueUnit.ExternalTrackerConfig().ExternalTrackerFormat, metas) + if err != nil { + log.Error("unable to expand template vars for issue url. issue: %s, err: %v", metas["index"], err) + ctx.ServerError("Expand", err) + return + } + ctx.Redirect(res) return } } else if err != nil && !repo_model.IsErrUnitTypeNotExist(err) { From 58c9d12933236c57be958a7e09b8049712624c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Gardfj=C3=A4ll?= Date: Fri, 1 Apr 2022 16:14:36 +0200 Subject: [PATCH 03/10] A pull-mirror repo should be marked as such on creation (#19295) Right now, a pull-mirror repo does not get marked as such until *after* the mirroring completes. In the meantime, it will show up (in API and UI) as a regular repo. --- integrations/mirror_pull_test.go | 1 + modules/repository/create.go | 1 + 2 files changed, 2 insertions(+) diff --git a/integrations/mirror_pull_test.go b/integrations/mirror_pull_test.go index 385f0f5a6f5fb..dd66974e04007 100644 --- a/integrations/mirror_pull_test.go +++ b/integrations/mirror_pull_test.go @@ -46,6 +46,7 @@ func TestMirrorPull(t *testing.T) { Status: repo_model.RepositoryBeingMigrated, }) assert.NoError(t, err) + assert.True(t, mirrorRepo.IsMirror, "expected pull-mirror repo to be marked as a mirror immediately after its creation") ctx := context.Background() diff --git a/modules/repository/create.go b/modules/repository/create.go index 33039d77c28c1..21d45c896e650 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -54,6 +54,7 @@ func CreateRepository(doer, u *user_model.User, opts models.CreateRepoOptions) ( Status: opts.Status, IsEmpty: !opts.AutoInit, TrustModel: opts.TrustModel, + IsMirror: opts.IsMirror, } var rollbackRepo *repo_model.Repository From 4c5cb1e2f2c7a1fcc3b151192bd45bd1bbc090bd Mon Sep 17 00:00:00 2001 From: Norwin Date: Fri, 1 Apr 2022 17:31:40 +0200 Subject: [PATCH 04/10] Improve package registry docs (#19273) * Improve package registry docs * move new content down * add hint re upload a package * wording, formatting Co-authored-by: wxiaoguang --- docs/content/doc/packages/overview.en-us.md | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index f7809fc8a37bc..1e4209930f551 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -38,6 +38,29 @@ The following package managers are currently supported: **The following paragraphs only apply if Packages are not globally disabled!** +## Repository-Packages + +A package always belongs to an owner (a user or organisation), not a repository. +To link an (already uploaded) package to a repository, open the settings page +on that package and choose a repository to link this package to. +The entire package will be linked, not just a single version. + +Linking a package results in showing that package in the repository's package list, +and shows a link to the repository on the package site (as well as a link to the repository issues). + +## Access Restrictions + +| Package owner type | User | Organization | +|--------------------|------|--------------| +| **read** access | public, if user is public too; otherwise for this user only | public, if org is public, otherwise org members only | +| **write** access | owner only | org members with admin or write access to the org | + +N.B.: These access restrictions are [subject to change](https://github.com/go-gitea/gitea/issues/19270), where more finegrained control will be added via a dedicated organization team permission. + +## Create or upload a package + +Depending on the type of package, use the respective package-manager for that. Check out the sub-page of a specific package manager for instructions. + ## View packages You can view the packages of a repository on the repository page. From 4f27c289472a4cfafb5a9b0e38e6a3413c30c562 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 2 Apr 2022 00:34:57 +0800 Subject: [PATCH 05/10] Remove legacy `unknwon/com` package (#19298) Follows: #19284 * The `CopyDir` is only used inside test code * Rewrite `ToSnakeCase` with more test cases * The `RedisCacher` only put strings into cache, here we use internal `toStr` to replace the legacy `ToStr` * The `UniqueQueue` can use string as ID directly, no need to call `ToStr` --- cmd/manager_logging.go | 1 + contrib/pr/checkout.go | 2 +- go.mod | 2 +- integrations/integration_test.go | 4 +- integrations/migration-test/migration_test.go | 3 +- models/migrations/migrations_test.go | 2 +- models/unittest/fscopy.go | 103 ++++++++++++++++++ models/unittest/testdb.go | 4 +- modules/cache/cache_redis.go | 33 ++++-- modules/context/csrf.go | 15 ++- modules/nosql/manager.go | 1 + modules/sync/unique_queue.go | 25 ++--- modules/util/legacy.go | 40 ++++--- modules/util/legacy_test.go | 33 +++++- modules/util/string.go | 88 +++++++++++++++ modules/util/string_test.go | 48 ++++++++ services/webhook/webhook.go | 7 +- 17 files changed, 353 insertions(+), 58 deletions(-) create mode 100644 models/unittest/fscopy.go create mode 100644 modules/util/string.go create mode 100644 modules/util/string_test.go diff --git a/cmd/manager_logging.go b/cmd/manager_logging.go index eb311d28926c4..0043ea1e52ad4 100644 --- a/cmd/manager_logging.go +++ b/cmd/manager_logging.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" + "github.com/urfave/cli" ) diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index ceeba4de3b58e..42ccf88af8386 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -112,7 +112,7 @@ func runPR() { unittest.LoadFixtures() util.RemoveAll(setting.RepoRootPath) util.RemoveAll(models.LocalCopyPath()) - util.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath) + unittest.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath) log.Printf("[PR] Setting up router\n") // routers.GlobalInit() diff --git a/go.mod b/go.mod index 0378ccd5e7bb2..da2f1e03dd8ac 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,6 @@ require ( github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 - github.com/unknwon/com v1.0.1 github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361 github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae github.com/unrolled/render v1.4.1 @@ -251,6 +250,7 @@ require ( github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/toqueteos/webbrowser v1.2.0 // indirect github.com/ulikunitz/xz v0.5.10 // indirect + github.com/unknwon/com v1.0.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.1 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect diff --git a/integrations/integration_test.go b/integrations/integration_test.go index c778fb8013459..9e0445cae7307 100644 --- a/integrations/integration_test.go +++ b/integrations/integration_test.go @@ -254,7 +254,7 @@ func prepareTestEnv(t testing.TB, skip ...int) func() { assert.NoError(t, unittest.LoadFixtures()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) @@ -550,7 +550,7 @@ func resetFixtures(t *testing.T) { assert.NoError(t, queue.GetManager().FlushAll(context.Background(), -1)) assert.NoError(t, unittest.LoadFixtures()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index 0518dd1179892..6e55807c271e3 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/integrations" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" @@ -60,7 +61,7 @@ func initMigrationTest(t *testing.T) func() { assert.True(t, len(setting.RepoRootPath) != 0) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) diff --git a/models/migrations/migrations_test.go b/models/migrations/migrations_test.go index a17eba54e82be..a1fd49a8b9f20 100644 --- a/models/migrations/migrations_test.go +++ b/models/migrations/migrations_test.go @@ -203,7 +203,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En deferFn := PrintCurrentTest(t, ourSkip) assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), + assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { diff --git a/models/unittest/fscopy.go b/models/unittest/fscopy.go new file mode 100644 index 0000000000000..ff815e729d476 --- /dev/null +++ b/models/unittest/fscopy.go @@ -0,0 +1,103 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package unittest + +import ( + "errors" + "io" + "os" + "path" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +// Copy copies file from source to target path. +func Copy(src, dest string) error { + // Gather file information to set back later. + si, err := os.Lstat(src) + if err != nil { + return err + } + + // Handle symbolic link. + if si.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(src) + if err != nil { + return err + } + // NOTE: os.Chmod and os.Chtimes don't recognize symbolic link, + // which will lead "no such file or directory" error. + return os.Symlink(target, dest) + } + + sr, err := os.Open(src) + if err != nil { + return err + } + defer sr.Close() + + dw, err := os.Create(dest) + if err != nil { + return err + } + defer dw.Close() + + if _, err = io.Copy(dw, sr); err != nil { + return err + } + + // Set back file information. + if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil { + return err + } + return os.Chmod(dest, si.Mode()) +} + +// CopyDir copy files recursively from source to target directory. +// +// The filter accepts a function that process the path info. +// and should return true for need to filter. +// +// It returns error when error occurs in underlying functions. +func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error { + // Check if target directory exists. + if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) { + return errors.New("file or directory already exists: " + destPath) + } + + err := os.MkdirAll(destPath, os.ModePerm) + if err != nil { + return err + } + + // Gather directory info. + infos, err := util.StatDir(srcPath, true) + if err != nil { + return err + } + + var filter func(filePath string) bool + if len(filters) > 0 { + filter = filters[0] + } + + for _, info := range infos { + if filter != nil && filter(info) { + continue + } + + curPath := path.Join(destPath, info) + if strings.HasSuffix(info, "/") { + err = os.MkdirAll(curPath, os.ModePerm) + } else { + err = Copy(path.Join(srcPath, info), curPath) + } + if err != nil { + return err + } + } + return nil +} diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index b6469fb309c22..b6924d4706808 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -104,7 +104,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string, fixtureFiles ...string) { if err = util.RemoveAll(repoRootPath); err != nil { fatalTestError("util.RemoveAll: %v\n", err) } - if err = util.CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { + if err = CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { fatalTestError("util.CopyDir: %v\n", err) } @@ -175,7 +175,7 @@ func PrepareTestEnv(t testing.TB) { assert.NoError(t, PrepareTestDatabase()) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) metaPath := filepath.Join(giteaRoot, "integrations", "gitea-repositories-meta") - assert.NoError(t, util.CopyDir(metaPath, setting.RepoRootPath)) + assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) assert.NoError(t, err) diff --git a/modules/cache/cache_redis.go b/modules/cache/cache_redis.go index e4b9a70f63f3a..ff6c8d424c149 100644 --- a/modules/cache/cache_redis.go +++ b/modules/cache/cache_redis.go @@ -6,11 +6,11 @@ package cache import ( "fmt" + "strconv" "time" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/nosql" - "code.gitea.io/gitea/modules/util" "gitea.com/go-chi/cache" "github.com/go-redis/redis/v8" @@ -24,20 +24,37 @@ type RedisCacher struct { occupyMode bool } -// Put puts value into cache with key and expire time. +// toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally +func toStr(v interface{}) string { + if v == nil { + return "" + } + switch v := v.(type) { + case string: + return v + case []byte: + return string(v) + case int: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + default: + return fmt.Sprint(v) // as what the old com.ToStr does in most cases + } +} + +// Put puts value (string type) into cache with key and expire time. // If expired is 0, it lives forever. func (c *RedisCacher) Put(key string, val interface{}, expire int64) error { + // this function is not well-designed, it only puts string values into cache key = c.prefix + key if expire == 0 { - if err := c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), 0).Err(); err != nil { + if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil { return err } } else { - dur, err := time.ParseDuration(util.ToStr(expire) + "s") - if err != nil { - return err - } - if err = c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), dur).Err(); err != nil { + dur := time.Duration(expire) * time.Second + if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil { return err } } diff --git a/modules/context/csrf.go b/modules/context/csrf.go index 1fb992e2ae409..4fc92705048c9 100644 --- a/modules/context/csrf.go +++ b/modules/context/csrf.go @@ -22,8 +22,10 @@ import ( "encoding/base32" "fmt" "net/http" + "strconv" "time" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" @@ -215,9 +217,16 @@ func Csrfer(opt CsrfOptions, ctx *Context) CSRF { } x.ID = "0" - uid := ctx.Session.Get(opt.SessionKey) - if uid != nil { - x.ID = util.ToStr(uid) + uidAny := ctx.Session.Get(opt.SessionKey) + if uidAny != nil { + switch uidVal := uidAny.(type) { + case string: + x.ID = uidVal + case int64: + x.ID = strconv.FormatInt(uidVal, 10) + default: + log.Error("invalid uid type in session: %T", uidAny) + } } needsNew := false diff --git a/modules/nosql/manager.go b/modules/nosql/manager.go index dab30812ce73f..93338fdc3f958 100644 --- a/modules/nosql/manager.go +++ b/modules/nosql/manager.go @@ -11,6 +11,7 @@ import ( "time" "code.gitea.io/gitea/modules/process" + "github.com/go-redis/redis/v8" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/modules/sync/unique_queue.go b/modules/sync/unique_queue.go index 414cc50f39a76..df115d7c96cc9 100644 --- a/modules/sync/unique_queue.go +++ b/modules/sync/unique_queue.go @@ -5,8 +5,6 @@ package sync -import "code.gitea.io/gitea/modules/util" - // UniqueQueue is a queue which guarantees only one instance of same // identity is in the line. Instances with same identity will be // discarded if there is already one in the line. @@ -53,10 +51,10 @@ func (q *UniqueQueue) IsClosed() <-chan struct{} { } // IDs returns the current ids in the pool -func (q *UniqueQueue) IDs() []interface{} { +func (q *UniqueQueue) IDs() []string { q.table.lock.Lock() defer q.table.lock.Unlock() - ids := make([]interface{}, 0, len(q.table.pool)) + ids := make([]string, 0, len(q.table.pool)) for id := range q.table.pool { ids = append(ids, id) } @@ -70,20 +68,19 @@ func (q *UniqueQueue) Queue() <-chan string { // Exist returns true if there is an instance with given identity // exists in the queue. -func (q *UniqueQueue) Exist(id interface{}) bool { - return q.table.IsRunning(util.ToStr(id)) +func (q *UniqueQueue) Exist(id string) bool { + return q.table.IsRunning(id) } // AddFunc adds new instance to the queue with a custom runnable function, // the queue is blocked until the function exits. -func (q *UniqueQueue) AddFunc(id interface{}, fn func()) { - idStr := util.ToStr(id) +func (q *UniqueQueue) AddFunc(id string, fn func()) { q.table.lock.Lock() - if _, ok := q.table.pool[idStr]; ok { + if _, ok := q.table.pool[id]; ok { q.table.lock.Unlock() return } - q.table.pool[idStr] = struct{}{} + q.table.pool[id] = struct{}{} if fn != nil { fn() } @@ -91,17 +88,17 @@ func (q *UniqueQueue) AddFunc(id interface{}, fn func()) { select { case <-q.closed: return - case q.queue <- idStr: + case q.queue <- id: return } } // Add adds new instance to the queue. -func (q *UniqueQueue) Add(id interface{}) { +func (q *UniqueQueue) Add(id string) { q.AddFunc(id, nil) } // Remove removes instance from the queue. -func (q *UniqueQueue) Remove(id interface{}) { - q.table.Stop(util.ToStr(id)) +func (q *UniqueQueue) Remove(id string) { + q.table.Stop(id) } diff --git a/modules/util/legacy.go b/modules/util/legacy.go index c7da5415349a0..d319faad09848 100644 --- a/modules/util/legacy.go +++ b/modules/util/legacy.go @@ -9,29 +9,37 @@ import ( "crypto/cipher" "crypto/rand" "errors" - - "github.com/unknwon/com" //nolint:depguard + "io" + "os" ) // CopyFile copies file from source to target path. func CopyFile(src, dest string) error { - return com.Copy(src, dest) -} + si, err := os.Lstat(src) + if err != nil { + return err + } -// CopyDir copy files recursively from source to target directory. -// It returns error when error occurs in underlying functions. -func CopyDir(srcPath, destPath string) error { - return com.CopyDir(srcPath, destPath) -} + sr, err := os.Open(src) + if err != nil { + return err + } + defer sr.Close() -// ToStr converts any interface to string. should be replaced. -func ToStr(value interface{}, args ...int) string { - return com.ToStr(value, args...) -} + dw, err := os.Create(dest) + if err != nil { + return err + } + defer dw.Close() -// ToSnakeCase converts a string to snake_case. should be replaced. -func ToSnakeCase(str string) string { - return com.ToSnakeCase(str) + if _, err = io.Copy(dw, sr); err != nil { + return err + } + + if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil { + return err + } + return os.Chmod(dest, si.Mode()) } // AESGCMEncrypt (from legacy package): encrypts plaintext with the given key using AES in GCM mode. should be replaced. diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go index cfda93d3ad34b..c41f7a008c224 100644 --- a/modules/util/legacy_test.go +++ b/modules/util/legacy_test.go @@ -7,12 +7,38 @@ package util import ( "crypto/aes" "crypto/rand" + "fmt" + "os" "testing" + "time" "github.com/stretchr/testify/assert" - "github.com/unknwon/com" //nolint:depguard ) +func TestCopyFile(t *testing.T) { + testContent := []byte("hello") + + tmpDir := os.TempDir() + now := time.Now() + srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro()) + dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro()) + + _ = os.Remove(srcFile) + _ = os.Remove(dstFile) + defer func() { + _ = os.Remove(srcFile) + _ = os.Remove(dstFile) + }() + + err := os.WriteFile(srcFile, testContent, 0o777) + assert.NoError(t, err) + err = CopyFile(srcFile, dstFile) + assert.NoError(t, err) + dstContent, err := os.ReadFile(dstFile) + assert.NoError(t, err) + assert.Equal(t, testContent, dstContent) +} + func TestAESGCM(t *testing.T) { t.Parallel() @@ -29,9 +55,4 @@ func TestAESGCM(t *testing.T) { assert.NoError(t, err) assert.Equal(t, plaintext, decrypted) - - // at the moment, we make sure the result is the same as the legacy package, this assertion can be removed in next round refactoring - legacy, err := com.AESGCMDecrypt(key, ciphertext) - assert.NoError(t, err) - assert.Equal(t, legacy, plaintext) } diff --git a/modules/util/string.go b/modules/util/string.go new file mode 100644 index 0000000000000..4301f75f99c91 --- /dev/null +++ b/modules/util/string.go @@ -0,0 +1,88 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import "github.com/yuin/goldmark/util" + +func isSnakeCaseUpper(c byte) bool { + return 'A' <= c && c <= 'Z' +} + +func isSnakeCaseLowerOrNumber(c byte) bool { + return 'a' <= c && c <= 'z' || '0' <= c && c <= '9' +} + +// ToSnakeCase convert the input string to snake_case format. +// +// Some samples. +// "FirstName" => "first_name" +// "HTTPServer" => "http_server" +// "NoHTTPS" => "no_https" +// "GO_PATH" => "go_path" +// "GO PATH" => "go_path" // space is converted to underscore. +// "GO-PATH" => "go_path" // hyphen is converted to underscore. +// +func ToSnakeCase(input string) string { + if len(input) == 0 { + return "" + } + + var res []byte + if len(input) == 1 { + c := input[0] + if isSnakeCaseUpper(c) { + res = []byte{c + 'a' - 'A'} + } else if isSnakeCaseLowerOrNumber(c) { + res = []byte{c} + } else { + res = []byte{'_'} + } + } else { + res = make([]byte, 0, len(input)*4/3) + pos := 0 + needSep := false + for pos < len(input) { + c := input[pos] + if c >= 0x80 { + res = append(res, c) + pos++ + continue + } + isUpper := isSnakeCaseUpper(c) + if isUpper || isSnakeCaseLowerOrNumber(c) { + end := pos + 1 + if isUpper { + // skip the following upper letters + for end < len(input) && isSnakeCaseUpper(input[end]) { + end++ + } + if end-pos > 1 && end < len(input) && isSnakeCaseLowerOrNumber(input[end]) { + end-- + } + } + // skip the following lower or number letters + for end < len(input) && (isSnakeCaseLowerOrNumber(input[end]) || input[end] >= 0x80) { + end++ + } + if needSep { + res = append(res, '_') + } + res = append(res, input[pos:end]...) + pos = end + needSep = true + } else { + res = append(res, '_') + pos++ + needSep = false + } + } + for i := 0; i < len(res); i++ { + if isSnakeCaseUpper(res[i]) { + res[i] += 'a' - 'A' + } + } + } + return util.BytesToReadOnlyString(res) +} diff --git a/modules/util/string_test.go b/modules/util/string_test.go new file mode 100644 index 0000000000000..49de29ab67a96 --- /dev/null +++ b/modules/util/string_test.go @@ -0,0 +1,48 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToSnakeCase(t *testing.T) { + cases := map[string]string{ + // all old cases from the legacy package + "HTTPServer": "http_server", + "_camelCase": "_camel_case", + "NoHTTPS": "no_https", + "Wi_thF": "wi_th_f", + "_AnotherTES_TCaseP": "_another_tes_t_case_p", + "ALL": "all", + "_HELLO_WORLD_": "_hello_world_", + "HELLO_WORLD": "hello_world", + "HELLO____WORLD": "hello____world", + "TW": "tw", + "_C": "_c", + + " sentence case ": "__sentence_case__", + " Mixed-hyphen case _and SENTENCE_case and UPPER-case": "_mixed_hyphen_case__and_sentence_case_and_upper_case", + + // new cases + " ": "_", + "A": "a", + "A0": "a0", + "a0": "a0", + "Aa0": "aa0", + "啊": "啊", + "A啊": "a啊", + "Aa啊b": "aa啊b", + "A啊B": "a啊_b", + "Aa啊B": "aa啊_b", + "TheCase2": "the_case2", + "ObjIDs": "obj_i_ds", // the strange database column name which already exists + } + for input, expected := range cases { + assert.Equal(t, expected, ToSnakeCase(input)) + } +} diff --git a/services/webhook/webhook.go b/services/webhook/webhook.go index 557dd147bfe8c..a3efc7535fc3c 100644 --- a/services/webhook/webhook.go +++ b/services/webhook/webhook.go @@ -6,6 +6,7 @@ package webhook import ( "fmt" + "strconv" "strings" repo_model "code.gitea.io/gitea/models/repo" @@ -106,7 +107,7 @@ func PrepareWebhook(w *webhook_model.Webhook, repo *repo_model.Repository, event return err } - go hookQueue.Add(repo.ID) + go hookQueue.Add(strconv.FormatInt(repo.ID, 10)) return nil } @@ -187,7 +188,7 @@ func PrepareWebhooks(repo *repo_model.Repository, event webhook_model.HookEventT return err } - go hookQueue.Add(repo.ID) + go hookQueue.Add(strconv.FormatInt(repo.ID, 10)) return nil } @@ -239,7 +240,7 @@ func ReplayHookTask(w *webhook_model.Webhook, uuid string) error { return err } - go hookQueue.Add(t.RepoID) + go hookQueue.Add(strconv.FormatInt(t.RepoID, 10)) return nil } From 43ff92e122550e2d7cc15e23402fc1a7c8e39e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Gardfj=C3=A4ll?= Date: Fri, 1 Apr 2022 20:29:57 +0200 Subject: [PATCH 06/10] An attempt to sync a non-mirror repo must give 400 (Bad Request) (#19300) * An attempt to sync a non-mirror repo must give 400 (Bad Request) * add missing return statement Co-authored-by: Lunny Xiao Co-authored-by: techknowlogick --- integrations/api_repo_test.go | 21 +++++++++++++++++++++ routers/api/v1/repo/mirror.go | 11 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/integrations/api_repo_test.go b/integrations/api_repo_test.go index ce1ecb1d43d8a..b585ad15e3a1e 100644 --- a/integrations/api_repo_test.go +++ b/integrations/api_repo_test.go @@ -405,6 +405,27 @@ func testAPIRepoMigrateConflict(t *testing.T, u *url.URL) { }) } +// mirror-sync must fail with "400 (Bad Request)" when an attempt is made to +// sync a non-mirror repository. +func TestAPIMirrorSyncNonMirrorRepo(t *testing.T) { + defer prepareTestEnv(t)() + + session := loginUser(t, "user2") + token := getTokenForLoggedInUser(t, session) + + var repo api.Repository + req := NewRequest(t, "GET", "/api/v1/repos/user2/repo1") + resp := MakeRequest(t, req, http.StatusOK) + DecodeJSON(t, resp, &repo) + assert.EqualValues(t, false, repo.Mirror) + + req = NewRequestf(t, "POST", "/api/v1/repos/user2/repo1/mirror-sync?token=%s", token) + resp = session.MakeRequest(t, req, http.StatusBadRequest) + errRespJSON := map[string]string{} + DecodeJSON(t, resp, &errRespJSON) + assert.Equal(t, "Repository is not a mirror", errRespJSON["message"]) +} + func TestAPIOrgRepoCreate(t *testing.T) { testCases := []struct { ctxUserID int64 diff --git a/routers/api/v1/repo/mirror.go b/routers/api/v1/repo/mirror.go index c9ac3e829230e..d7facd24d96cc 100644 --- a/routers/api/v1/repo/mirror.go +++ b/routers/api/v1/repo/mirror.go @@ -5,8 +5,10 @@ package repo import ( + "errors" "net/http" + repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" @@ -48,6 +50,15 @@ func MirrorSync(ctx *context.APIContext) { return } + if _, err := repo_model.GetMirrorByRepoID(repo.ID); err != nil { + if errors.Is(err, repo_model.ErrMirrorNotExist) { + ctx.Error(http.StatusBadRequest, "MirrorSync", "Repository is not a mirror") + return + } + ctx.Error(http.StatusInternalServerError, "MirrorSync", err) + return + } + mirror_service.StartToMirror(repo.ID) ctx.Status(http.StatusOK) From cf5d4a7230e716f77a54a1591e90b34b21b2a7b8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 2 Apr 2022 04:14:14 +0800 Subject: [PATCH 07/10] Upgrade xorm/builder from v0.3.9 to v0.3.10 (#19296) xorm/builder v0.3.10 add support to EXISTS and NOT EXISTS. --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index da2f1e03dd8ac..957e8fc68b3f5 100644 --- a/go.mod +++ b/go.mod @@ -100,7 +100,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 mvdan.cc/xurls/v2 v2.4.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 - xorm.io/builder v0.3.9 + xorm.io/builder v0.3.10 xorm.io/xorm v1.2.5 ) diff --git a/go.sum b/go.sum index d8624a28e67ff..d43cc56e8d6cd 100644 --- a/go.sum +++ b/go.sum @@ -2353,7 +2353,8 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 h1:mUcz5b3FJbP5Cvdq7Khzn6J9OCUQJaBwgBkCR+MOwSs= strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1:FJGmPh3vz9jSos1L/F91iAgnC/aejc0wIIrF2ZwJxdY= -xorm.io/builder v0.3.9 h1:Sd65/LdWyO7LR8+Cbd+e7mm3sK/7U9k0jS3999IDHMc= xorm.io/builder v0.3.9/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= +xorm.io/builder v0.3.10 h1:Rvkncad3Lo9YIVqCbgIf6QnpR/HcW3IEr0AANNpuyMQ= +xorm.io/builder v0.3.10/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= xorm.io/xorm v1.2.5 h1:tqN7OhN8P9xi52qBb76I8m5maAJMz/SSbgK2RGPCPbo= xorm.io/xorm v1.2.5/go.mod h1:fTG8tSjk6O1BYxwuohZUK+S1glnRycsCF05L1qQyEU0= From 7b4c3c7bb12504f107122ce799948daf1e36b3e8 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 2 Apr 2022 08:59:04 +0100 Subject: [PATCH 08/10] Prevent intermittent NPE in queue tests (#19301) There appears to be an intermittent NPE in queue tests relating to the deferred shutdown/terminate functions. This PR more formally asserts that shutdown and termination occurs before starting and finishing the tests but leaves the defer in place to ensure that if there is an issue shutdown/termination will occur. Signed-off-by: Andrew Thornton --- modules/queue/queue_channel_test.go | 49 ++++++++++++--- modules/queue/queue_disk_channel_test.go | 76 ++++++++++++++++++------ 2 files changed, 97 insertions(+), 28 deletions(-) diff --git a/modules/queue/queue_channel_test.go b/modules/queue/queue_channel_test.go index d30b90886151f..949c4528931e7 100644 --- a/modules/queue/queue_channel_test.go +++ b/modules/queue/queue_channel_test.go @@ -128,6 +128,8 @@ func TestChannelQueue_Pause(t *testing.T) { queueShutdown := []func(){} queueTerminate := []func(){} + terminated := make(chan struct{}) + queue, err = NewChannelQueue(handle, ChannelQueueConfiguration{ WorkerPoolConfiguration: WorkerPoolConfiguration{ @@ -142,15 +144,18 @@ func TestChannelQueue_Pause(t *testing.T) { }, &testData{}) assert.NoError(t, err) - go queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) + go func() { + queue.Run(func(shutdown func()) { + lock.Lock() + defer lock.Unlock() + queueShutdown = append(queueShutdown, shutdown) + }, func(terminate func()) { + lock.Lock() + defer lock.Unlock() + queueTerminate = append(queueTerminate, terminate) + }) + close(terminated) + }() // Shutdown and Terminate in defer defer func() { @@ -278,4 +283,30 @@ func TestChannelQueue_Pause(t *testing.T) { } assert.Equal(t, test1.TestString, result1.TestString) assert.Equal(t, test1.TestInt, result1.TestInt) + + lock.Lock() + callbacks := make([]func(), len(queueShutdown)) + copy(callbacks, queueShutdown) + queueShutdown = queueShutdown[:0] + lock.Unlock() + // Now shutdown the queue + for _, callback := range callbacks { + callback() + } + + // terminate the queue + lock.Lock() + callbacks = make([]func(), len(queueTerminate)) + copy(callbacks, queueTerminate) + queueShutdown = queueTerminate[:0] + lock.Unlock() + for _, callback := range callbacks { + callback() + } + select { + case <-terminated: + case <-time.After(10 * time.Second): + assert.Fail(t, "Queue should have terminated") + return + } } diff --git a/modules/queue/queue_disk_channel_test.go b/modules/queue/queue_disk_channel_test.go index f092bb1f56872..22b4f0f452ef4 100644 --- a/modules/queue/queue_disk_channel_test.go +++ b/modules/queue/queue_disk_channel_test.go @@ -221,6 +221,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) { queueShutdown := []func(){} queueTerminate := []func(){} + terminated := make(chan struct{}) tmpDir, err := os.MkdirTemp("", "persistable-channel-queue-pause-test-data") assert.NoError(t, err) @@ -237,15 +238,18 @@ func TestPersistableChannelQueue_Pause(t *testing.T) { }, &testData{}) assert.NoError(t, err) - go queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) + go func() { + queue.Run(func(shutdown func()) { + lock.Lock() + defer lock.Unlock() + queueShutdown = append(queueShutdown, shutdown) + }, func(terminate func()) { + lock.Lock() + defer lock.Unlock() + queueTerminate = append(queueTerminate, terminate) + }) + close(terminated) + }() // Shutdown and Terminate in defer defer func() { @@ -417,7 +421,10 @@ func TestPersistableChannelQueue_Pause(t *testing.T) { case <-handleChan: assert.Fail(t, "Handler processing should have stopped") return - default: + case <-terminated: + case <-time.After(10 * time.Second): + assert.Fail(t, "Queue should have terminated") + return } lock.Lock() @@ -425,6 +432,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) { lock.Unlock() // Reopen queue + terminated = make(chan struct{}) queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ DataDir: tmpDir, BatchLength: 1, @@ -442,15 +450,18 @@ func TestPersistableChannelQueue_Pause(t *testing.T) { paused, _ = pausable.IsPausedIsResumed() - go queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) + go func() { + queue.Run(func(shutdown func()) { + lock.Lock() + defer lock.Unlock() + queueShutdown = append(queueShutdown, shutdown) + }, func(terminate func()) { + lock.Lock() + defer lock.Unlock() + queueTerminate = append(queueTerminate, terminate) + }) + close(terminated) + }() select { case <-handleChan: @@ -510,4 +521,31 @@ func TestPersistableChannelQueue_Pause(t *testing.T) { assert.Equal(t, test2.TestString, result4.TestString) assert.Equal(t, test2.TestInt, result4.TestInt) + + lock.Lock() + callbacks = make([]func(), len(queueShutdown)) + copy(callbacks, queueShutdown) + queueShutdown = queueShutdown[:0] + lock.Unlock() + // Now shutdown the queue + for _, callback := range callbacks { + callback() + } + + // terminate the queue + lock.Lock() + callbacks = make([]func(), len(queueTerminate)) + copy(callbacks, queueTerminate) + queueShutdown = queueTerminate[:0] + lock.Unlock() + for _, callback := range callbacks { + callback() + } + + select { + case <-time.After(10 * time.Second): + assert.Fail(t, "Queue should have terminated") + return + case <-terminated: + } } From 27c34dd011cceb8232d1c3307f87b53a147c75c3 Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Sun, 3 Apr 2022 00:17:41 +0000 Subject: [PATCH 09/10] [skip ci] Updated translations via Crowdin --- options/locale/locale_zh-CN.ini | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 2b4936bacc41e..62a1967748e03 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1181,6 +1181,7 @@ projects.board.deletion_desc=删除项目看板会将所有相关问题移至“ projects.board.color=颜色 projects.open=开启 projects.close=关闭 +projects.board.assigned_to=指派给 issues.desc=组织 bug 报告、任务和里程碑。 issues.filter_assignees=筛选指派人 @@ -1373,6 +1374,8 @@ issues.lock.reason=锁定原因 issues.lock.title=锁定有关此问题的对话。 issues.unlock.title=解锁有关此问题的对话。 issues.comment_on_locked=您不能对锁定的问题发表评论。 +issues.delete=删除 +issues.delete.title=是否删除工单? issues.tracker=时间跟踪 issues.start_tracking_short=启动计时器 issues.start_tracking=开始时间跟踪 @@ -1413,6 +1416,8 @@ issues.due_date_remove=到期时间 %s %s 已删除 issues.due_date_overdue=过期 issues.due_date_invalid=到期日期无效或超出范围。请使用 'yyyy-mm-dd' 格式。 issues.dependency.title=依赖工单 +issues.dependency.issue_no_dependencies=没有设置依赖项。 +issues.dependency.pr_no_dependencies=没有设置依赖项。 issues.dependency.add=添加依赖工单... issues.dependency.cancel=取消 issues.dependency.remove=删除 @@ -1942,6 +1947,7 @@ settings.event_pull_request_review=已审核的合并请求 settings.event_pull_request_review_desc=合并请求被批准、拒绝或提出审查意见 settings.event_pull_request_sync=合并请求被同步 settings.event_pull_request_sync_desc=合并请求被同步。 +settings.event_package=软件包 settings.branch_filter=分支过滤 settings.branch_filter_desc=推送、创建，删除分支事件的分支白名单，使用 glob 模式匹配指定。若为空或 *，则将报告所有分支的事件。语法文档见 github.com/gobwas/glob。示例：master,{master,release*}。 settings.active=激活 @@ -2537,6 +2543,13 @@ repos.forks=派生数 repos.issues=工单数 repos.size=大小 +packages.owner=所有者 +packages.creator=创建者 +packages.name=名称 +packages.version=版本 +packages.type=类型 +packages.repository=仓库 +packages.size=大小 defaulthooks=默认Web钩子 defaulthooks.desc=当某些 Gitea 事件触发时，Web 钩子自动向服务器发出 HTTP POST 请求。这里定义的 Web 钩子是默认配置，将被复制到所有新的仓库中。详情请访问 Web 钩子指南。 @@ -2975,4 +2988,57 @@ error.no_unit_allowed_repo=您没有被允许访问此仓库的任何单元。 error.unit_not_allowed=您没有权限访问此仓库单元 [packages] +title=软件包 +filter.type=类型 +filter.type.all=所有 +details=详情 +details.author=作者 +details.project_site=项目站点 +details.license=许可协议 +container.multi_arch=OS / Arch +container.layers=镜像层 +container.labels=标签 +container.labels.key=键 +container.labels.value=值 +generic.download=从命令行下载软件包： +generic.documentation=关于通用注册中心的更多信息，请参阅 文档 。 +maven.registry=在您项目的 pom.xml 文件中设置此注册中心： +maven.install=要使用这个软件包，在 pom.xml 文件中的 依赖项 块中包含以下内容： +maven.install2=通过命令行运行： +maven.download=要下载依赖项，请通过命令行运行： +maven.documentation=关于 Maven 注册中心的更多信息，请参阅 文档。 +nuget.registry=从命令行设置此注册中心： +nuget.install=要使用 Nuget 安装软件包，请运行以下命令： +nuget.documentation=关于 Nuget 注册中心的更多信息，请参阅 文档。 +nuget.dependency.framework=目标框架 +npm.registry=在您项目的 .npmrc 文件中设置此注册中心： +npm.install=要使用 npm 安装软件包，请运行以下命令： +npm.install2=或将其添加到 package.json 文件： +npm.documentation=关于 npm 注册中心的更多信息，请参阅 文档。 +npm.dependencies=依赖项 +npm.dependencies.development=开发依赖 +npm.dependencies.peer=Peer 依赖 +npm.dependencies.optional=可选依赖 +npm.details.tag=标签 +pypi.requires=需要 Python +pypi.install=要使用 pip 安装软件包，请运行以下命令： +pypi.documentation=关于 PyPI 注册中心的更多信息，请参阅 文档。 +rubygems.install=要使用 gem 安装软件包，请运行以下命令： +rubygems.install2=或将它添加到 Gemfile： +rubygems.dependencies.runtime=运行时依赖 +rubygems.dependencies.development=开发依赖 +rubygems.required.ruby=需要 Ruby 版本 +rubygems.required.rubygems=需要 RubyGem 版本 +rubygems.documentation=关于 RubyGems 注册中心的更多信息，请参阅 文档 。 +settings.link=将此软件包链接到仓库 +settings.link.description=如果您将一个软件包与一个代码库链接起来，软件包将显示在代码库的软件包列表中。 +settings.link.select=选择仓库 +settings.link.button=更新仓库链接 +settings.link.success=仓库链接已成功更新。 +settings.link.error=更新仓库链接失败。 +settings.delete=删除软件包 +settings.delete.description=删除软件包是永久性的，无法撤消。 +settings.delete.notice=您将要删除 %s (%s)。此操作是不可逆的，您确定吗？ +settings.delete.success=软件包已被删除。 +settings.delete.error=删除软件包失败。 From d242511e86c3a6d8a7013100845d2cdc8eb5252c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 3 Apr 2022 17:46:48 +0800 Subject: [PATCH 10/10] Remove legacy unmaintained packages, refactor to support change default locale (#19308) Remove two unmaintained vendor packages `i18n` and `paginater`. Changes: * Rewrite `i18n` package with a more clear fallback mechanism. Fix an unstable `Tr` behavior, add more tests. * Refactor the legacy `Paginater` to `Paginator`, test cases are kept unchanged. Trivial enhancement (no breaking for end users): * Use the first locale in LANGS setting option as the default, add a log to prevent from surprising users. --- custom/conf/app.example.ini | 1 + .../doc/advanced/config-cheat-sheet.en-us.md | 3 +- .../doc/advanced/customizing-gitea.en-us.md | 2 + go.mod | 2 - go.sum | 5 - integrations/auth_ldap_test.go | 2 +- integrations/branches_test.go | 3 +- integrations/pull_merge_test.go | 2 +- integrations/release_test.go | 2 +- integrations/repo_branch_test.go | 2 +- integrations/signin_test.go | 2 +- integrations/signup_test.go | 8 +- integrations/user_test.go | 2 +- modules/context/pagination.go | 9 +- modules/markup/markdown/toc.go | 3 +- modules/paginator/paginator.go | 203 ++++++++++++ modules/paginator/paginator_test.go | 311 ++++++++++++++++++ modules/timeutil/since.go | 3 +- modules/timeutil/since_test.go | 2 +- modules/translation/i18n/i18n.go | 143 ++++++++ modules/translation/i18n/i18n_test.go | 56 ++++ modules/translation/translation.go | 21 +- modules/web/middleware/locale.go | 6 +- routers/web/repo/issue_content_history.go | 2 +- routers/web/user/setting/profile.go | 3 +- services/cron/setting.go | 2 +- 26 files changed, 758 insertions(+), 42 deletions(-) create mode 100644 modules/paginator/paginator.go create mode 100644 modules/paginator/paginator_test.go create mode 100644 modules/translation/i18n/i18n.go create mode 100644 modules/translation/i18n/i18n_test.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 8cc22f1d14a5d..822be00baeb24 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2117,6 +2117,7 @@ PATH = ;[i18n] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; The first locale will be used as the default if user browser's language doesn't match any locale in the list. ;LANGS = en-US,zh-CN,zh-HK,zh-TW,de-DE,fr-FR,nl-NL,lv-LV,ru-RU,uk-UA,ja-JP,es-ES,pt-BR,pt-PT,pl-PL,bg-BG,it-IT,fi-FI,tr-TR,cs-CZ,sr-SP,sv-SE,ko-KR,el-GR,fa-IR,hu-HU,id-ID,ml-IN ;NAMES = English,简体中文,繁體中文（香港）,繁體中文（台灣）,Deutsch,français,Nederlands,latviešu,русский,Українська,日本語,español,português do Brasil,Português de Portugal,polski,български,italiano,suomi,Türkçe,čeština,српски,svenska,한국어,ελληνικά,فارسی,magyar nyelv,bahasa Indonesia,മലയാളം diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 25247a6805a89..b3c015cb88dd7 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -997,7 +997,8 @@ Default templates for project boards: ## i18n (`i18n`) -- `LANGS`: **en-US,zh-CN,zh-HK,zh-TW,de-DE,fr-FR,nl-NL,lv-LV,ru-RU,ja-JP,es-ES,pt-BR,pt-PT,pl-PL,bg-BG,it-IT,fi-FI,tr-TR,cs-CZ,sr-SP,sv-SE,ko-KR,el-GR,fa-IR,hu-HU,id-ID,ml-IN**: List of locales shown in language selector +- `LANGS`: **en-US,zh-CN,zh-HK,zh-TW,de-DE,fr-FR,nl-NL,lv-LV,ru-RU,ja-JP,es-ES,pt-BR,pt-PT,pl-PL,bg-BG,it-IT,fi-FI,tr-TR,cs-CZ,sr-SP,sv-SE,ko-KR,el-GR,fa-IR,hu-HU,id-ID,ml-IN**: + List of locales shown in language selector. The first locale will be used as the default if user browser's language doesn't match any locale in the list. - `NAMES`: **English,简体中文,繁體中文（香港）,繁體中文（台灣）,Deutsch,français,Nederlands,latviešu,русский,日本語,español,português do Brasil,Português de Portugal,polski,български,italiano,suomi,Türkçe,čeština,српски,svenska,한국어,ελληνικά,فارسی,magyar nyelv,bahasa Indonesia,മലയാളം**: Visible names corresponding to the locales ## U2F (`U2F`) **DEPRECATED** diff --git a/docs/content/doc/advanced/customizing-gitea.en-us.md b/docs/content/doc/advanced/customizing-gitea.en-us.md index 39a08308b8ede..1a8386fc3e1d0 100644 --- a/docs/content/doc/advanced/customizing-gitea.en-us.md +++ b/docs/content/doc/advanced/customizing-gitea.en-us.md @@ -299,6 +299,8 @@ LANGS = en-US,foo-BAR NAMES = English,FooBar ``` +The first locale will be used as the default if user browser's language doesn't match any locale in the list. + Locales may change between versions, so keeping track of your customized locales is highly encouraged. ### Readmes diff --git a/go.mod b/go.mod index 957e8fc68b3f5..bfb87a1b37a8b 100644 --- a/go.mod +++ b/go.mod @@ -78,8 +78,6 @@ require ( github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 - github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361 - github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae github.com/unrolled/render v1.4.1 github.com/urfave/cli v1.22.5 github.com/xanzy/go-gitlab v0.58.0 diff --git a/go.sum b/go.sum index d43cc56e8d6cd..d969c26bf533d 100644 --- a/go.sum +++ b/go.sum @@ -1503,10 +1503,6 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs= github.com/unknwon/com v1.0.1/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= -github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361 h1:4Ij5sX4JEzCCY/CCl8trJHey1tPsIDomYTZf145GKk0= -github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361/go.mod h1:+5rDk6sDGpl3azws3O+f+GpFSyN9GVr0K8cvQLQM2ZQ= -github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae h1:ihaXiJkaca54IaCSnEXtE/uSZOmPxKZhDfVLrzZLFDs= -github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae/go.mod h1:1fdkY6xxl6ExVs2QFv7R0F5IRZHKA8RahhB9fMC9RvM= github.com/unrolled/render v1.4.1 h1:VdpMc2YkAOWzbmC/P2yoHhRDXgsaCQHcTJ1KK6SNCA4= github.com/unrolled/render v1.4.1/go.mod h1:cK4RSTTVdND5j9EYEc0LAMOvdG11JeiKjyjfyZRvV2w= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -2272,7 +2268,6 @@ gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AW gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.44.2/go.mod h1:M3Cogqpuv0QCi3ExAY5V4uOt4qb/R3xZubo9m8lK5wg= -gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/integrations/auth_ldap_test.go b/integrations/auth_ldap_test.go index 93334aec0094a..0eee5ae0cd905 100644 --- a/integrations/auth_ldap_test.go +++ b/integrations/auth_ldap_test.go @@ -16,10 +16,10 @@ import ( "code.gitea.io/gitea/models/organization" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/translation/i18n" "code.gitea.io/gitea/services/auth" "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) type ldapUser struct { diff --git a/integrations/branches_test.go b/integrations/branches_test.go index aa4df6ac6a333..551c5f8af8b5d 100644 --- a/integrations/branches_test.go +++ b/integrations/branches_test.go @@ -9,8 +9,9 @@ import ( "net/url" "testing" + "code.gitea.io/gitea/modules/translation/i18n" + "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) func TestViewBranches(t *testing.T) { diff --git a/integrations/pull_merge_test.go b/integrations/pull_merge_test.go index 4e063bbdb6f2e..d7cb042e9c127 100644 --- a/integrations/pull_merge_test.go +++ b/integrations/pull_merge_test.go @@ -24,10 +24,10 @@ import ( "code.gitea.io/gitea/modules/git" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/translation/i18n" "code.gitea.io/gitea/services/pull" "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) func testPullMerge(t *testing.T, session *TestSession, user, repo, pullnum string, mergeStyle repo_model.MergeStyle) *httptest.ResponseRecorder { diff --git a/integrations/release_test.go b/integrations/release_test.go index 88591a05e91a6..d75d74956e5e9 100644 --- a/integrations/release_test.go +++ b/integrations/release_test.go @@ -14,10 +14,10 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/translation/i18n" "github.com/PuerkitoBio/goquery" "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) func createNewRelease(t *testing.T, session *TestSession, repoURL, tag, title string, preRelease, draft bool) { diff --git a/integrations/repo_branch_test.go b/integrations/repo_branch_test.go index ade5d673b9c9c..30a446ccec1e7 100644 --- a/integrations/repo_branch_test.go +++ b/integrations/repo_branch_test.go @@ -13,9 +13,9 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/translation/i18n" "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) func testCreateBranch(t testing.TB, session *TestSession, user, repo, oldRefSubURL, newBranchName string, expectedStatus int) string { diff --git a/integrations/signin_test.go b/integrations/signin_test.go index a6e4b7d4d2f19..811f9326ecd7f 100644 --- a/integrations/signin_test.go +++ b/integrations/signin_test.go @@ -11,9 +11,9 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/translation/i18n" "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) func testLoginFailed(t *testing.T, username, password, message string) { diff --git a/integrations/signup_test.go b/integrations/signup_test.go index 87dea2fbe714e..7b456743766c7 100644 --- a/integrations/signup_test.go +++ b/integrations/signup_test.go @@ -13,9 +13,9 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation/i18n" "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) func TestSignup(t *testing.T) { @@ -68,9 +68,9 @@ func TestSignupEmail(t *testing.T) { wantStatus int wantMsg string }{ - {"exampleUser@example.com\r\n", http.StatusOK, i18n.Tr("en", "form.email_invalid", nil)}, - {"exampleUser@example.com\r", http.StatusOK, i18n.Tr("en", "form.email_invalid", nil)}, - {"exampleUser@example.com\n", http.StatusOK, i18n.Tr("en", "form.email_invalid", nil)}, + {"exampleUser@example.com\r\n", http.StatusOK, i18n.Tr("en", "form.email_invalid")}, + {"exampleUser@example.com\r", http.StatusOK, i18n.Tr("en", "form.email_invalid")}, + {"exampleUser@example.com\n", http.StatusOK, i18n.Tr("en", "form.email_invalid")}, {"exampleUser@example.com", http.StatusSeeOther, ""}, } diff --git a/integrations/user_test.go b/integrations/user_test.go index f7c9acb057dfb..e8fbccd51e8a0 100644 --- a/integrations/user_test.go +++ b/integrations/user_test.go @@ -11,9 +11,9 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/translation/i18n" "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) func TestViewUser(t *testing.T) { diff --git a/modules/context/pagination.go b/modules/context/pagination.go index 107cbf618637c..c0079c29505e1 100644 --- a/modules/context/pagination.go +++ b/modules/context/pagination.go @@ -10,19 +10,19 @@ import ( "net/url" "strings" - "github.com/unknwon/paginater" + "code.gitea.io/gitea/modules/paginator" ) -// Pagination provides a pagination via Paginater and additional configurations for the link params used in rendering +// Pagination provides a pagination via paginator.Paginator and additional configurations for the link params used in rendering type Pagination struct { - Paginater *paginater.Paginater + Paginater *paginator.Paginator urlParams []string } // NewPagination creates a new instance of the Pagination struct func NewPagination(total, page, issueNum, numPages int) *Pagination { p := &Pagination{} - p.Paginater = paginater.New(total, page, issueNum, numPages) + p.Paginater = paginator.New(total, page, issueNum, numPages) return p } @@ -53,5 +53,6 @@ func (p *Pagination) SetDefaultParams(ctx *Context) { p.AddParam(ctx, "sort", "SortType") p.AddParam(ctx, "q", "Keyword") p.AddParam(ctx, "tab", "TabName") + // do not add any more uncommon params here! p.AddParam(ctx, "t", "queryType") } diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go index 189821c341abe..9d11b771f7f3c 100644 --- a/modules/markup/markdown/toc.go +++ b/modules/markup/markdown/toc.go @@ -8,7 +8,8 @@ import ( "fmt" "net/url" - "github.com/unknwon/i18n" + "code.gitea.io/gitea/modules/translation/i18n" + "github.com/yuin/goldmark/ast" ) diff --git a/modules/paginator/paginator.go b/modules/paginator/paginator.go new file mode 100644 index 0000000000000..873cfe49d4657 --- /dev/null +++ b/modules/paginator/paginator.go @@ -0,0 +1,203 @@ +// Copyright 2022 The Gitea Authors. +// Copyright 2015 Unknwon. Licensed under the Apache License, Version 2.0 + +package paginator + +/* +In template: + +```html +{{if not .Page.IsFirst}}[First](1){{end}} +{{if .Page.HasPrevious}}[Previous]({{.Page.Previous}}){{end}} + +{{range .Page.Pages}} + {{if eq .Num -1}} + ... + {{else}} + {{.Num}}{{if .IsCurrent}}(current){{end}} + {{end}} +{{end}} + +{{if .Page.HasNext}}[Next]({{.Page.Next}}){{end}} +{{if not .Page.IsLast}}[Last]({{.Page.TotalPages}}){{end}} +``` + +Output: + +``` +[First](1) [Previous](2) ... 2 3(current) 4 ... [Next](4) [Last](5) +``` +*/ + +// Paginator represents a set of results of pagination calculations. +type Paginator struct { + total int // total rows count + pagingNum int // how many rows in one page + current int // current page number + numPages int // how many pages to show on the UI +} + +// New initialize a new pagination calculation and returns a Paginator as result. +func New(total, pagingNum, current, numPages int) *Paginator { + if pagingNum <= 0 { + pagingNum = 1 + } + if current <= 0 { + current = 1 + } + p := &Paginator{total, pagingNum, current, numPages} + if p.current > p.TotalPages() { + p.current = p.TotalPages() + } + return p +} + +// IsFirst returns true if current page is the first page. +func (p *Paginator) IsFirst() bool { + return p.current == 1 +} + +// HasPrevious returns true if there is a previous page relative to current page. +func (p *Paginator) HasPrevious() bool { + return p.current > 1 +} + +func (p *Paginator) Previous() int { + if !p.HasPrevious() { + return p.current + } + return p.current - 1 +} + +// HasNext returns true if there is a next page relative to current page. +func (p *Paginator) HasNext() bool { + return p.total > p.current*p.pagingNum +} + +func (p *Paginator) Next() int { + if !p.HasNext() { + return p.current + } + return p.current + 1 +} + +// IsLast returns true if current page is the last page. +func (p *Paginator) IsLast() bool { + if p.total == 0 { + return true + } + return p.total > (p.current-1)*p.pagingNum && !p.HasNext() +} + +// Total returns number of total rows. +func (p *Paginator) Total() int { + return p.total +} + +// TotalPages returns number of total pages. +func (p *Paginator) TotalPages() int { + if p.total == 0 { + return 1 + } + return (p.total + p.pagingNum - 1) / p.pagingNum +} + +// Current returns current page number. +func (p *Paginator) Current() int { + return p.current +} + +// PagingNum returns number of page size. +func (p *Paginator) PagingNum() int { + return p.pagingNum +} + +// Page presents a page in the paginator. +type Page struct { + num int + isCurrent bool +} + +func (p *Page) Num() int { + return p.num +} + +func (p *Page) IsCurrent() bool { + return p.isCurrent +} + +func getMiddleIdx(numPages int) int { + return (numPages + 1) / 2 +} + +// Pages returns a list of nearby page numbers relative to current page. +// If value is -1 means "..." that more pages are not showing. +func (p *Paginator) Pages() []*Page { + if p.numPages == 0 { + return []*Page{} + } else if p.numPages == 1 && p.TotalPages() == 1 { + // Only show current page. + return []*Page{{1, true}} + } + + // Total page number is less or equal. + if p.TotalPages() <= p.numPages { + pages := make([]*Page, p.TotalPages()) + for i := range pages { + pages[i] = &Page{i + 1, i+1 == p.current} + } + return pages + } + + numPages := p.numPages + offsetIdx := 0 + hasMoreNext := false + + // Check more previous and next pages. + previousNum := getMiddleIdx(p.numPages) - 1 + if previousNum > p.current-1 { + previousNum -= previousNum - (p.current - 1) + } + nextNum := p.numPages - previousNum - 1 + if p.current+nextNum > p.TotalPages() { + delta := nextNum - (p.TotalPages() - p.current) + nextNum -= delta + previousNum += delta + } + + offsetVal := p.current - previousNum + if offsetVal > 1 { + numPages++ + offsetIdx = 1 + } + + if p.current+nextNum < p.TotalPages() { + numPages++ + hasMoreNext = true + } + + pages := make([]*Page, numPages) + + // There are more previous pages. + if offsetIdx == 1 { + pages[0] = &Page{-1, false} + } + // There are more next pages. + if hasMoreNext { + pages[len(pages)-1] = &Page{-1, false} + } + + // Check previous pages. + for i := 0; i < previousNum; i++ { + pages[offsetIdx+i] = &Page{i + offsetVal, false} + } + + pages[offsetIdx+previousNum] = &Page{p.current, true} + + // Check next pages. + for i := 1; i <= nextNum; i++ { + pages[offsetIdx+previousNum+i] = &Page{p.current + i, false} + } + + return pages +} diff --git a/modules/paginator/paginator_test.go b/modules/paginator/paginator_test.go new file mode 100644 index 0000000000000..ce7b7275e1b91 --- /dev/null +++ b/modules/paginator/paginator_test.go @@ -0,0 +1,311 @@ +// Copyright 2022 The Gitea Authors. +// Copyright 2015 Unknwon. Licensed under the Apache License, Version 2.0 + +package paginator + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPaginator(t *testing.T) { + t.Run("Basic logics", func(t *testing.T) { + p := New(0, -1, -1, 0) + assert.Equal(t, 1, p.PagingNum()) + assert.True(t, p.IsFirst()) + assert.False(t, p.HasPrevious()) + assert.Equal(t, 1, p.Previous()) + assert.False(t, p.HasNext()) + assert.Equal(t, 1, p.Next()) + assert.True(t, p.IsLast()) + assert.Equal(t, 0, p.Total()) + + p = New(1, 10, 2, 0) + assert.Equal(t, 10, p.PagingNum()) + assert.True(t, p.IsFirst()) + assert.False(t, p.HasPrevious()) + assert.False(t, p.HasNext()) + assert.True(t, p.IsLast()) + + p = New(10, 10, 1, 0) + assert.Equal(t, 10, p.PagingNum()) + assert.True(t, p.IsFirst()) + assert.False(t, p.HasPrevious()) + assert.False(t, p.HasNext()) + assert.True(t, p.IsLast()) + + p = New(11, 10, 1, 0) + assert.Equal(t, 10, p.PagingNum()) + assert.True(t, p.IsFirst()) + assert.False(t, p.HasPrevious()) + assert.True(t, p.HasNext()) + assert.Equal(t, 2, p.Next()) + assert.False(t, p.IsLast()) + + p = New(11, 10, 2, 0) + assert.Equal(t, 10, p.PagingNum()) + assert.False(t, p.IsFirst()) + assert.True(t, p.HasPrevious()) + assert.Equal(t, 1, p.Previous()) + assert.False(t, p.HasNext()) + assert.True(t, p.IsLast()) + + p = New(20, 10, 2, 0) + assert.Equal(t, 10, p.PagingNum()) + assert.False(t, p.IsFirst()) + assert.True(t, p.HasPrevious()) + assert.False(t, p.HasNext()) + assert.True(t, p.IsLast()) + + p = New(25, 10, 2, 0) + assert.Equal(t, 10, p.PagingNum()) + assert.False(t, p.IsFirst()) + assert.True(t, p.HasPrevious()) + assert.True(t, p.HasNext()) + assert.False(t, p.IsLast()) + }) + + t.Run("Generate pages", func(t *testing.T) { + p := New(0, 10, 1, 0) + pages := p.Pages() + assert.Equal(t, 0, len(pages)) + }) + + t.Run("Only current page", func(t *testing.T) { + p := New(0, 10, 1, 1) + pages := p.Pages() + assert.Equal(t, 1, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.True(t, pages[0].IsCurrent()) + + p = New(1, 10, 1, 1) + pages = p.Pages() + assert.Equal(t, 1, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.True(t, pages[0].IsCurrent()) + }) + + t.Run("Total page number is less or equal", func(t *testing.T) { + p := New(1, 10, 1, 2) + pages := p.Pages() + assert.Equal(t, 1, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.True(t, pages[0].IsCurrent()) + + p = New(11, 10, 1, 2) + pages = p.Pages() + assert.Equal(t, 2, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.True(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.False(t, pages[1].IsCurrent()) + + p = New(11, 10, 2, 2) + pages = p.Pages() + assert.Equal(t, 2, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.True(t, pages[1].IsCurrent()) + + p = New(25, 10, 2, 3) + pages = p.Pages() + assert.Equal(t, 3, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.True(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + }) + + t.Run("Has more previous pages ", func(t *testing.T) { + // ... 2 + p := New(11, 10, 2, 1) + pages := p.Pages() + assert.Equal(t, 2, len(pages)) + assert.Equal(t, -1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.True(t, pages[1].IsCurrent()) + + // ... 2 3 + p = New(21, 10, 2, 2) + pages = p.Pages() + assert.Equal(t, 3, len(pages)) + assert.Equal(t, -1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.True(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + + // ... 2 3 4 + p = New(31, 10, 3, 3) + pages = p.Pages() + assert.Equal(t, 4, len(pages)) + assert.Equal(t, -1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.False(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.True(t, pages[2].IsCurrent()) + assert.Equal(t, 4, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + + // ... 3 4 5 + p = New(41, 10, 4, 3) + pages = p.Pages() + assert.Equal(t, 4, len(pages)) + assert.Equal(t, -1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 3, pages[1].Num()) + assert.False(t, pages[1].IsCurrent()) + assert.Equal(t, 4, pages[2].Num()) + assert.True(t, pages[2].IsCurrent()) + assert.Equal(t, 5, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + + // ... 4 5 6 7 8 9 10 + p = New(100, 10, 9, 7) + pages = p.Pages() + assert.Equal(t, 8, len(pages)) + assert.Equal(t, -1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 4, pages[1].Num()) + assert.False(t, pages[1].IsCurrent()) + assert.Equal(t, 5, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + assert.Equal(t, 6, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + assert.Equal(t, 7, pages[4].Num()) + assert.False(t, pages[4].IsCurrent()) + assert.Equal(t, 8, pages[5].Num()) + assert.False(t, pages[5].IsCurrent()) + assert.Equal(t, 9, pages[6].Num()) + assert.True(t, pages[6].IsCurrent()) + assert.Equal(t, 10, pages[7].Num()) + assert.False(t, pages[7].IsCurrent()) + }) + + t.Run("Has more next pages", func(t *testing.T) { + // 1 ... + p := New(21, 10, 1, 1) + pages := p.Pages() + assert.Equal(t, 2, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.True(t, pages[0].IsCurrent()) + assert.Equal(t, -1, pages[1].Num()) + assert.False(t, pages[1].IsCurrent()) + + // 1 2 ... + p = New(21, 10, 1, 2) + pages = p.Pages() + assert.Equal(t, 3, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.True(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.False(t, pages[1].IsCurrent()) + assert.Equal(t, -1, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + + // 1 2 3 ... + p = New(31, 10, 2, 3) + pages = p.Pages() + assert.Equal(t, 4, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.True(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + assert.Equal(t, -1, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + + // 1 2 3 ... + p = New(41, 10, 2, 3) + pages = p.Pages() + assert.Equal(t, 4, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.True(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + assert.Equal(t, -1, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + + // 1 2 3 4 5 6 7 ... + p = New(100, 10, 1, 7) + pages = p.Pages() + assert.Equal(t, 8, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.True(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.False(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + assert.Equal(t, 4, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + assert.Equal(t, 5, pages[4].Num()) + assert.False(t, pages[4].IsCurrent()) + assert.Equal(t, 6, pages[5].Num()) + assert.False(t, pages[5].IsCurrent()) + assert.Equal(t, 7, pages[6].Num()) + assert.False(t, pages[6].IsCurrent()) + assert.Equal(t, -1, pages[7].Num()) + assert.False(t, pages[7].IsCurrent()) + + // 1 2 3 4 5 6 7 ... + p = New(100, 10, 2, 7) + pages = p.Pages() + assert.Equal(t, 8, len(pages)) + assert.Equal(t, 1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.True(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + assert.Equal(t, 4, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + assert.Equal(t, 5, pages[4].Num()) + assert.False(t, pages[4].IsCurrent()) + assert.Equal(t, 6, pages[5].Num()) + assert.False(t, pages[5].IsCurrent()) + assert.Equal(t, 7, pages[6].Num()) + assert.False(t, pages[6].IsCurrent()) + assert.Equal(t, -1, pages[7].Num()) + assert.False(t, pages[7].IsCurrent()) + }) + + t.Run("Has both more previous and next pages", func(t *testing.T) { + // ... 2 3 ... + p := New(35, 10, 2, 2) + pages := p.Pages() + assert.Equal(t, 4, len(pages)) + assert.Equal(t, -1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.True(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.False(t, pages[2].IsCurrent()) + assert.Equal(t, -1, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + + // ... 2 3 4 ... + p = New(49, 10, 3, 3) + pages = p.Pages() + assert.Equal(t, 5, len(pages)) + assert.Equal(t, -1, pages[0].Num()) + assert.False(t, pages[0].IsCurrent()) + assert.Equal(t, 2, pages[1].Num()) + assert.False(t, pages[1].IsCurrent()) + assert.Equal(t, 3, pages[2].Num()) + assert.True(t, pages[2].IsCurrent()) + assert.Equal(t, 4, pages[3].Num()) + assert.False(t, pages[3].IsCurrent()) + assert.Equal(t, -1, pages[4].Num()) + assert.False(t, pages[4].IsCurrent()) + }) +} diff --git a/modules/timeutil/since.go b/modules/timeutil/since.go index c0240907ae8df..38b12829ad463 100644 --- a/modules/timeutil/since.go +++ b/modules/timeutil/since.go @@ -12,8 +12,7 @@ import ( "time" "code.gitea.io/gitea/modules/setting" - - "github.com/unknwon/i18n" + "code.gitea.io/gitea/modules/translation/i18n" ) // Seconds-based time units diff --git a/modules/timeutil/since_test.go b/modules/timeutil/since_test.go index 1379e71c3d67c..49951b6e4100b 100644 --- a/modules/timeutil/since_test.go +++ b/modules/timeutil/since_test.go @@ -12,9 +12,9 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/translation/i18n" "github.com/stretchr/testify/assert" - "github.com/unknwon/i18n" ) var BaseDate time.Time diff --git a/modules/translation/i18n/i18n.go b/modules/translation/i18n/i18n.go new file mode 100644 index 0000000000000..664e457ecf79f --- /dev/null +++ b/modules/translation/i18n/i18n.go @@ -0,0 +1,143 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package i18n + +import ( + "errors" + "fmt" + "reflect" + "strings" + + "code.gitea.io/gitea/modules/log" + + "gopkg.in/ini.v1" +) + +var ( + ErrLocaleAlreadyExist = errors.New("lang already exists") + + DefaultLocales = NewLocaleStore() +) + +type locale struct { + store *LocaleStore + langName string + langDesc string + messages *ini.File +} + +type LocaleStore struct { + // at the moment, all these fields are readonly after initialization + langNames []string + langDescs []string + localeMap map[string]*locale + defaultLang string +} + +func NewLocaleStore() *LocaleStore { + return &LocaleStore{localeMap: make(map[string]*locale)} +} + +// AddLocaleByIni adds locale by ini into the store +func (ls *LocaleStore) AddLocaleByIni(langName, langDesc string, localeFile interface{}, otherLocaleFiles ...interface{}) error { + if _, ok := ls.localeMap[langName]; ok { + return ErrLocaleAlreadyExist + } + iniFile, err := ini.LoadSources(ini.LoadOptions{ + IgnoreInlineComment: true, + UnescapeValueCommentSymbols: true, + }, localeFile, otherLocaleFiles...) + if err == nil { + iniFile.BlockMode = false + lc := &locale{store: ls, langName: langName, langDesc: langDesc, messages: iniFile} + ls.langNames = append(ls.langNames, lc.langName) + ls.langDescs = append(ls.langDescs, lc.langDesc) + ls.localeMap[lc.langName] = lc + } + return err +} + +func (ls *LocaleStore) HasLang(langName string) bool { + _, ok := ls.localeMap[langName] + return ok +} + +func (ls *LocaleStore) ListLangNameDesc() (names, desc []string) { + return ls.langNames, ls.langDescs +} + +// SetDefaultLang sets default language as a fallback +func (ls *LocaleStore) SetDefaultLang(lang string) { + ls.defaultLang = lang +} + +// Tr translates content to target language. fall back to default language. +func (ls *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string { + l, ok := ls.localeMap[lang] + if !ok { + l, ok = ls.localeMap[ls.defaultLang] + } + if ok { + return l.Tr(trKey, trArgs...) + } + return trKey +} + +// Tr translates content to locale language. fall back to default language. +func (l *locale) Tr(trKey string, trArgs ...interface{}) string { + var section string + + idx := strings.IndexByte(trKey, '.') + if idx > 0 { + section = trKey[:idx] + trKey = trKey[idx+1:] + } + + trMsg := trKey + if trIni, err := l.messages.Section(section).GetKey(trKey); err == nil { + trMsg = trIni.Value() + } else if l.store.defaultLang != "" && l.langName != l.store.defaultLang { + // try to fall back to default + if defaultLocale, ok := l.store.localeMap[l.store.defaultLang]; ok { + if trIni, err = defaultLocale.messages.Section(section).GetKey(trKey); err == nil { + trMsg = trIni.Value() + } + } + } + + if len(trArgs) > 0 { + fmtArgs := make([]interface{}, 0, len(trArgs)) + for _, arg := range trArgs { + val := reflect.ValueOf(arg) + if val.Kind() == reflect.Slice { + // before, it can accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f), it's an unstable behavior + // now, we restrict the strange behavior and only support: + // 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...) + // 2. Tr(lang, key, args...) as Sprintf(msg, args...) + if len(trArgs) == 1 { + for i := 0; i < val.Len(); i++ { + fmtArgs = append(fmtArgs, val.Index(i).Interface()) + } + } else { + log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs) + break + } + } else { + fmtArgs = append(fmtArgs, arg) + } + } + return fmt.Sprintf(trMsg, fmtArgs...) + } + return trMsg +} + +func ResetDefaultLocales() { + DefaultLocales = NewLocaleStore() +} + +// Tr use default locales to translate content to target language. +func Tr(lang, trKey string, trArgs ...interface{}) string { + return DefaultLocales.Tr(lang, trKey, trArgs...) +} diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go new file mode 100644 index 0000000000000..70066016cfe43 --- /dev/null +++ b/modules/translation/i18n/i18n_test.go @@ -0,0 +1,56 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package i18n + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Tr(t *testing.T) { + testData1 := []byte(` +.dot.name = Dot Name +fmt = %[1]s %[2]s + +[section] +sub = Sub String +mixed = test value; more text +`) + + testData2 := []byte(` +fmt = %[2]s %[1]s + +[section] +sub = Changed Sub String +`) + + ls := NewLocaleStore() + assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1)) + assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2)) + ls.SetDefaultLang("lang1") + + result := ls.Tr("lang1", "fmt", "a", "b") + assert.Equal(t, "a b", result) + + result = ls.Tr("lang2", "fmt", "a", "b") + assert.Equal(t, "b a", result) + + result = ls.Tr("lang1", "section.sub") + assert.Equal(t, "Sub String", result) + + result = ls.Tr("lang2", "section.sub") + assert.Equal(t, "Changed Sub String", result) + + result = ls.Tr("", ".dot.name") + assert.Equal(t, "Dot Name", result) + + result = ls.Tr("lang2", "section.mixed") + assert.Equal(t, `test value; more text`, result) + + langs, descs := ls.ListLangNameDesc() + assert.Equal(t, []string{"lang1", "lang2"}, langs) + assert.Equal(t, []string{"Lang1", "Lang2"}, descs) +} diff --git a/modules/translation/translation.go b/modules/translation/translation.go index fd38e4d510081..da9d9b9b6851c 100644 --- a/modules/translation/translation.go +++ b/modules/translation/translation.go @@ -11,8 +11,8 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation/i18n" - "github.com/unknwon/i18n" "golang.org/x/text/language" ) @@ -54,13 +54,13 @@ func TryTr(lang, format string, args ...interface{}) (string, bool) { // InitLocales loads the locales func InitLocales() { - i18n.Reset() + i18n.ResetDefaultLocales() localeNames, err := options.Dir("locale") if err != nil { log.Fatal("Failed to list locale files: %v", err) } - localFiles := make(map[string][]byte) + localFiles := make(map[string][]byte, len(localeNames)) for _, name := range localeNames { localFiles[name], err = options.Locale(name) if err != nil { @@ -76,16 +76,21 @@ func InitLocales() { matcher = language.NewMatcher(supportedTags) for i := range setting.Names { key := "locale_" + setting.Langs[i] + ".ini" - if err = i18n.SetMessageWithDesc(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil { + if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil { log.Error("Failed to set messages to %s: %v", setting.Langs[i], err) } } - i18n.SetDefaultLang("en-US") + if len(setting.Langs) != 0 { + defaultLangName := setting.Langs[0] + if defaultLangName != "en-US" { + log.Info("Use the first locale (%s) in LANGS setting option as default", defaultLangName) + } + i18n.DefaultLocales.SetDefaultLang(defaultLangName) + } - allLangs = make([]*LangType, 0, i18n.Count()) + langs, descs := i18n.DefaultLocales.ListLangNameDesc() + allLangs = make([]*LangType, 0, len(langs)) allLangMap = map[string]*LangType{} - langs := i18n.ListLangs() - descs := i18n.ListLangDescs() for i, v := range langs { l := &LangType{v, descs[i]} allLangs = append(allLangs, l) diff --git a/modules/web/middleware/locale.go b/modules/web/middleware/locale.go index 3daf5f32d45de..de8e49796541b 100644 --- a/modules/web/middleware/locale.go +++ b/modules/web/middleware/locale.go @@ -9,8 +9,8 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/translation/i18n" - "github.com/unknwon/i18n" "golang.org/x/text/language" ) @@ -28,8 +28,8 @@ func Locale(resp http.ResponseWriter, req *http.Request) translation.Locale { } } - // Check again in case someone modify by purpose. - if lang != "" && !i18n.IsExist(lang) { + // Check again in case someone changes the supported language list. + if lang != "" && !i18n.DefaultLocales.HasLang(lang) { lang = "" changeLang = false } diff --git a/routers/web/repo/issue_content_history.go b/routers/web/repo/issue_content_history.go index 21b22aa7109ec..11cc8a2a6f559 100644 --- a/routers/web/repo/issue_content_history.go +++ b/routers/web/repo/issue_content_history.go @@ -18,9 +18,9 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/translation/i18n" "github.com/sergi/go-diff/diffmatchpatch" - "github.com/unknwon/i18n" ) // GetContentHistoryOverview get overview diff --git a/routers/web/user/setting/profile.go b/routers/web/user/setting/profile.go index 7a875e38cb94a..0123b9b523396 100644 --- a/routers/web/user/setting/profile.go +++ b/routers/web/user/setting/profile.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation/i18n" "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" @@ -31,8 +32,6 @@ import ( "code.gitea.io/gitea/services/agit" "code.gitea.io/gitea/services/forms" user_service "code.gitea.io/gitea/services/user" - - "github.com/unknwon/i18n" ) const ( diff --git a/services/cron/setting.go b/services/cron/setting.go index f0683393ff827..9b59a562f709a 100644 --- a/services/cron/setting.go +++ b/services/cron/setting.go @@ -7,7 +7,7 @@ package cron import ( "time" - "github.com/unknwon/i18n" + "code.gitea.io/gitea/modules/translation/i18n" ) // Config represents a basic configuration interface that cron task