Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor #4

Merged
merged 1 commit into from
Jun 25, 2022
Merged

refactor #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 44 additions & 127 deletions modules/translation/i18n/i18n.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@ import (
"errors"
"fmt"
"reflect"
"strings"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"

"gopkg.in/ini.v1"
)
Expand All @@ -25,112 +23,73 @@ var (
type locale struct {
store *LocaleStore
langName string
messages *ini.File
textMap map[int]string // the map key (idx) is generated by store's textIdxMap
}

type LocaleStore struct {
// After initializing has finished, these fields are read-only.
langNames []string
langDescs []string

langOffsets []int
translationKeys []string
keyToOffset map[string]int
translationValues []string
localeMap map[string]*locale
textIdxMap map[string]int

localeMap map[string]*locale

defaultLang string
defaultLangKeysLen int
defaultLang string
}

func NewLocaleStore() *LocaleStore {
return &LocaleStore{localeMap: make(map[string]*locale), keyToOffset: make(map[string]int)}
return &LocaleStore{localeMap: make(map[string]*locale), textIdxMap: make(map[string]int)}
}

// AddLocaleByIni adds locale by ini into the store
func (ls *LocaleStore) AddLocaleByIni(langName, langDesc string, localeFile interface{}, otherLocaleFiles ...interface{}) error {
// if source is a string, then the file is loaded
// if source is a []byte, then the content is used
func (ls *LocaleStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
if _, ok := ls.localeMap[langName]; ok {
return ErrLocaleAlreadyExist
}
iniFile, err := ini.LoadSources(ini.LoadOptions{
IgnoreInlineComment: true,
UnescapeValueCommentSymbols: true,
}, localeFile, otherLocaleFiles...)
if err == nil {
// Common code between production and development.
ls.langNames = append(ls.langNames, langName)
ls.langDescs = append(ls.langDescs, langDesc)

// Specify the offset for translationValues.
ls.langOffsets = append(ls.langOffsets, len(ls.langOffsets))

// Distinguish between production and development
// For development, live-reload of the translation files is important.
// For production, we can do some expensive work and then make the querying fast.
if setting.IsProd {
// use en-US as the hardcoded default/fallback language.
if langName == "en-US" {
idx := 0
// Store all key, value into two slices.
for _, section := range iniFile.Sections() {
for _, key := range section.Keys() {
ls.translationKeys = append(ls.translationKeys, section.Name()+"#"+key.Name())
ls.translationValues = append(ls.translationValues, key.Value())

ls.keyToOffset[strings.TrimPrefix(section.Name()+"."+key.Name(), "DEFAULT.")] = idx
idx++
}
}

ls.defaultLangKeysLen = len(ls.translationKeys)
}, source)
if err != nil {
return fmt.Errorf("unable to load ini: %w", err)
}
iniFile.BlockMode = false

// Common code between production and development.
ls.langNames = append(ls.langNames, langName)
ls.langDescs = append(ls.langDescs, langDesc)

lc := &locale{store: ls, langName: langName, textMap: make(map[int]string)}
ls.localeMap[lc.langName] = lc
for _, section := range iniFile.Sections() {
for _, key := range section.Keys() {
var trKey string
if section.Name() == "" || section.Name() == "DEFAULT" {
trKey = key.Name()
} else {
// Go trough all the keys that the defaultLang has and append it to translationValues.
// If the lang doesn't have a value for the translation, use the defaultLang's one.
for i := 0; i < ls.defaultLangKeysLen; i++ {
splitted := strings.SplitN(ls.translationKeys[i], "#", 2)
// TODO: optimize for repeated sequential access of section.
section, err := iniFile.GetSection(splitted[0])
if err != nil {
// Section not found? Use the defaultLang's value for this translation key.
ls.translationValues = append(ls.translationValues, ls.translationValues[i])
continue
}
key, err := section.GetKey(splitted[1])
if err != nil {
// Key not found? Use the defaultLang's value for this translation key.
ls.translationValues = append(ls.translationValues, ls.translationValues[i])
continue
}
ls.translationValues = append(ls.translationValues, key.Value())
}
trKey = section.Name() + "." + key.Name()
}

// Help Go's GC.
iniFile = nil

// Specify the offset for translationValues.
ls.langOffsets = append(ls.langOffsets, len(ls.langOffsets))

// Stub info for `HasLang`
ls.localeMap[langName] = nil
} else {
// Add the language to the localeMap.
iniFile.BlockMode = false
lc := &locale{store: ls, langName: langName, messages: iniFile}
ls.localeMap[lc.langName] = lc
textIdx, ok := ls.textIdxMap[trKey]
if !ok {
textIdx = len(ls.textIdxMap)
ls.textIdxMap[trKey] = textIdx
}
lc.textMap[textIdx] = key.Value()
}
}
return err
iniFile = nil
return nil
}

func (ls *LocaleStore) HasLang(langName string) bool {
_, ok := ls.localeMap[langName]
return ok
}

func (ls *LocaleStore) ListLangNameDescOffsets() (names, desc []string, offsets []int) {
return ls.langNames, ls.langDescs, ls.langOffsets
func (ls *LocaleStore) ListLangNameDesc() (names, desc []string) {
return ls.langNames, ls.langDescs
}

// SetDefaultLang sets default language as a fallback
Expand All @@ -152,22 +111,15 @@ func (ls *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string {

// Tr translates content to locale language. fall back to default language.
func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
var section string

idx := strings.IndexByte(trKey, '.')
if idx > 0 {
section = trKey[:idx]
trKey = trKey[idx+1:]
}

trMsg := trKey
if trIni, err := l.messages.Section(section).GetKey(trKey); err == nil {
trMsg = trIni.Value()
} else if l.store.defaultLang != "" && l.langName != l.store.defaultLang {
// try to fall back to default
if defaultLocale, ok := l.store.localeMap[l.store.defaultLang]; ok {
if trIni, err = defaultLocale.messages.Section(section).GetKey(trKey); err == nil {
trMsg = trIni.Value()
textIdx, ok := l.store.textIdxMap[trKey]
if ok {
if msg, ok := l.textMap[textIdx]; ok {
trMsg = msg // use current translation
} else if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
// try to use default locale's translation
if msg, ok := def.textMap[textIdx]; ok {
trMsg = msg
}
}
}
Expand Down Expand Up @@ -206,38 +158,3 @@ func ResetDefaultLocales() {
func Tr(lang, trKey string, trArgs ...interface{}) string {
return DefaultLocales.Tr(lang, trKey, trArgs...)
}

// TrOffset uses the default-locales to translate content to target language.
// It uses the pre-computed translation keys->values.
func TrOffset(offset int, trKey string, trArgs ...interface{}) string {
// Get the offset of the translation key.
keyOffset := DefaultLocales.keyToOffset[trKey]
// Now adjust to use the language's translation of the key.
keyOffset += offset * DefaultLocales.defaultLangKeysLen

trMsg := DefaultLocales.translationValues[keyOffset]
if len(trArgs) > 0 {
fmtArgs := make([]interface{}, 0, len(trArgs))
for _, arg := range trArgs {
val := reflect.ValueOf(arg)
if val.Kind() == reflect.Slice {
// before, it can accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f), it's an unstable behavior
// now, we restrict the strange behavior and only support:
// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
if len(trArgs) == 1 {
for i := 0; i < val.Len(); i++ {
fmtArgs = append(fmtArgs, val.Index(i).Interface())
}
} else {
log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs)
break
}
} else {
fmtArgs = append(fmtArgs, arg)
}
}
return fmt.Sprintf(trMsg, fmtArgs...)
}
return trMsg
}
3 changes: 1 addition & 2 deletions modules/translation/i18n/i18n_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ sub = Changed Sub String
result = ls.Tr("lang2", "section.mixed")
assert.Equal(t, `test value; <span style="color: red; background: none;">more text</span>`, result)

langs, descs, offsets := ls.ListLangNameDescOffsets()
langs, descs := ls.ListLangNameDesc()
assert.Equal(t, []string{"lang1", "lang2"}, langs)
assert.Equal(t, []string{"Lang1", "Lang2"}, descs)
assert.Equal(t, []int{0, 1}, offsets)
}
43 changes: 4 additions & 39 deletions modules/translation/translation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ type Locale interface {
// LangType represents a lang type
type LangType struct {
Lang, Name string // these fields are used directly in templates: {{range .AllLangs}}{{.Lang}}{{.Name}}{{end}}
Offset int
}

var (
Expand All @@ -43,7 +42,7 @@ func AllLangs() []*LangType {

// TryTr tries to do the translation, if no translation, it returns (format, false)
func TryTr(lang, format string, args ...interface{}) (string, bool) {
s := i18n.DefaultLocales.Tr(lang, format, args...)
s := i18n.Tr(lang, format, args...)
// now the i18n library is not good enough and we can only use this hacky method to detect whether the transaction exists
idx := strings.IndexByte(format, '.')
defaultText := format
Expand All @@ -53,29 +52,6 @@ func TryTr(lang, format string, args ...interface{}) (string, bool) {
return s, s != defaultText
}

// moveToFront moves needle to the front of haystack, in place if possible.
// Ref: https://github.com/golang/go/wiki/SliceTricks#move-to-front-or-prepend-if-not-present-in-place-if-possible
func moveToFront(needle string, haystack []string) []string {
if len(haystack) != 0 && haystack[0] == needle {
return haystack
}
prev := needle
for i, elem := range haystack {
switch {
case i == 0:
haystack[0] = needle
prev = elem
case elem == needle:
haystack[i] = prev
return haystack
default:
haystack[i] = prev
prev = elem
}
}
return append(haystack, prev)
}

// InitLocales loads the locales
func InitLocales() {
i18n.ResetDefaultLocales()
Expand All @@ -98,17 +74,12 @@ func InitLocales() {
}

matcher = language.NewMatcher(supportedTags)

// Make sure en-US is always the first in the slice.
setting.Names = moveToFront("English", setting.Names)
setting.Langs = moveToFront("en-US", setting.Langs)
for i := range setting.Names {
key := "locale_" + setting.Langs[i] + ".ini"
if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil {
log.Error("Failed to set messages to %s: %v", setting.Langs[i], err)
}
}

if len(setting.Langs) != 0 {
defaultLangName := setting.Langs[0]
if defaultLangName != "en-US" {
Expand All @@ -117,11 +88,11 @@ func InitLocales() {
i18n.DefaultLocales.SetDefaultLang(defaultLangName)
}

langs, descs, offsets := i18n.DefaultLocales.ListLangNameDescOffsets()
langs, descs := i18n.DefaultLocales.ListLangNameDesc()
allLangs = make([]*LangType, 0, len(langs))
allLangMap = map[string]*LangType{}
for i, v := range langs {
l := &LangType{v, descs[i], offsets[i]}
l := &LangType{v, descs[i]}
allLangs = append(allLangs, l)
allLangMap[v] = l
}
Expand All @@ -141,23 +112,17 @@ func Match(tags ...language.Tag) language.Tag {
// locale represents the information of localization.
type locale struct {
Lang, LangName string // these fields are used directly in templates: .i18n.Lang
// Stores the offset for the locale. The value is utilized by the 'TrOffset' function
// to change the translation key's found index (for the default language) to the locale's index.
Offset int
}

// NewLocale return a locale
func NewLocale(lang string) Locale {
langName := "unknown"
offset := 0
if l, ok := allLangMap[lang]; ok {
langName = l.Name
offset = l.Offset
}
return &locale{
Lang: lang,
LangName: langName,
Offset: offset,
}
}

Expand All @@ -168,7 +133,7 @@ func (l *locale) Language() string {
// Tr translates content to target language.
func (l *locale) Tr(format string, args ...interface{}) string {
if setting.IsProd {
return i18n.TrOffset(l.Offset, format, args...)
return i18n.Tr(l.Lang, format, args...)
}

// in development, we should show an error if a translation key is missing
Expand Down