build: added building, packaging, and localization tools (#744)

Main feature here is the `intl` tool that allows people to quickly create or update localization files with simple:

```
tools/intl de
```

This will parse the code for localization strings, and compare it against `de` locale, removing unused, and assigning `null` to new strings.

We can also make it even easier for people by running:

```
tools/intl all
```

after every localization string change in the codebase, which will update all existing locales so people can just browse through their ones and fill in nulls.

.gitignore

@@ -1 +1,2 @@
+release
 *.zip

README.md

@@ -692,6 +692,22 @@ mp.commandv('script-message-to', 'uosc', 'disable-elements', mp.get_script_name(
 
 Using `'user'` as `script_id` will overwrite user's `disable_elements` config. Elements will be enabled only when neither user, nor any script requested them to be disabled.
 
+## Contributing
+
+### Localization
+
+If you want to help localizing uosc by either adding a new locale or fixing one that is not up to date, start by running this while in the repository root:
+
+```
+tools/intl languagecode
+```
+
+`languagecode` can be any existing locale in `dist/scripts/uosc/intl/` directory, or any [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag). If it doesn't exist yet, the `intl` tool will create it.
+
+This will parse the codebase for localization strings and use them to either update existing locale by removing unused and setting untranslated strings to `null`, or create a new one with all `null` strings.
+
+You can then navigate to `dist/scripts/uosc/intl/languagecode.json` and start translating.
+
 ## Why _uosc_?
 
 It stood for micro osc as it used to render just a couple rectangles before it grew to what it is today. And now it means a minimalist UI design direction where everything is out of your way until needed.

go.mod

@@ -0,0 +1,8 @@
+module uosc/bins
+
+go 1.21.3
+
+require (
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	k8s.io/apimachinery v0.28.3
+)

go.sum

@@ -0,0 +1,4 @@
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A=
+k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8=

src/intl.go

@@ -0,0 +1,282 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/exp/maps"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+func main() {
+	cwd, err := os.Getwd()
+	check(err)
+	uoscRootRelative := "dist/scripts/uosc"
+	intlRootRelative := uoscRootRelative + "/intl"
+	uoscRoot := filepath.Join(cwd, uoscRootRelative)
+
+	// Check we're in correct location
+	if stat, err := os.Stat(uoscRoot); os.IsNotExist(err) || !stat.IsDir() {
+		fmt.Printf(`Directory "%s" doesn't exist. Make sure you're running this tool in uosc's project root folder as current working directory.`, uoscRootRelative)
+		os.Exit(1)
+	}
+
+	// Help
+	if len(os.Args) <= 1 || len(os.Args) > 1 && sets.New("--help", "-h").Has(os.Args[1]) {
+		fmt.Printf(`Updates or creates a localization files by parsing the codebase for localization strings, and (re)constructing the locale files with them.
+Strings no longer in use are removed. Strings not yet translated are set to "null".
+
+Usage:
+
+  intl [languages]
+
+Parameters:
+
+  languages    A comma separated list of language codes to update
+               or create. Use 'all' to update all existing locales.
+
+Examples:
+
+  > intl xy
+  Create a new locale xy.
+
+  > intl de,es
+  Update de and es locales.
+
+  > intl all
+  Update everything inside "%s".
+
+`, intlRootRelative)
+		os.Exit(0)
+	}
+
+	var locales []string
+	if os.Args[1] == "all" {
+		intlRoot := filepath.Join(cwd, intlRootRelative)
+		locales = must(listFilenamesOfType(intlRoot, ".json"))
+	} else {
+		locales = strings.Split(os.Args[1], ",")
+	}
+
+	holePunchLocales(locales, uoscRoot)
+
+}
+
+func holePunchLocales(locales []string, rootPath string) {
+	fmt.Println("Creating localization holes for:", strings.Join(locales, ", "))
+
+	fnName := 't'
+	spaces := sets.New(' ', '\t', '\n')
+	enclosers := sets.New('"', '\'')
+	wordBreaks := sets.New('=', '*', '+', '-', '/', '(', ')', '^', '%', '#', '@', '!', '~', '`', '"', '\'', ' ', '\t', '\n')
+	escape := '\\'
+	openParen := '('
+	localizationStrings := sets.New[string]()
+
+	// Contents processor to extract localization strings
+	// Solution doesn't check if function calls are commented out or not.
+	processFile := func(path string) {
+		escapesNum := 0
+		f := must(os.Open(path))
+		currentStr := ""
+		currentEncloser := '"'
+		prevRune := ' '
+
+		type lexFn func(r rune)
+		var currentLexer lexFn
+		var accumulateString lexFn
+		var findOpenEncloser lexFn
+		var findOpenParen lexFn
+		var findFn lexFn
+
+		commitStr := func() {
+			localizationStrings.Insert(currentStr)
+			currentStr = ""
+			currentLexer = findFn
+		}
+
+		accumulateString = func(r rune) {
+			if r == currentEncloser && escapesNum%2 == 0 {
+				commitStr()
+			} else {
+				if r == escape {
+					escapesNum++
+				} else {
+					escapesNum = 0
+				}
+				currentStr += string(r)
+			}
+		}
+
+		findOpenEncloser = func(r rune) {
+			if !spaces.Has(r) {
+				if enclosers.Has(r) {
+					currentEncloser = r
+					currentLexer = accumulateString
+				} else {
+					currentLexer = findFn
+				}
+			}
+		}
+
+		findOpenParen = func(r rune) {
+			if !spaces.Has(r) {
+				if r == openParen {
+					currentLexer = findOpenEncloser
+				} else {
+					currentLexer = findFn
+				}
+			}
+		}
+
+		findFn = func(b rune) {
+			if b == fnName && wordBreaks.Has(prevRune) {
+				currentLexer = findOpenParen
+			}
+		}
+
+		currentLexer = findFn
+		br := bufio.NewReader(f)
+
+		for {
+			r, _, err := br.ReadRune()
+
+			if err != nil && !errors.Is(err, io.EOF) {
+				panic(err)
+			}
+
+			// end of file
+			if err != nil {
+				break
+			}
+
+			currentLexer(r)
+
+			prevRune = r
+			escapesNum = 0
+		}
+	}
+
+	// Find localization strings in lua files
+	check(filepath.WalkDir(rootPath, func(fp string, fi os.DirEntry, err error) error {
+		check(err)
+
+		if ext := filepath.Ext(fp); ext == ".lua" {
+			processFile(fp)
+		}
+
+		return nil
+	}))
+
+	fmt.Println("Found localization strings:", localizationStrings.Len())
+
+	// Create new or punch holes and filter unused strings from existing locales
+	for _, locale := range locales {
+		localePath := filepath.Join(rootPath, "intl", locale+".json")
+		isNew := true
+
+		// Parse old json
+		oldLocaleData := make(map[string]interface{})
+		localeContents, err := os.ReadFile(localePath)
+		if err == nil {
+			isNew = false
+			check(json.Unmarshal(localeContents, &oldLocaleData))
+		} else if !errors.Is(err, os.ErrNotExist) {
+			check(err)
+		}
+
+		// Merge into new locale for current codebase
+		var localeData = make(map[string]interface{})
+		removed := sets.List(sets.New[string](maps.Keys(oldLocaleData)...).Difference(localizationStrings))
+		untranslated := []string{}
+
+		for _, str := range sets.List(localizationStrings) {
+			if old, ok := oldLocaleData[str]; ok {
+				localeData[str] = old
+			} else {
+				localeData[str] = nil
+			}
+
+			if localeData[str] == nil {
+				untranslated = append(untranslated, str)
+			}
+		}
+
+		// Output
+		resultJson := must(JSONMarshalIndent(localeData, "", "\t"))
+		check(os.WriteFile(localePath, resultJson, 0644))
+		fmt.Println()
+
+		// Stats
+		newOrUpdatingMsg := "Updating existing locale"
+		if len(removed) == 0 && len(untranslated) == 0 {
+			newOrUpdatingMsg = "Locale is up to date"
+		} else if isNew {
+			newOrUpdatingMsg = "Creating new locale"
+		}
+		fmt.Println("[[", locale, "]]>", newOrUpdatingMsg)
+		if len(removed) > 0 {
+			fmt.Println("• Removed:")
+			for _, str := range removed {
+				fmt.Printf("  '%s'\n", str)
+			}
+		}
+		if len(untranslated) > 0 {
+			fmt.Println("• Untranslated:")
+			for _, str := range untranslated {
+				fmt.Printf("  '%s'\n", str)
+			}
+		}
+	}
+}
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func must[T any](t T, err error) T {
+	check(err)
+	return t
+}
+
+func listFilenamesOfType(directoryPath string, extension string) ([]string, error) {
+	files := []string{}
+	extension = strings.ToLower(extension)
+
+	dirEntries, err := os.ReadDir(directoryPath)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range dirEntries {
+		if entry.IsDir() {
+			continue
+		}
+		filename := entry.Name()
+		ext := filepath.Ext(filename)
+		if strings.ToLower(ext) == extension {
+			files = append(files, filename[:len(filename)-len(ext)])
+		}
+	}
+
+	return files, nil
+}
+
+// Because the default `json.Marshal` HTML escapes `&,<,>` characters and it can't be turned off...
+func JSONMarshalIndent(t interface{}, prefix string, indent string) ([]byte, error) {
+	buffer := &bytes.Buffer{}
+	encoder := json.NewEncoder(buffer)
+	encoder.SetEscapeHTML(false)
+	encoder.SetIndent(prefix, indent)
+	err := encoder.Encode(t)
+	return buffer.Bytes(), err
+}

tools/build.ps1

@@ -0,0 +1,37 @@
+# Build macros to build and compress binaries for all platforms.
+# Requirements: go and upx
+
+Function Abort($Message) {
+	Write-Output "Error: $Message"
+	Write-Output "Aborting!"
+	Exit 1
+}
+
+if (!(Test-Path -Path "$PWD/src" -PathType Container)) {
+	Abort("'src' directory not found. Make sure this script is run in uosc's repository root as current working directory.")
+}
+
+if ($args[0] -eq "intl") {
+	$env:GOARCH = "amd64"
+
+	Write-Output "Building for Windows..."
+	$env:GOOS = "windows"
+	go build -ldflags "-s -w" -o ./tools/intl.exe src/intl.go
+	upx --brute ./tools/intl.exe
+
+	Write-Output "Building for Linux..."
+	$env:GOOS = "linux"
+	go build -ldflags "-s -w" -o ./tools/intl-linux src/intl.go
+	upx --brute ./tools/intl-linux
+
+	Write-Output "Building for MacOS..."
+	$env:GOOS = "darwin"
+	go build -ldflags "-s -w" -o ./tools/intl-darwin src/intl.go
+	upx --brute ./tools/intl-darwin
+
+	Remove-Item Env:\GOOS
+	Remove-Item Env:\GOARCH
+}
+else {
+	Write-Output "Pass what to build. Available: intl"
+}

tools/intl

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+if [ "$(uname)" == "Darwin" ]; then
+	"$SCRIPT_DIR/intl-darwin" $*
+else
+	"$SCRIPT_DIR/intl-linux" $*
+fi