From 5f9dad50db22532d0999984f8853ac2dd345d38e Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 22 Aug 2023 11:20:35 +0100 Subject: [PATCH 1/8] Also fill "one" pluralisation --- scripts/gen-i18n.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gen-i18n.ts b/scripts/gen-i18n.ts index 6671992..ef2227b 100644 --- a/scripts/gen-i18n.ts +++ b/scripts/gen-i18n.ts @@ -311,6 +311,7 @@ for (const tr of translatables) { } else { _.set(trObj, path, { "other": tr, + "one": tr, }) } } From 1e216e04a45778aaeaf33760521cabeda9b8df20 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 22 Aug 2023 12:19:21 +0100 Subject: [PATCH 2/8] Add utility to re-key the translation files Remove prune utility as it isn't necessary anymore in either Weblate or Localazy --- package.json | 6 ++-- scripts/common.ts | 55 +++++++++++++++++++++++++++++++ scripts/gen-i18n.ts | 31 +++++------------- scripts/prune-i18n.ts | 76 ------------------------------------------- scripts/rekey.ts | 59 +++++++++++++++++++++++++++++++++ yarn.lock | 10 ++++++ 6 files changed, 137 insertions(+), 100 deletions(-) create mode 100644 scripts/common.ts delete mode 100644 scripts/prune-i18n.ts create mode 100644 scripts/rekey.ts diff --git a/package.json b/package.json index 4bf6c23..4411592 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,8 @@ ], "bin": { "matrix-gen-i18n": "scripts/gen-i18n.js", - "matrix-prune-i18n": "scripts/prune-i18n.js", - "matrix-compare-i18n-files": "scripts/compare-file.js" + "matrix-compare-i18n-files": "scripts/compare-file.js", + "matrix-i18n-rekey": "scripts/rekey.js" }, "scripts": { "build:ts": "tsc", @@ -37,11 +37,13 @@ "@babel/parser": "^7.18.5", "@babel/traverse": "^7.18.5", "lodash": "^4.17.21", + "minimist": "^1.2.8", "walk": "^2.3.15" }, "devDependencies": { "@types/babel__traverse": "^7.17.1", "@types/lodash": "^4.14.197", + "@types/minimist": "^1.2.2", "@types/node": "^18.0.0", "@types/walk": "^2.3.1", "typescript": "^4.7.4" diff --git a/scripts/common.ts b/scripts/common.ts new file mode 100644 index 0000000..a8677ff --- /dev/null +++ b/scripts/common.ts @@ -0,0 +1,55 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import fs from "fs"; + +export const NESTING_KEY = process.env["NESTING_KEY"] || "|"; +export const INPUT_FILE = process.env["INPUT_FILE"] || 'src/i18n/strings/en_EN.json'; +export const OUTPUT_FILE = process.env["OUTPUT_FILE"] || 'src/i18n/strings/en_EN.json'; + +export type Translation = string | { + one?: string; + other: string; +}; + +export interface Translations { + [key: string]: Translation | Translations; +} + +export function getPath(key: string): string[] { + return key.split(NESTING_KEY); +} + +export function getKeys(translations: Translations | Translation, path = ""): string[] { + // base case + if (typeof translations === "string" || "other" in translations) { + return [path]; + } + + if (path) path += NESTING_KEY; + return Object.keys(translations).flatMap(key => getKeys(translations[key], path + key)); +} + +export function getTranslations(file = INPUT_FILE): Readonly { + return JSON.parse(fs.readFileSync(file, { encoding: 'utf8' })); +} + +export function putTranslations(translations: Translations, file = OUTPUT_FILE): void { + fs.writeFileSync( + file, + JSON.stringify(translations, null, 4) + "\n" + ); +} diff --git a/scripts/gen-i18n.ts b/scripts/gen-i18n.ts index ef2227b..87a62f3 100644 --- a/scripts/gen-i18n.ts +++ b/scripts/gen-i18n.ts @@ -42,6 +42,13 @@ import { } from "@babel/types"; import { ParserPlugin } from "@babel/parser"; import _ from "lodash"; +import { + getPath, + getTranslations, + OUTPUT_FILE, + putTranslations, + Translations +} from "./common"; // Find the package.json for the project we're running gen-18n against const projectPackageJsonPath = path.join(process.cwd(), 'package.json'); @@ -53,10 +60,6 @@ const TRANSLATIONS_FUNCS = ['_t', '_td', '_tDom'] // "matrix_i18n_extra_translation_funcs" key .concat(projectPackageJson.matrix_i18n_extra_translation_funcs || []); -const NESTING_KEY = process.env["NESTING_KEY"] || "|"; -const INPUT_TRANSLATIONS_FILE = process.env["INPUT_FILE"] || 'src/i18n/strings/en_EN.json'; -const OUTPUT_FILE = process.env["OUTPUT_FILE"] || 'src/i18n/strings/en_EN.json'; - // NB. The sync version of walk is broken for single files, // so we walk all of res rather than just res/home.html. // https://git.daplie.com/Daplie/node-walk/merge_requests/1 fixes it, @@ -240,16 +243,7 @@ function getTranslationsOther(file: string): Set { return trs; } -type Translation = string | { - one?: string; - other: string; -}; - -interface Translations { - [key: string]: Translation | Translations; -} - -const inputTranslationsRaw: Readonly = JSON.parse(fs.readFileSync(INPUT_TRANSLATIONS_FILE, { encoding: 'utf8' })); +const inputTranslationsRaw = getTranslations(); const translatables = new Set(); const plurals = new Set(); @@ -297,10 +291,6 @@ for (const path of SEARCH_PATHS) { } } -function getPath(key: string): string[] { - return key.split(NESTING_KEY); -} - const trObj: Translations = {}; for (const tr of translatables) { const path = getPath(tr); @@ -316,10 +306,7 @@ for (const tr of translatables) { } } -fs.writeFileSync( - OUTPUT_FILE, - JSON.stringify(trObj, null, 4) + "\n" -); +putTranslations(trObj); console.log(); console.log(`Wrote ${translatables.size} strings to ${OUTPUT_FILE}`); diff --git a/scripts/prune-i18n.ts b/scripts/prune-i18n.ts deleted file mode 100644 index d6861d1..0000000 --- a/scripts/prune-i18n.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2017 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* - * Looks through all the translation files and removes any strings - * which don't appear in en_EN.json. - * Use this if you remove a translation, but merge any outstanding changes - * from Weblate first, or you'll need to resolve the conflict in Weblate. - */ - -import * as path from "path"; -import * as fs from "fs"; - -const I18NDIR = 'src/i18n/strings'; - -type TranslationFile = { - [key: string]: string; -} - -const enStringsRaw = JSON.parse(fs.readFileSync(path.join(I18NDIR, 'en_EN.json')).toString()) as TranslationFile; - -const enStrings = new Set(); -for (const str of Object.keys(enStringsRaw)) { - const parts = str.split('|'); - if (parts.length > 1) { - enStrings.add(parts[0]); - } else { - enStrings.add(str); - } -} - -for (const filename of fs.readdirSync(I18NDIR)) { - if (filename === 'en_EN.json') continue; - if (filename === 'basefile.json') continue; - if (!filename.endsWith('.json')) continue; - - const trs = JSON.parse(fs.readFileSync(path.join(I18NDIR, filename)).toString()) as TranslationFile; - const oldLen = Object.keys(trs).length; - for (const tr of Object.keys(trs)) { - const parts = tr.split('|'); - const trKey = parts.length > 1 ? parts[0] : tr; - if (!enStrings.has(trKey)) { - delete trs[tr]; - } - - // Clean up for when a string gets pluralised, - // to not leave behind the un-pluralised variant which causes warnings - if (parts.length > 1 && trKey in trs) { - delete trs[trKey]; - } - } - - const removed = oldLen - Object.keys(trs).length; - if (removed > 0) { - console.log(`${filename}: removed ${removed} translations`); - // XXX: This is totally relying on the impl serialising the JSON object in the - // same order as they were parsed from the file. JSON.stringify() has a specific argument - // that can be used to control the order, but JSON.parse() lacks any kind of equivalent. - // Empirically this does maintain the order on my system, so I'm going to leave it like - // this for now. - fs.writeFileSync(path.join(I18NDIR, filename), JSON.stringify(trs, undefined, 4) + "\n"); - } -} diff --git a/scripts/rekey.ts b/scripts/rekey.ts new file mode 100644 index 0000000..34bee4e --- /dev/null +++ b/scripts/rekey.ts @@ -0,0 +1,59 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import parseArgs from "minimist"; +import _ from "lodash"; +import { getPath, getTranslations, putTranslations, Translations } from "./common"; +import fs from "fs"; +import path from "path"; + +const I18NDIR = "src/i18n/strings"; + +const argv = parseArgs<{ + copy: boolean; +}>(process.argv.slice(2), { + boolean: ["copy", "move", "case-insensitive", "find-and-replace"], +}); + +const [oldPath, newPath] = argv._.map(getPath); +const sourceTranslations = getTranslations(); + +const translation = _.get(sourceTranslations, oldPath); +if (!translation) { + throw new Error("Old key not present in source translations"); +} + +function updateTranslations(translations: Translations): void { + const value = _.get(translations, oldPath); + if (!value) return; + + _.set(translations, newPath, _.get(translations, oldPath)); + + if (!argv.copy) { + _.unset(translations, oldPath); + } +} + +for (const filename of fs.readdirSync(I18NDIR)) { + if (!filename.endsWith(".json")) continue; + const file = path.join(I18NDIR, filename); + const translations = getTranslations(file); + updateTranslations(translations); + putTranslations(translations, file); +} + + + diff --git a/yarn.lock b/yarn.lock index 68f2d04..405b341 100644 --- a/yarn.lock +++ b/yarn.lock @@ -141,6 +141,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.197.tgz#e95c5ddcc814ec3e84c891910a01e0c8a378c54b" integrity sha512-BMVOiWs0uNxHVlHBgzTIqJYmj+PgCo4euloGF+5m4okL3rEYzM2EEv78mw8zWSMM57dM7kVIgJ2QDvwHSoCI5g== +"@types/minimist@^1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" + integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== + "@types/node@*", "@types/node@^18.0.0": version "18.0.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.0.0.tgz#67c7b724e1bcdd7a8821ce0d5ee184d3b4dd525a" @@ -223,6 +228,11 @@ lodash@^4.17.21: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +minimist@^1.2.8: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" From 2f405fefbf8ee842f4972a2a4aa787bb6b73fea1 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 22 Aug 2023 17:56:33 +0100 Subject: [PATCH 3/8] Fix tag replacement validation for literal keys --- scripts/gen-i18n.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/gen-i18n.ts b/scripts/gen-i18n.ts index 87a62f3..97bb511 100644 --- a/scripts/gen-i18n.ts +++ b/scripts/gen-i18n.ts @@ -188,8 +188,9 @@ function getTranslationsJs(file: string): [keys: Set, plurals: Set 2 && isObjectExpression(p.node.arguments[2])) { const tagMap = p.node.arguments[2]; for (const prop of tagMap.properties || []) { - if (isObjectProperty(prop) && isStringLiteral(prop.key)) { - const tag = prop.key.value; + if (isObjectProperty(prop) && (isStringLiteral(prop.key) || isIdentifier(prop.key))) { + const tag = isIdentifier(prop.key) ? prop.key.name : prop.key.value; + // RegExp same as in src/languageHandler.js const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`); if (!tKey.match(regexp)) { From 191ecb68294ce461e5f63fba7a6751c5b1b64746 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 22 Aug 2023 17:56:51 +0100 Subject: [PATCH 4/8] Fix tag & placeholder validation for non-english keys --- scripts/gen-i18n.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/gen-i18n.ts b/scripts/gen-i18n.ts index 97bb511..eeb7db3 100644 --- a/scripts/gen-i18n.ts +++ b/scripts/gen-i18n.ts @@ -47,6 +47,7 @@ import { getTranslations, OUTPUT_FILE, putTranslations, + Translation, Translations } from "./common"; @@ -122,7 +123,7 @@ function getFormatStrings(str: string): Set { return formatStrings; } -function getTranslationsJs(file: string): [keys: Set, plurals: Set] { +function getTranslationsJs(file: string, translations: Readonly): [keys: Set, plurals: Set] { const contents = fs.readFileSync(file, { encoding: 'utf8' }); const keys = new Set(); @@ -169,11 +170,14 @@ function getTranslationsJs(file: string): [keys: Set, plurals: Set, plurals: Set(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`); - if (!tKey.match(regexp)) { - throw new Error(`No match for ${regexp} in ${tKey}`); + if (!englishValue.match(regexp)) { + throw new Error(`No match for ${regexp} in ${englishValue}`); } } } @@ -267,7 +271,7 @@ const walkOpts: WalkOptions = { let keys: Set; let pluralKeys = new Set(); if (fileStats.name.endsWith('.js') || fileStats.name.endsWith('.ts') || fileStats.name.endsWith('.tsx')) { - [keys, pluralKeys] = getTranslationsJs(fullPath); + [keys, pluralKeys] = getTranslationsJs(fullPath, inputTranslationsRaw); } else if (fileStats.name.endsWith('.html')) { keys = getTranslationsOther(fullPath); } else { From 05e39068fcf0f1c28a94c3756bfc7e63f3a72544 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 22 Aug 2023 18:30:38 +0100 Subject: [PATCH 5/8] null-guard for new translations to now explode --- scripts/gen-i18n.ts | 48 +++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/scripts/gen-i18n.ts b/scripts/gen-i18n.ts index eeb7db3..8bfd4e7 100644 --- a/scripts/gen-i18n.ts +++ b/scripts/gen-i18n.ts @@ -170,35 +170,37 @@ function getTranslationsJs(file: string, translations: Readonly): // had to use a _td to compensate) so is expected. if (tKey === null) return; - const rawValue: Translation = _.get(translations, getPath(tKey)); - const englishValue = typeof rawValue === "string" ? rawValue : rawValue.other; - // check the format string against the args // We only check _t: _td has no args if (isIdentifier(p.node.callee) && p.node.callee.name === '_t') { try { - const placeholders = getFormatStrings(englishValue); - for (const placeholder of placeholders) { - if (p.node.arguments.length < 2 || !isObjectExpression(p.node.arguments[1])) { - throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`); - } - const value = getObjectValue(p.node.arguments[1], placeholder); - if (value === null) { - throw new Error(`No value found for placeholder '${placeholder}'`); + const rawValue: Translation | undefined = _.get(translations, getPath(tKey)); + const englishValue = typeof rawValue === "string" ? rawValue : rawValue?.other; + + if (englishValue) { + const placeholders = getFormatStrings(englishValue); + for (const placeholder of placeholders) { + if (p.node.arguments.length < 2 || !isObjectExpression(p.node.arguments[1])) { + throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`); + } + const value = getObjectValue(p.node.arguments[1], placeholder); + if (value === null) { + throw new Error(`No value found for placeholder '${placeholder}'`); + } } - } - // Validate tag replacements - if (p.node.arguments.length > 2 && isObjectExpression(p.node.arguments[2])) { - const tagMap = p.node.arguments[2]; - for (const prop of tagMap.properties || []) { - if (isObjectProperty(prop) && (isStringLiteral(prop.key) || isIdentifier(prop.key))) { - const tag = isIdentifier(prop.key) ? prop.key.name : prop.key.value; - - // RegExp same as in src/languageHandler.js - const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`); - if (!englishValue.match(regexp)) { - throw new Error(`No match for ${regexp} in ${englishValue}`); + // Validate tag replacements + if (p.node.arguments.length > 2 && isObjectExpression(p.node.arguments[2])) { + const tagMap = p.node.arguments[2]; + for (const prop of tagMap.properties || []) { + if (isObjectProperty(prop) && (isStringLiteral(prop.key) || isIdentifier(prop.key))) { + const tag = isIdentifier(prop.key) ? prop.key.name : prop.key.value; + + // RegExp same as in src/languageHandler.js + const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`); + if (!englishValue.match(regexp)) { + throw new Error(`No match for ${regexp} in ${englishValue}`); + } } } } From c9dd70bc61a6259845049df1c5a9ea28b35c049d Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 23 Aug 2023 10:25:02 +0100 Subject: [PATCH 6/8] Add utility to find usages of keys --- scripts/common.ts | 12 +++ scripts/find-usage.ts | 183 ++++++++++++++++++++++++++++++++++++++++++ scripts/gen-i18n.ts | 16 +--- 3 files changed, 196 insertions(+), 15 deletions(-) create mode 100644 scripts/find-usage.ts diff --git a/scripts/common.ts b/scripts/common.ts index a8677ff..9cec1e4 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -15,6 +15,7 @@ limitations under the License. */ import fs from "fs"; +import { isBinaryExpression, isStringLiteral, isTemplateLiteral, Node } from "@babel/types"; export const NESTING_KEY = process.env["NESTING_KEY"] || "|"; export const INPUT_FILE = process.env["INPUT_FILE"] || 'src/i18n/strings/en_EN.json'; @@ -53,3 +54,14 @@ export function putTranslations(translations: Translations, file = OUTPUT_FILE): JSON.stringify(translations, null, 4) + "\n" ); } + +export function getTKey(arg: Node): string | null { + if (isStringLiteral(arg)) { + return arg.value; + } else if (isBinaryExpression(arg) && arg.operator === '+') { + return getTKey(arg.left)! + getTKey(arg.right)!; + } else if (isTemplateLiteral(arg)) { + return arg.quasis.map(q => q.value.raw).join(''); + } + return null; +} diff --git a/scripts/find-usage.ts b/scripts/find-usage.ts new file mode 100644 index 0000000..87f09e1 --- /dev/null +++ b/scripts/find-usage.ts @@ -0,0 +1,183 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * Finds code usages of a specific i18n key or count of usages per key if no key specified + */ + +import * as path from "path"; +import * as fs from "fs"; +import { WalkOptions, walkSync } from "walk"; +import * as parser from "@babel/parser"; +import traverse from "@babel/traverse"; +import { + isIdentifier, + isCallExpression, + isNewExpression, +} from "@babel/types"; +import { ParserPlugin } from "@babel/parser"; +import _ from "lodash"; +import { getTKey } from "./common"; + +// Find the package.json for the project we're running gen-18n against +const projectPackageJsonPath = path.join(process.cwd(), 'package.json'); +const projectPackageJson = require(projectPackageJsonPath); + +const TRANSLATIONS_FUNCS = ['_t', '_td', '_tDom'] + // Add some addition translation functions to look out that are specified + // per project in package.json under the + // "matrix_i18n_extra_translation_funcs" key + .concat(projectPackageJson.matrix_i18n_extra_translation_funcs || []); + +// NB. The sync version of walk is broken for single files, +// so we walk all of res rather than just res/home.html. +// https://git.daplie.com/Daplie/node-walk/merge_requests/1 fixes it, +// or if we get bored waiting for it to be merged, we could switch +// to a project that's actively maintained. +const SEARCH_PATHS = ['src', 'res']; + +function getTranslationsJs(file: string): Map { + const contents = fs.readFileSync(file, { encoding: 'utf8' }); + + const trs = new Map(); + + try { + const plugins: ParserPlugin[] = [ + // https://babeljs.io/docs/en/babel-parser#plugins + "classProperties", + "objectRestSpread", + "throwExpressions", + "exportDefaultFrom", + "decorators-legacy", + ]; + + if (file.endsWith(".js") || file.endsWith(".jsx")) { + // All JS is assumed to be React + plugins.push("jsx"); + } else if (file.endsWith(".ts")) { + // TS can't use JSX unless it's a TSX file (otherwise angle casts fail) + plugins.push("typescript"); + } else if (file.endsWith(".tsx")) { + // When the file is a TSX file though, enable JSX parsing + plugins.push("typescript", "jsx"); + } + + const babelParsed = parser.parse(contents, { + allowImportExportEverywhere: true, + errorRecovery: true, + sourceFilename: file, + tokens: true, + plugins, + }); + traverse(babelParsed, { + enter: (p) => { + if ( + (isNewExpression(p.node) || isCallExpression(p.node)) && + isIdentifier(p.node.callee) && + TRANSLATIONS_FUNCS.includes(p.node.callee.name) + ) { + const tKey = getTKey(p.node.arguments[0]); + + // This happens whenever we call _t with non-literals (ie. whenever we've + // had to use a _td to compensate) so is expected. + if (tKey === null) return; + + if (trs.has(tKey)) { + trs.get(tKey)!.push(file); + } else { + trs.set(tKey, [file]); + } + } + }, + }); + } catch (e) { + console.error(e); + process.exit(1); + } + + return trs; +} + +function getTranslationsOther(file: string): Map { + const contents = fs.readFileSync(file, { encoding: 'utf8' }); + + const trs = new Map(); + + // Taken from element-web src/components/structures/HomePage.js + const translationsRegex = /_t\(['"]([\s\S]*?)['"]\)/mg; + let matches: RegExpExecArray | null; + while (matches = translationsRegex.exec(contents)) { + if (trs.has(matches[1])) { + trs.get(matches[1])!.push(file); + } else { + trs.set(matches[1], [file]); + } + } + return trs; +} + +const keyUsages = new Map(); + +const walkOpts: WalkOptions = { + listeners: { + names: function(root, nodeNamesArray) { + // Sort the names case insensitively and alphabetically to + // maintain some sense of order between the different strings. + nodeNamesArray.sort((a, b) => { + a = a.toLowerCase(); + b = b.toLowerCase(); + if (a > b) return 1; + if (a < b) return -1; + return 0; + }); + }, + file: function(root, fileStats, next) { + const fullPath = path.join(root, fileStats.name); + + let trs: Map; + if (fileStats.name.endsWith('.js') || fileStats.name.endsWith('.ts') || fileStats.name.endsWith('.tsx')) { + trs = getTranslationsJs(fullPath); + } else if (fileStats.name.endsWith('.html')) { + trs = getTranslationsOther(fullPath); + } else { + return; + } + + for (const key of trs.keys()) { + if (keyUsages.has(key)) { + keyUsages.get(key)!.push(...trs.get(key)!); + } else { + keyUsages.set(key, trs.get(key)!); + } + } + }, + } +}; + +for (const path of SEARCH_PATHS) { + if (fs.existsSync(path)) { + walkSync(path, walkOpts); + } +} + +const key = process.argv[2]; + +if (key) { + console.log(`Consumers of "${key}":`, keyUsages.get(key)); +} else { + const sorted = _.sortBy([...keyUsages.keys()], k => -keyUsages.get(k)!.length); + console.table(Object.fromEntries(sorted.map(key => [key.substring(0, 120), keyUsages.get(key)!.length]))); +} diff --git a/scripts/gen-i18n.ts b/scripts/gen-i18n.ts index 8bfd4e7..8b87f89 100644 --- a/scripts/gen-i18n.ts +++ b/scripts/gen-i18n.ts @@ -30,20 +30,17 @@ import * as parser from "@babel/parser"; import traverse from "@babel/traverse"; import { isStringLiteral, - isBinaryExpression, - isTemplateLiteral, isIdentifier, isCallExpression, isNewExpression, isObjectProperty, isObjectExpression, ObjectExpression, - Node, } from "@babel/types"; import { ParserPlugin } from "@babel/parser"; import _ from "lodash"; import { - getPath, + getPath, getTKey, getTranslations, OUTPUT_FILE, putTranslations, @@ -77,17 +74,6 @@ function getObjectValue(obj: ObjectExpression, key: string): any { return null; } -function getTKey(arg: Node): string | null { - if (isStringLiteral(arg)) { - return arg.value; - } else if (isBinaryExpression(arg) && arg.operator === '+') { - return getTKey(arg.left)! + getTKey(arg.right)!; - } else if (isTemplateLiteral(arg)) { - return arg.quasis.map(q => q.value.raw).join(''); - } - return null; -} - function getFormatStrings(str: string): Set { // Match anything that starts with % // We could make a regex that matched the full placeholder, but this From 81cfe89f40b07e02f022a1edacec3d2510a781e9 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 23 Aug 2023 10:38:23 +0100 Subject: [PATCH 7/8] Update --- scripts/rekey.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rekey.ts b/scripts/rekey.ts index 34bee4e..88eb7a1 100644 --- a/scripts/rekey.ts +++ b/scripts/rekey.ts @@ -33,7 +33,7 @@ const sourceTranslations = getTranslations(); const translation = _.get(sourceTranslations, oldPath); if (!translation) { - throw new Error("Old key not present in source translations"); + throw new Error(`"${argv._[0]}" key not present in source translations`); } function updateTranslations(translations: Translations): void { From 216c13d9aaba0ab88fdbff9e09cb5327f55f7469 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 24 Aug 2023 10:25:57 +0100 Subject: [PATCH 8/8] Update find-usage.ts --- scripts/find-usage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/find-usage.ts b/scripts/find-usage.ts index 87f09e1..7c35399 100644 --- a/scripts/find-usage.ts +++ b/scripts/find-usage.ts @@ -32,7 +32,7 @@ import { ParserPlugin } from "@babel/parser"; import _ from "lodash"; import { getTKey } from "./common"; -// Find the package.json for the project we're running gen-18n against +// Find the package.json for the project we're running against const projectPackageJsonPath = path.join(process.cwd(), 'package.json'); const projectPackageJson = require(projectPackageJsonPath);