Skip to content

Commit

Permalink
Fix plurals detection
Browse files Browse the repository at this point in the history
  • Loading branch information
@tassoevan
tassoevan committed Jan 2, 2024
1 parent 15f86d8 commit bdd7ff1
Show file tree
Hide file tree
Showing 28 changed files with 182 additions and 156 deletions.
245 changes: 130 additions & 115 deletions apps/meteor/.scripts/translation-check.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import type { PathLike } from 'node:fs';
import fs from 'node:fs';
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { inspect } from 'node:util';
Expand All @@ -8,36 +7,6 @@ import fg from 'fast-glob';
import i18next from 'i18next';
import supportsColor from 'supports-color';

const validateKeys = (json: Record<string, string>, usedKeys: { key: string; replaces: string[] }[]) =>
usedKeys
.filter(({ key }) => typeof json[key] !== 'undefined')
.reduce((prev, cur) => {
const { key, replaces } = cur;

const miss = replaces.filter((replace) => json[key] && json[key].indexOf(replace) === -1);

if (miss.length > 0) {
prev.push({ key, miss });
}

return prev;
}, [] as { key: string; miss: string[] }[]);

const removeMissingKeys = async (i18nFiles: string[], usedKeys: { key: string; replaces: string[] }[]) => {
for await (const file of i18nFiles) {
const json = await parseFile(file);
if (Object.keys(json).length === 0) {
return;
}

validateKeys(json, usedKeys).forEach(({ key }) => {
delete json[key];
});

await writeFile(file, JSON.stringify(json, null, 2), 'utf8');
}
};

const hasDuplicatedKeys = (content: string, json: Record<string, string>) => {
const matchKeys = content.matchAll(/^\s+"([^"]+)"/gm);

Expand All @@ -46,41 +15,8 @@ const hasDuplicatedKeys = (content: string, json: Record<string, string>) => {
return allKeys.length !== Object.keys(json).length;
};

const checkUniqueKeys = (content: string, json: Record<string, string>, filename: string) => {
if (hasDuplicatedKeys(content, json)) {
throw new Error(`Duplicated keys found on file ${filename}`);
}
};

const validate = (i18nFiles: string[], usedKeys: { key: string; replaces: string[] }[]) => {
const totalErrors = i18nFiles.reduce((errors, file) => {
const content = fs.readFileSync(file, 'utf8');
const json = JSON.parse(content);

checkUniqueKeys(content, json, file);

const result = validateKeys(json, usedKeys);

if (result.length === 0) {
return errors;
}

console.log('\n## File', file, `(${result.length} errors)`);

result.forEach(({ key, miss }) => {
console.log('\n- Key:', key, '\n Missing variables:', miss.join(', '));
});

return errors + result.length;
}, 0);

if (totalErrors > 0) {
throw new Error(`\n${totalErrors} errors found`);
}
};

const parseFile = async (path: PathLike) => {
const content = await readFile(path, 'utf8');
const content = await readFile(path, 'utf-8');
let json: Record<string, string>;
try {
json = JSON.parse(content);
Expand All @@ -106,9 +42,37 @@ const parseFile = async (path: PathLike) => {
return json;
};

const oldPlaceholderFormat = /__[a-zA-Z_]+__/g;
const insertTranslation = (json: Record<string, string>, refKey: string, [key, value]: [key: string, value: string]) => {
const entries = Object.entries(json);

const refIndex = entries.findIndex(([entryKey]) => entryKey === refKey);

if (refIndex === -1) {
throw new Error(`Reference key ${refKey} not found`);
}

const movingEntries = entries.slice(refIndex + 1);

for (const [key] of movingEntries) {
delete json[key];
}

json[key] = value;

for (const [key, value] of movingEntries) {
json[key] = value;
}
};

const persistFile = async (path: PathLike, json: Record<string, string>) => {
const content = JSON.stringify(json, null, 2);

await writeFile(path, content, 'utf-8');
};

const checkPlaceholdersFormat = (json: Record<string, string>, path: PathLike) => {
const oldPlaceholderFormat = /__([a-zA-Z_]+)__/g;

const checkPlaceholdersFormat = async ({ json, path, fix = false }: { json: Record<string, string>; path: PathLike; fix?: boolean }) => {
const outdatedKeys = Object.entries(json)
.map(([key, value]) => ({
key,
Expand All @@ -118,11 +82,23 @@ const checkPlaceholdersFormat = (json: Record<string, string>, path: PathLike) =
.filter((outdatedKey): outdatedKey is { key: string; value: string; placeholders: RegExpMatchArray } => !!outdatedKey.placeholders);

if (outdatedKeys.length > 0) {
throw new Error(
`Outdated placeholder format on file ${path}: ${outdatedKeys
.map((key) => inspect(key, { colors: !!supportsColor.stdout }))
.join(', ')}`,
);
const message = `Outdated placeholder format on file ${path}: ${inspect(outdatedKeys, { colors: !!supportsColor.stdout })}`;

if (fix) {
console.warn(message);

for (const { key, value } of outdatedKeys) {
const newValue = value.replace(oldPlaceholderFormat, (_, name) => `{{${name}}}`);

json[key] = newValue;
}

await persistFile(path, json);

return;
}

throw new Error(message);
}
};

Expand All @@ -136,7 +112,7 @@ export const extractSingularKeys = (json: Record<string, string>, lng: string) =
const singularKeys = new Set(
Object.keys(json).map((key) => {
for (const pluralSuffix of pluralSuffixes) {
if (key.endsWith(`_${pluralSuffix}`)) {
if (key.endsWith(pluralSuffix)) {
return key.slice(0, -pluralSuffix.length);
}
}
Expand All @@ -148,87 +124,126 @@ export const extractSingularKeys = (json: Record<string, string>, lng: string) =
return [singularKeys, pluralSuffixes] as const;
};

const checkMissingPlurals = (json: Record<string, string>, path: PathLike, lng: string) => {
const checkMissingPlurals = async ({
json,
path,
lng,
fix = false,
}: {
json: Record<string, string>;
path: PathLike;
lng: string;
fix?: boolean;
}) => {
const [singularKeys, pluralSuffixes] = extractSingularKeys(json, lng);

const missingPluralKeys: { singularKey: string; pluralKeys: string[] }[] = [];
const missingPluralKeys: { singularKey: string; existing: string[]; missing: string[] }[] = [];

for (const singularKey of singularKeys) {
if (singularKey in json) {
continue;
}

const pluralKeys = pluralSuffixes.map((suffix) => `${singularKey}_${suffix}`);
const pluralKeys = pluralSuffixes.map((suffix) => `${singularKey}${suffix}`);

const missingOnes = pluralKeys.filter((key) => !(key in json));
const existing = pluralKeys.filter((key) => key in json);
const missing = pluralKeys.filter((key) => !(key in json));

if (missingOnes.length > 0) {
missingPluralKeys.push({ singularKey, pluralKeys: missingOnes });
if (missing.length > 0) {
missingPluralKeys.push({ singularKey, existing, missing });
}
}

if (missingPluralKeys.length > 0) {
throw new Error(`Missing plural keys on file ${path}: ${inspect(missingPluralKeys, { colors: !!supportsColor.stdout })}`);
const message = `Missing plural keys on file ${path}: ${inspect(missingPluralKeys, { colors: !!supportsColor.stdout })}`;

if (fix) {
console.warn(message);

for (const { existing, missing } of missingPluralKeys) {
for (const missingKey of missing) {
const refKey = existing.slice(-1)[0];
const value = json[refKey];
insertTranslation(json, refKey, [missingKey, value]);
}
}

await persistFile(path, json);

return;
}

throw new Error(message);
}
};

const checkExceedingKeys = (
json: Record<string, string>,
path: PathLike,
lng: string,
sourceJson: Record<string, string>,
sourceLng: string,
) => {
const checkExceedingKeys = async ({
json,
path,
lng,
sourceJson,
sourceLng,
fix = false,
}: {
json: Record<string, string>;
path: PathLike;
lng: string;
sourceJson: Record<string, string>;
sourceLng: string;
fix?: boolean;
}) => {
const [singularKeys] = extractSingularKeys(json, lng);
const [sourceSingularKeys] = extractSingularKeys(sourceJson, sourceLng);

const exceedingKeys = [...singularKeys].filter((key) => !sourceSingularKeys.has(key));

if (exceedingKeys.length > 0) {
throw new Error(`Exceeding keys on file ${path}: ${inspect(exceedingKeys, { colors: !!supportsColor.stdout })}`);
const message = `Exceeding keys on file ${path}: ${inspect(exceedingKeys, { colors: !!supportsColor.stdout })}`;

if (fix) {
for (const key of exceedingKeys) {
delete json[key];
}

await persistFile(path, json);

return;
}

throw new Error(message);
}
};

const checkFiles = async (sourceDirPath: string, sourceFile: string, fix = false) => {
const sourcePath = join(sourceDirPath, sourceFile);
const checkFiles = async (sourceDirPath: string, sourceLng: string, fix = false) => {
const sourcePath = join(sourceDirPath, `${sourceLng}.i18n.json`);
const sourceJson = await parseFile(sourcePath);

checkPlaceholdersFormat(sourceJson, sourcePath);
checkMissingPlurals(sourceJson, sourcePath, 'en');
await checkPlaceholdersFormat({ json: sourceJson, path: sourcePath, fix });
await checkMissingPlurals({ json: sourceJson, path: sourcePath, lng: sourceLng, fix });

const i18nFiles = await fg([join(sourceDirPath, `**/*.i18n.json`), `!${sourcePath}`]);

const languageFileRegex = /\/([^\/]*?).i18n.json$/;
const translations = await Promise.all(
i18nFiles.map(async (path) => ({ path, json: await parseFile(path), lng: /\/(.*).i18n.json$/.exec(path)?.[1] ?? 'FIXME' })),
i18nFiles.map(async (path) => {
const lng = languageFileRegex.exec(path)?.[1];
if (!lng) {
throw new Error(`Invalid language file path ${path}`);
}

return { path, json: await parseFile(path), lng };
}),
);

for await (const { path, json, lng } of translations) {
checkPlaceholdersFormat(json, path);
checkMissingPlurals(json, path, lng);
checkExceedingKeys(json, path, lng, sourceJson, 'en');
await checkPlaceholdersFormat({ json, path, fix });
await checkMissingPlurals({ json, path, lng, fix });
await checkExceedingKeys({ json, path, lng, sourceJson, sourceLng, fix });
}

const regexVar = /__[a-zA-Z_]+__/g;

const usedKeys = Object.entries(sourceJson)
.map(([key, value]) => {
const replaces = value.match(regexVar);
return {
key,
replaces,
};
})
.filter((usedKey): usedKey is { key: string; replaces: RegExpMatchArray } => !!usedKey.replaces);

if (fix) {
return removeMissingKeys(i18nFiles, usedKeys);
}

validate(i18nFiles, usedKeys);
};

const fix = process.argv[2] === '--fix';
checkFiles('./packages/rocketchat-i18n/i18n', 'en.i18n.json', fix).catch((e) => {
checkFiles('./packages/rocketchat-i18n/i18n', 'en', fix).catch((e) => {
console.error(e);
process.exit(1);
});
1 change: 1 addition & 0 deletions apps/meteor/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@
"@types/strict-uri-encode": "^2.0.1",
"@types/string-strip-html": "^5.0.1",
"@types/supertest": "^2.0.15",
"@types/supports-color": "~7.2.0",
"@types/textarea-caret": "^3.0.2",
"@types/ua-parser-js": "^0.7.38",
"@types/use-subscription": "^1.0.1",
Expand Down
4 changes: 2 additions & 2 deletions apps/meteor/packages/rocketchat-i18n/i18n/ar.i18n.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2575,7 +2575,7 @@
"leave-c_description": "إذن لمغادرة القنوات",
"leave-p": "مغادرة المجموعات الخاصة",
"leave-p_description": "إذن لمغادرة المجموعات الخاصة",
"Lets_get_you_new_one": "دعنا نحضر لك واحدة جديدة!",
"Lets_get_you_new_one_": "دعنا نحضر لك واحدة جديدة!",
"Link_Preview": "رابط المعاينة",
"List_of_Channels": "قائمة Channels",
"List_of_departments_for_forward": "قائمة الأقسام المسموح بإعادة توجيهها (اختياري)",
Expand Down Expand Up @@ -4886,4 +4886,4 @@
"Enterprise": "مؤسسة",
"UpgradeToGetMore_engagement-dashboard_Title": "التحليلات",
"UpgradeToGetMore_auditing_Title": "تدقيق الرسائل"
}
}
4 changes: 2 additions & 2 deletions apps/meteor/packages/rocketchat-i18n/i18n/ca.i18n.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2549,7 +2549,7 @@
"leave-c_description": "Permís per sortir de canals",
"leave-p": "Sortir de grups privats",
"leave-p_description": "Permís per sortir de grups privats",
"Lets_get_you_new_one": "Et portem un de nou!",
"Lets_get_you_new_one_": "Et portem un de nou!",
"List_of_Channels": "Llista de canals",
"List_of_departments_for_forward": "Llista de departaments permesos per reenviament (opcional)",
"List_of_departments_for_forward_description": "Permetre establir una llista restringida de departaments que poden rebre xats d'aquest departament",
Expand Down Expand Up @@ -4690,4 +4690,4 @@
"Enterprise": "Empresa",
"UpgradeToGetMore_engagement-dashboard_Title": "Analítiques",
"UpgradeToGetMore_auditing_Title": "Auditoria de missatges"
}
}
4 changes: 2 additions & 2 deletions apps/meteor/packages/rocketchat-i18n/i18n/cs.i18n.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2169,7 +2169,7 @@
"Leave_the_description_field_blank_if_you_dont_want_to_show_the_role": "Pokud nechcete zobrazovat roli, ponechte pole popisu prázdné",
"leave-c": "Odejít z místností",
"leave-p": "Opustit soukromé skupiny",
"Lets_get_you_new_one": "Pojďme si pořídit nový!",
"Lets_get_you_new_one_": "Pojďme si pořídit nový!",
"List_of_Channels": "Seznam místností",
"List_of_departments_for_forward": "Seznam oddělení povolených pro přesměrování (volitelné)",
"List_of_departments_for_forward_description": "Omezit oddělení do kterých je možné přesměrovat konverzace z aktuálního",
Expand Down Expand Up @@ -3965,4 +3965,4 @@
"Enterprise": "Korporace",
"UpgradeToGetMore_engagement-dashboard_Title": "Analytika",
"UpgradeToGetMore_auditing_Title": "Audit zpráv"
}
}
Loading

0 comments on commit bdd7ff1

Please sign in to comment.