Skip to content

Commit

Permalink
fix(h5p-server): variant equivalents for Chinese (#2057)
Browse files Browse the repository at this point in the history
* fix: variant equivalents for editor language

* fix: language variants equivalents in H5P libraries

* fix: fallback mechanism

* refactor: corrected jsdocs
  • Loading branch information
sr258 authored Feb 18, 2022
1 parent 940672f commit 0fd9cc6
Show file tree
Hide file tree
Showing 5 changed files with 174 additions and 38 deletions.
9 changes: 7 additions & 2 deletions packages/h5p-server/assets/editorLanguages.json
Original file line number Diff line number Diff line change
@@ -1,23 +1,28 @@
[
"ar",
"bs",
"cs",
"de",
"el",
"en",
"es-cr",
"es-mx",
"es",
"es-mx",
"et",
"eu",
"fi",
"fr",
"it",
"ko",
"nb",
"nl",
"nn",
"pl",
"pt-br",
"pt",
"ru",
"tr"
"sl",
"sv",
"tr",
"zh-cn"
]
4 changes: 4 additions & 0 deletions packages/h5p-server/assets/variantEquivalents.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[
["zh-cn", "zh-hans", "zh"],
["zh-hant", "zh-tw", "zh-hant-tw", "zh-tw"]
]
33 changes: 28 additions & 5 deletions packages/h5p-server/src/H5PEditor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import defaultMetadataSemanticsLanguageFile from '../assets/translations/metadat
import editorAssetList from './editorAssetList.json';
import defaultRenderer from './renderers/default';
import supportedLanguageList from '../assets/editorLanguages.json';
import variantEquivalents from '../assets/variantEquivalents.json';

import ContentManager from './ContentManager';
import { ContentMetadata } from './ContentMetadata';
Expand Down Expand Up @@ -1199,16 +1200,38 @@ export default class H5PEditor {
* @param language
*/
private getLanguageReplacer(language: string): (script: string) => string {
if (supportedLanguageList.includes(language)) {
const cleanLanguage = language.toLocaleLowerCase();

// obvious case: the language file exists
if (supportedLanguageList.includes(cleanLanguage)) {
return (f) =>
f.replace('language/en.js', `language/${language}.js`);
f.replace('language/en.js', `language/${cleanLanguage}.js`);
}

// check if equivalent variants exist (e.g. zh-hans, zh-cn, zh)
const variantList = variantEquivalents.find((l) =>
l.includes(cleanLanguage)
);
if (variantList) {
const alternativeVariant = variantList.find((v) =>
supportedLanguageList.includes(v)
);
if (alternativeVariant) {
return (f) =>
f.replace(
'language/en.js',
`language/${alternativeVariant}.js`
);
}
}
const languageWithRegion = language.replace(/-.+$/, '');
if (supportedLanguageList.includes(languageWithRegion)) {

// fallback to language without variant code
const languageWithoutVariant = cleanLanguage.replace(/-.+$/, '');
if (supportedLanguageList.includes(languageWithoutVariant)) {
return (f) =>
f.replace(
'language/en.js',
`language/${languageWithRegion}.js`
`language/${languageWithoutVariant}.js`
);
}
return (f) => f;
Expand Down
126 changes: 95 additions & 31 deletions packages/h5p-server/src/LibraryManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
} from './types';
import TranslatorWithFallback from './helpers/TranslatorWithFallback';
import SimpleLockProvider from './implementation/SimpleLockProvider';
import variantEquivalents from '../assets/variantEquivalents.json';

const log = new Logger('LibraryManager');

Expand Down Expand Up @@ -150,51 +151,85 @@ export default class LibraryManager {
library: ILibraryName,
language: string
): Promise<string> {
const cleanLanguage = language.toLocaleLowerCase();
try {
log.debug(
`loading language ${language} for library ${LibraryName.toUberName(
library
)}`
// Try full language code first
return await this.getLanguageWithoutFallback(
library,
cleanLanguage
);
const languageFileAsString =
await this.libraryStorage.getFileAsString(
library,
`language/${language}.json`
);
// If the implementation has specified one, we use a hook to alter
// the language files to match the structure of the altered
// semantics.
if (this.alterLibraryLanguageFile) {
log.debug('Calling hook to alter language file of library.');
return JSON.stringify({
semantics: this.alterLibraryLanguageFile(
library,
JSON.parse(languageFileAsString).semantics,
language
)
});
}
return languageFileAsString;
} catch (ignored) {
// The language file didn't exist
log.debug(
`language '${language}' not found for ${LibraryName.toUberName(
`language '${cleanLanguage}' not found for ${LibraryName.toUberName(
library
)}`
);

// Try known variant equivalents (e.g. zh-hans, zh-cn, zh)
const variantList = variantEquivalents
.find((l) => l.includes(cleanLanguage))
?.filter((v) => v !== cleanLanguage);
if (variantList) {
for (const variant of variantList) {
try {
log.debug(`Trying variant equivalent ${variant}`);
// eslint-disable-next-line no-await-in-loop
return await this.getLanguageWithoutFallback(
library,
variant
);
} catch {
log.debug(
`Variant equivalent ${variant} not found. Continuing...`
);
}
}
}

// Try fallback to language without variant
const languageCodeMatch = /^([a-zA-Z]+)-([a-zA-Z]+)$/.exec(
language
cleanLanguage
);
if (languageCodeMatch && languageCodeMatch.length === 3) {
log.debug(
`Language code ${language} seems to contain country code. Trying without it.`
`Language code ${cleanLanguage} seems to contain country code. Trying without it.`
);
return this.getLanguage(library, languageCodeMatch[1]);
try {
return await this.getLanguageWithoutFallback(
library,
languageCodeMatch[1]
);
} catch {
log.debug(`Language ${languageCodeMatch[1]} not found`);
}
}
if (language !== 'en' && language !== '.en') {
return this.getLanguage(library, 'en');

// Fallback to English
if (cleanLanguage !== 'en' && cleanLanguage !== '.en') {
try {
return await this.getLanguageWithoutFallback(library, 'en');
} catch {
log.debug('Language en not found');
}
try {
return await this.getLanguageWithoutFallback(
library,
'.en'
);
} catch {
log.debug('Language .en not found');
}
}
if (language === 'en') {
return this.getLanguage(library, '.en');
if (cleanLanguage === 'en') {
try {
return await this.getLanguageWithoutFallback(
library,
'.en'
);
} catch {
log.debug('Language .en not found');
}
}
return null;
}
Expand Down Expand Up @@ -870,4 +905,33 @@ export default class LibraryManager {
throw error;
}
}

private async getLanguageWithoutFallback(
library: ILibraryName,
language: string
): Promise<string> {
log.debug(
`loading language ${language} for library ${LibraryName.toUberName(
library
)}`
);
const languageFileAsString = await this.libraryStorage.getFileAsString(
library,
`language/${language}.json`
);
// If the implementation has specified one, we use a hook to alter
// the language files to match the structure of the altered
// semantics.
if (this.alterLibraryLanguageFile) {
log.debug('Calling hook to alter language file of library.');
return JSON.stringify({
semantics: this.alterLibraryLanguageFile(
library,
JSON.parse(languageFileAsString).semantics,
language
)
});
}
return languageFileAsString;
}
}
40 changes: 40 additions & 0 deletions packages/h5p-server/test/LibraryManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,46 @@ describe('basic file library manager functionality', () => {
});
});

it('returns the zh language file for language zh-CN', async () => {
const storage = new FileLibraryStorage(
`${__dirname}/../../../test/data/libraries`
);
storage.getFileAsString = async (library, file): Promise<string> => {
if (file === 'language/zh.json') {
return '{}';
} else {
throw new Error('language file not there');
}
};
const libManager = new LibraryManager(storage);

const language = await libManager.getLanguage(
{ machineName: 'H5P.Example1', majorVersion: 1, minorVersion: 1 },
'zh-CN'
);
expect(JSON.parse(language)).toMatchObject({});
});

it('returns the zh-cn language file for language zh', async () => {
const storage = new FileLibraryStorage(
`${__dirname}/../../../test/data/libraries`
);
storage.getFileAsString = async (library, file): Promise<string> => {
if (file === 'language/zh-cn.json') {
return '{}';
} else {
throw new Error('language file not there');
}
};
const libManager = new LibraryManager(storage);

const language = await libManager.getLanguage(
{ machineName: 'H5P.Example1', majorVersion: 1, minorVersion: 1 },
'zh-CN'
);
expect(JSON.parse(language)).toMatchObject({});
});

it('returns the de language file for de-DE', async () => {
const libManager = new LibraryManager(
new FileLibraryStorage(`${__dirname}/../../../test/data/libraries`)
Expand Down

0 comments on commit 0fd9cc6

Please sign in to comment.