diff --git a/src/cmd/xliff/compose.ts b/src/cmd/xliff/compose.ts index b1fe4777..c96cda02 100644 --- a/src/cmd/xliff/compose.ts +++ b/src/cmd/xliff/compose.ts @@ -1,34 +1,22 @@ -import {Arguments} from 'yargs'; const {promises: {readFile, writeFile, mkdir}} = require('fs'); -import {join, resolve, dirname} from 'path'; +import {join, extname, dirname} from 'path'; -const yfm2xliff = require('@doc-tools/yfm2xliff/lib/cjs'); +import markdownTranslation, {ComposeParameters} from '@diplodoc/markdown-translation'; +import {Arguments, Argv} from 'yargs'; +import {eachLimit} from 'async'; import {ArgvService} from '../../services'; import {glob, logger} from '../../utils'; -import {MD_EXT_NAME, SKL_EXT_NAME, XLF_EXT_NAME} from './constants'; const command = 'compose'; const description = 'compose xliff and skeleton into documentation'; -const compose = {command, description, handler}; +const compose = {command, description, handler, builder}; -const XLFExtPattern = `\\.${XLF_EXT_NAME}$`; -const XLFExtFlags = 'mui'; -const XLFExtRegExp = new RegExp(XLFExtPattern, XLFExtFlags); - -const SKL_MD_GLOB = `**/*.${SKL_EXT_NAME}.${MD_EXT_NAME}`; -const XLF_GLOB = `**/*.${XLF_EXT_NAME}`; - -const composer = async (xliff: string, skeleton: string) => new Promise((res, rej) => - yfm2xliff.compose(xliff, skeleton, (err: Error, composed: string) => { - if (err) { - rej(err); - } - - return res(composed); - })); +const SKL_MD_GLOB = '**/*.skl.md'; +const XLF_GLOB = '**/*.xliff'; +const MAX_CONCURRENCY = 50; class ComposeError extends Error { path: string; @@ -40,6 +28,23 @@ class ComposeError extends Error { } } +const USAGE = 'yfm xliff compose \ +--input \ +--ouput '; + +function builder(argv: Argv) { + return argv + .option('input', { + alias: 'i', + describe: 'input folder with xliff and skeleton files', + type: 'string', + }).option('output', { + alias: 'o', + describe: 'output folder where translated markdown will be stored', + type: 'string', + }).demandOption(['input', 'output'], USAGE); +} + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ async function handler(args: Arguments) { ArgvService.init({ @@ -48,105 +53,167 @@ async function handler(args: Arguments) { const {input, output} = args; - logger.info(input, 'composing skeleton and xliff files'); + let cache = {}; + let skeletonPaths: string[] = []; + let xliffPaths: string[] = []; try { - let cache = {}; - let found = []; - - ({state: {found, cache}} = await glob(join(input, SKL_MD_GLOB), { + ({state: {found: skeletonPaths, cache}} = await glob(join(input, SKL_MD_GLOB), { nosort: false, cache, })); - const sklPaths = found; - - ({state: {found, cache}} = await glob(join(input, XLF_GLOB), { + ({state: {found: xliffPaths, cache}} = await glob(join(input, XLF_GLOB), { nosort: false, cache, })); - const xlfPaths = found; - - if (!xlfPaths?.length || xlfPaths?.length !== sklPaths?.length) { - throw new ComposeError('failed reading skeleton and xliff files', input); + if (xliffPaths.length !== skeletonPaths.length) { + throw new ComposeError('number of xliff and skeleton files does\'not match', input); } + } catch (err) { + if (err instanceof Error || err instanceof ComposeError) { + const file = err instanceof ComposeError ? err.path : input; - logger.info(input, 'reading skeleton and xliff files'); - - const [skls, xlfs, paths] = await Promise.all([ - Promise.all(sklPaths.map(readFn)), - Promise.all(xlfPaths.map(readFn)), - Promise.all(xlfPaths.map(async (path: string) => - path.replace(XLFExtRegExp, '').replace(input, output)))]); + logger.error(file, err.message); + } + } - logger.info(input, 'finished reading skeleton and xliff files'); + const pipelineParameters = {input, output}; + const configuredPipeline = pipeline(pipelineParameters); - const composed = await Promise.all(paths.map(composeFn(xlfs, skls))); + try { + logger.info(input, 'staring translated markdown composition pipeline'); - await Promise.all(composed.map(writeFn)); + await eachLimit(xliffPaths, MAX_CONCURRENCY, configuredPipeline); - logger.info(output, 'finished composing into documentation'); + logger.info(input, 'finished translated markdown composition pipeline'); } catch (err) { if (err instanceof Error || err instanceof ComposeError) { - const message = err.message; + const file = err instanceof ComposeError ? err.path : input; - const file = err instanceof ComposeError ? err.path : ''; - - logger.error(file, message); + logger.error(file, err.message); } } } -async function readFn(path: string) { - logger.info(path, 'reading file'); +export type PipelineParameters = { + input: string; + output: string; +}; + +function pipeline(params: PipelineParameters) { + const {input, output} = params; + + return async (xliffPath: string) => { + const extension = extname(xliffPath); + const extensionLessPath = xliffPath.replace(extension, ''); + const skeletonPath = extensionLessPath + '.skl.md'; + + const readerParameters = {xliffPath, skeletonPath}; + const read = await reader(readerParameters); - let file; + const composerParameters = { + ...read, + skeletonPath, + xliffPath, + }; + const {markdown} = await composer(composerParameters); + + const inputRelativePath = extensionLessPath.slice(input.length); + const markdownPath = join(output, inputRelativePath) + '.md'; + + const writerParameters = { + markdown, + markdownPath, + }; + await writer(writerParameters); + }; +} + +export type ReaderParameters = { + skeletonPath: string; + xliffPath: string; +}; + +async function reader(params: ReaderParameters) { + const {skeletonPath, xliffPath} = params; + + let skeleton; + let xlf; try { - file = readFile(resolve(path), {encoding: 'utf-8'}); + logger.info(skeletonPath, 'reading skeleton file'); + + skeleton = await readFile(skeletonPath, {encoding: 'utf-8'}); + + logger.info(skeletonPath, 'finished reading skeleton file'); } catch (err) { - throw new ComposeError(err.message, path); + if (err instanceof Error) { + throw new ComposeError(err.message, skeletonPath); + } } - return file; + try { + logger.info(xliffPath, 'reading xliff file'); + + xlf = await readFile(xliffPath, {encoding: 'utf-8'}); + + logger.info(xliffPath, 'finished reading xliff file'); + } catch (err) { + if (err instanceof Error) { + throw new ComposeError(err.message, xliffPath); + } + } + + return {skeleton, xlf}; } -export type ComposeFnOutput = { - path: string; - composed: string; -}; +export type ComposerParameters = { + skeletonPath: string; + xliffPath: string; +} & ComposeParameters; -function composeFn(xlfs: string[], skls: string[]) { - return async (path: string, i: number): Promise => { - logger.info(path, 'composing skeleton and xliff files'); +async function composer(params: ComposerParameters) { + const {skeletonPath, xliffPath} = params; + let markdown; - let composed; + try { + logger.info(skeletonPath, 'composing markdown from xliff and skeleton'); + logger.info(xliffPath, 'composing markdown from xliff and skeleton'); + + markdown = markdownTranslation.compose(params); - try { - composed = await composer(xlfs[i], skls[i]) as string; - } catch (err) { - throw new ComposeError(err.message, path); + logger.info(skeletonPath, 'finished composing markdown from xliff and skeleton'); + logger.info(xliffPath, 'finished composing markdown from xliff and skeleton'); + } catch (err) { + if (err instanceof Error) { + throw new ComposeError(err.message, `${xliffPath} ${skeletonPath}`); } + } - return { - composed, - path, - }; - }; + return {markdown}; } -async function writeFn({composed, path}: ComposeFnOutput) { - const file = `${path}.md`; +export type WriterParameters = { + markdown: string; + markdownPath: string; +}; - logger.info(file, 'writing composed file'); +async function writer(params: WriterParameters) { + const {markdown, markdownPath} = params; try { - await mkdir(dirname(path), {recursive: true}); + logger.info(markdownPath, 'writing markdown file'); + + await mkdir(dirname(markdownPath), {recursive: true}); + await writeFile(markdownPath, markdown); - return writeFile(file, composed); + logger.info(markdownPath, 'finished writing markdown file'); } catch (err) { - throw new ComposeError(err.message, file); + if (err instanceof Error) { + throw new ComposeError(err.message, markdownPath); + } } }