Skip to content

Commit

Permalink
feat(cmd/xliff): compose xliff and skeleton into markdown
Browse files Browse the repository at this point in the history 
@diplodoc/markdown-translation integration to compose xliff and
skeleton files into translated markdown
  • Loading branch information
@moki
moki committed May 17, 2023
1 parent 603d990 commit f9a4115
Showing 1 changed file with 143 additions and 76 deletions.
219 changes: 143 additions & 76 deletions src/cmd/xliff/compose.ts
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,22 @@
import {Arguments} from 'yargs';
const {promises: {readFile, writeFile, mkdir}} = require('fs');
import {join, resolve, dirname} from 'path';
import {join, extname, dirname} from 'path';

const yfm2xliff = require('@doc-tools/yfm2xliff/lib/cjs');
import markdownTranslation, {ComposeParameters} from '@diplodoc/markdown-translation';
import {Arguments, Argv} from 'yargs';
import {eachLimit} from 'async';

import {ArgvService} from '../../services';
import {glob, logger} from '../../utils';
import {MD_EXT_NAME, SKL_EXT_NAME, XLF_EXT_NAME} from './constants';

const command = 'compose';

const description = 'compose xliff and skeleton into documentation';

const compose = {command, description, handler};
const compose = {command, description, handler, builder};

const XLFExtPattern = `\\.${XLF_EXT_NAME}$`;
const XLFExtFlags = 'mui';
const XLFExtRegExp = new RegExp(XLFExtPattern, XLFExtFlags);

const SKL_MD_GLOB = `**/*.${SKL_EXT_NAME}.${MD_EXT_NAME}`;
const XLF_GLOB = `**/*.${XLF_EXT_NAME}`;

const composer = async (xliff: string, skeleton: string) => new Promise((res, rej) =>
yfm2xliff.compose(xliff, skeleton, (err: Error, composed: string) => {
if (err) {
rej(err);
}

return res(composed);
}));
const SKL_MD_GLOB = '**/*.skl.md';
const XLF_GLOB = '**/*.xliff';
const MAX_CONCURRENCY = 50;

class ComposeError extends Error {
path: string;
Expand All @@ -40,6 +28,23 @@ class ComposeError extends Error {
}
}

const USAGE = 'yfm xliff compose \
--input <folder-with-xliff-and-skeleton> \
--ouput <folder-to-store-translated-markdown>';

function builder<T>(argv: Argv<T>) {
return argv
.option('input', {
alias: 'i',
describe: 'input folder with xliff and skeleton files',
type: 'string',
}).option('output', {
alias: 'o',
describe: 'output folder where translated markdown will be stored',
type: 'string',
}).demandOption(['input', 'output'], USAGE);
}

/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
async function handler(args: Arguments<any>) {
ArgvService.init({
Expand All @@ -48,105 +53,167 @@ async function handler(args: Arguments<any>) {

const {input, output} = args;

logger.info(input, 'composing skeleton and xliff files');
let cache = {};
let skeletonPaths: string[] = [];
let xliffPaths: string[] = [];

try {
let cache = {};
let found = [];

({state: {found, cache}} = await glob(join(input, SKL_MD_GLOB), {
({state: {found: skeletonPaths, cache}} = await glob(join(input, SKL_MD_GLOB), {
nosort: false,
cache,
}));

const sklPaths = found;

({state: {found, cache}} = await glob(join(input, XLF_GLOB), {
({state: {found: xliffPaths, cache}} = await glob(join(input, XLF_GLOB), {
nosort: false,
cache,
}));

const xlfPaths = found;

if (!xlfPaths?.length || xlfPaths?.length !== sklPaths?.length) {
throw new ComposeError('failed reading skeleton and xliff files', input);
if (xliffPaths.length !== skeletonPaths.length) {
throw new ComposeError('number of xliff and skeleton files does\'not match', input);
}
} catch (err) {
if (err instanceof Error || err instanceof ComposeError) {
const file = err instanceof ComposeError ? err.path : input;

logger.info(input, 'reading skeleton and xliff files');

const [skls, xlfs, paths] = await Promise.all([
Promise.all(sklPaths.map(readFn)),
Promise.all(xlfPaths.map(readFn)),
Promise.all(xlfPaths.map(async (path: string) =>
path.replace(XLFExtRegExp, '').replace(input, output)))]);
logger.error(file, err.message);
}
}

logger.info(input, 'finished reading skeleton and xliff files');
const pipelineParameters = {input, output};
const configuredPipeline = pipeline(pipelineParameters);

const composed = await Promise.all(paths.map(composeFn(xlfs, skls)));
try {
logger.info(input, 'staring translated markdown composition pipeline');

await Promise.all(composed.map(writeFn));
await eachLimit(xliffPaths, MAX_CONCURRENCY, configuredPipeline);

logger.info(output, 'finished composing into documentation');
logger.info(input, 'finished translated markdown composition pipeline');
} catch (err) {
if (err instanceof Error || err instanceof ComposeError) {
const message = err.message;
const file = err instanceof ComposeError ? err.path : input;

const file = err instanceof ComposeError ? err.path : '';

logger.error(file, message);
logger.error(file, err.message);
}
}
}

async function readFn(path: string) {
logger.info(path, 'reading file');
export type PipelineParameters = {
input: string;
output: string;
};

function pipeline(params: PipelineParameters) {
const {input, output} = params;

return async (xliffPath: string) => {
const extension = extname(xliffPath);
const extensionLessPath = xliffPath.replace(extension, '');
const skeletonPath = extensionLessPath + '.skl.md';

const readerParameters = {xliffPath, skeletonPath};
const read = await reader(readerParameters);

let file;
const composerParameters = {
...read,
skeletonPath,
xliffPath,
};
const {markdown} = await composer(composerParameters);

const inputRelativePath = extensionLessPath.slice(input.length);
const markdownPath = join(output, inputRelativePath) + '.md';

const writerParameters = {
markdown,
markdownPath,
};
await writer(writerParameters);
};
}

export type ReaderParameters = {
skeletonPath: string;
xliffPath: string;
};

async function reader(params: ReaderParameters) {
const {skeletonPath, xliffPath} = params;

let skeleton;
let xlf;

try {
file = readFile(resolve(path), {encoding: 'utf-8'});
logger.info(skeletonPath, 'reading skeleton file');

skeleton = await readFile(skeletonPath, {encoding: 'utf-8'});

logger.info(skeletonPath, 'finished reading skeleton file');
} catch (err) {
throw new ComposeError(err.message, path);
if (err instanceof Error) {
throw new ComposeError(err.message, skeletonPath);
}
}

return file;
try {
logger.info(xliffPath, 'reading xliff file');

xlf = await readFile(xliffPath, {encoding: 'utf-8'});

logger.info(xliffPath, 'finished reading xliff file');
} catch (err) {
if (err instanceof Error) {
throw new ComposeError(err.message, xliffPath);
}
}

return {skeleton, xlf};
}

export type ComposeFnOutput = {
path: string;
composed: string;
};
export type ComposerParameters = {
skeletonPath: string;
xliffPath: string;
} & ComposeParameters;

function composeFn(xlfs: string[], skls: string[]) {
return async (path: string, i: number): Promise<ComposeFnOutput> => {
logger.info(path, 'composing skeleton and xliff files');
async function composer(params: ComposerParameters) {
const {skeletonPath, xliffPath} = params;
let markdown;

let composed;
try {
logger.info(skeletonPath, 'composing markdown from xliff and skeleton');
logger.info(xliffPath, 'composing markdown from xliff and skeleton');

markdown = markdownTranslation.compose(params);

try {
composed = await composer(xlfs[i], skls[i]) as string;
} catch (err) {
throw new ComposeError(err.message, path);
logger.info(skeletonPath, 'finished composing markdown from xliff and skeleton');
logger.info(xliffPath, 'finished composing markdown from xliff and skeleton');
} catch (err) {
if (err instanceof Error) {
throw new ComposeError(err.message, `${xliffPath} ${skeletonPath}`);
}
}

return {
composed,
path,
};
};
return {markdown};
}

async function writeFn({composed, path}: ComposeFnOutput) {
const file = `${path}.md`;
export type WriterParameters = {
markdown: string;
markdownPath: string;
};

logger.info(file, 'writing composed file');
async function writer(params: WriterParameters) {
const {markdown, markdownPath} = params;

try {
await mkdir(dirname(path), {recursive: true});
logger.info(markdownPath, 'writing markdown file');

await mkdir(dirname(markdownPath), {recursive: true});
await writeFile(markdownPath, markdown);

return writeFile(file, composed);
logger.info(markdownPath, 'finished writing markdown file');
} catch (err) {
throw new ComposeError(err.message, file);
if (err instanceof Error) {
throw new ComposeError(err.message, markdownPath);
}
}
}

Expand Down

0 comments on commit f9a4115

Please sign in to comment.