From ad1b8798f67c803d67545152ba06f1981f1a5215 Mon Sep 17 00:00:00 2001 From: Daniel Kuznetsov Date: Sat, 6 Jul 2024 01:27:04 +0300 Subject: [PATCH] feat: implement an option to always provide `vcsPath` in metadata for md->md transformations (sans tests) --- src/models.ts | 38 ++++-- src/resolvers/md2md.ts | 13 ++- src/services/contributors.ts | 35 +++--- src/services/metadata.ts | 216 ++++++++++++++++++++--------------- src/steps/processPages.ts | 22 +--- src/utils/markup.ts | 4 +- 6 files changed, 185 insertions(+), 143 deletions(-) diff --git a/src/models.ts b/src/models.ts index efee4e82..761f2c08 100644 --- a/src/models.ts +++ b/src/models.ts @@ -27,6 +27,31 @@ export type UserByLoginFunction = (login: string) => Promise export type CollectionOfPluginsFunction = (output: string, options: PluginOptions) => string; export type GetModifiedTimeByPathFunction = (filepath: string) => number | undefined; +/** + * VCS integration configuration object. + * Future VCS futures should be configured with this one, not with + * `VCSConnectorConfig`. + */ +interface VCSConfiguration { + /** + * Externally accessible base URI for a resource where a particular documentation + * source is hosted. + * + * This configuration parameter is used to directly control the Edit button behaviour + * in the Diplodoc documentation viewer(s). + * + * For example, if the following applies: + * - Repo with doc source is hosted on GitHub (say, https://github.com/foo-org/bar), + * - Within that particular repo, the directory that is being passed as an `--input` + * parameter to the CLI is located at `docs/`, + * - Whenever the Edit button is pressed, you wish to direct your readers to the + * respective document's source on `main` branch + * + * you should pass `https://github.com/foo-org/bar/tree/main/docs` as a value for this parameter. + */ + remoteBase: string; +} + interface YfmConfig { varsPreset: VarsPreset; ignore: string[]; @@ -41,6 +66,7 @@ interface YfmConfig { ignoreStage: string; singlePage: boolean; removeHiddenTocItems: boolean; + vcs?: VCSConfiguration; connector?: VCSConnectorConfig; lang?: Lang; langs?: Lang[]; @@ -199,20 +225,14 @@ export interface Contributors { [email: string]: Contributor; } -export interface FileData { - tmpInputFilePath: string; - inputFolderPathLength: number; - fileContent: string; - sourcePath?: string; -} - export interface MetaDataOptions { - fileData: FileData; + pathData: PathData; isContributorsEnabled?: boolean; vcsConnector?: VCSConnector; addSystemMeta?: boolean; addSourcePath?: boolean; resources?: Resources; + shouldAlwaysAddVCSPath?: boolean; } export interface PluginOptions { @@ -236,7 +256,7 @@ export interface Plugin { export interface ResolveMd2MdOptions { inputPath: string; outputPath: string; - metadata?: MetaDataOptions; + metadata: MetaDataOptions; } export interface ResolverOptions { diff --git a/src/resolvers/md2md.ts b/src/resolvers/md2md.ts index 368c12f5..ae63ddf6 100644 --- a/src/resolvers/md2md.ts +++ b/src/resolvers/md2md.ts @@ -12,16 +12,17 @@ import {getContentWithUpdatedMetadata} from '../services/metadata'; import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types'; export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise { - const {inputPath, outputPath, metadata} = options; + const {inputPath, outputPath, metadata: metadataOptions} = options; const {input, output, changelogs: changelogsSetting} = ArgvService.getConfig(); const resolvedInputPath = resolve(input, inputPath); - const vars = getVarsPerFile(inputPath); + + const varsPreset = getVarsPerFile(inputPath); const content = await getContentWithUpdatedMetadata( readFileSync(resolvedInputPath, 'utf8'), - metadata, - vars.__system, - vars.__metadata, + metadataOptions, + varsPreset.__system as unknown, + varsPreset.__metadata, ); const {result, changelogs} = transformMd2Md(content, { @@ -30,7 +31,7 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise root: resolve(input), destRoot: resolve(output), collectOfPlugins: PluginService.getCollectOfPlugins(), - vars, + vars: varsPreset, log, copyFile, }); diff --git a/src/services/contributors.ts b/src/services/contributors.ts index 6d4c2303..ab6a0fb1 100644 --- a/src/services/contributors.ts +++ b/src/services/contributors.ts @@ -3,11 +3,17 @@ import {dirname, join} from 'path'; import {replaceDoubleToSingleQuotes} from '../utils'; import {REGEXP_INCLUDE_CONTENTS, REGEXP_INCLUDE_FILE_PATH} from '../constants'; -import {Contributor, Contributors, FileData} from '../models'; +import {Contributor, Contributors} from '../models'; import {FileContributors, VCSConnector} from '../vcs-connector/connector-models'; +export interface ContributorsServiceFileData { + resolvedFilePath: string; + inputFolderPathLength: number; + fileContent: string; +} + async function getFileContributorsMetadata( - fileData: FileData, + fileData: ContributorsServiceFileData, vcsConnector: VCSConnector, ): Promise { const contributors = await getFileContributorsString(fileData, vcsConnector); @@ -16,12 +22,12 @@ async function getFileContributorsMetadata( } async function getFileContributorsString( - fileData: FileData, + fileData: ContributorsServiceFileData, vcsConnector: VCSConnector, ): Promise { - const {tmpInputFilePath, inputFolderPathLength} = fileData; + const {resolvedFilePath, inputFolderPathLength} = fileData; - const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength); + const relativeFilePath = resolvedFilePath.substring(inputFolderPathLength); const fileContributors: FileContributors = await vcsConnector.getContributorsByPath(relativeFilePath); let nestedContributors: Contributors = {}; @@ -44,7 +50,7 @@ async function getFileContributorsString( } async function getContributorsForNestedFiles( - fileData: FileData, + fileData: ContributorsServiceFileData, vcsConnector: VCSConnector, ): Promise { const {fileContent, inputFolderPathLength} = fileData; @@ -77,10 +83,10 @@ async function getContributorsForNestedFiles( throw err; } - const newFileData: FileData = { + const newFileData: ContributorsServiceFileData = { ...fileData, fileContent: contentIncludeFile, - tmpInputFilePath: relativeIncludeFilePath, + resolvedFilePath: relativeIncludeFilePath, }; nestedContributors = await getContributorsForNestedFiles(newFileData, vcsConnector); @@ -95,10 +101,9 @@ async function getContributorsForNestedFiles( } function getRelativeIncludeFilePaths( - fileData: Pick, + {resolvedFilePath: tmpInputFilePath}: ContributorsServiceFileData, includeContents: string[], ): Set { - const {tmpInputFilePath} = fileData; const relativeIncludeFilePaths: Set = new Set(); includeContents.forEach((includeContent: string) => { @@ -118,10 +123,8 @@ function getRelativeIncludeFilePaths( return relativeIncludeFilePaths; } -async function getFileIncludes( - fileData: Pick, -) { - const {fileContent, tmpInputFilePath, inputFolderPathLength} = fileData; +async function getFileIncludes(fileData: ContributorsServiceFileData) { + const {fileContent, inputFolderPathLength} = fileData; const results = new Set(); @@ -130,7 +133,7 @@ async function getFileIncludes( return []; } const relativeIncludeFilePaths: Set = getRelativeIncludeFilePaths( - {tmpInputFilePath}, + fileData, includeContents, ); for (const relativeIncludeFilePath of relativeIncludeFilePaths.values()) { @@ -151,7 +154,7 @@ async function getFileIncludes( const includedPaths = await getFileIncludes({ inputFolderPathLength, fileContent: contentIncludeFile, - tmpInputFilePath: relativeIncludeFilePath, + resolvedFilePath: relativeIncludeFilePath, }); includedPaths.forEach((path) => results.add(path)); } diff --git a/src/services/metadata.ts b/src/services/metadata.ts index 24bcb424..4c883259 100644 --- a/src/services/metadata.ts +++ b/src/services/metadata.ts @@ -1,43 +1,92 @@ import {dump, load} from 'js-yaml'; - import {VCSConnector} from '../vcs-connector/connector-models'; -import {MetaDataOptions, Metadata, Resources, VarsMetadata, YfmToc} from '../models'; +import {MetaDataOptions, Metadata, PathData, Resources, VarsMetadata, YfmToc} from '../models'; import { getAuthorDetails, updateAuthorMetadataStringByAuthorLogin, updateAuthorMetadataStringByFilePath, } from './authors'; import { + ContributorsServiceFileData, getFileContributorsMetadata, getFileContributorsString, getFileIncludes, } from './contributors'; import {isObject} from './utils'; -import {сarriage} from '../utils'; +import {carriageReturn} from '../utils'; import {REGEXP_AUTHOR, metadataBorder} from '../constants'; import {TocService} from './index'; +function resolveAdditionalMetadata({ + pathData, + shouldAlwaysAddVCSPath, +}: MetaDataOptions): Record { + return { + vcsPath: shouldAlwaysAddVCSPath ? pathData.pathToFile : undefined, + }; +} + async function getContentWithUpdatedMetadata( fileContent: string, - options?: MetaDataOptions, + options: MetaDataOptions, systemVars?: unknown, metadataVars?: VarsMetadata, ): Promise { - let result; + const {pathData, addSystemMeta, addSourcePath, resources} = options ?? {}; - result = getContentWithUpdatedStaticMetadata({ + const withUpdatedStaticMeta = getContentWithUpdatedStaticMetadata({ fileContent, - sourcePath: options?.fileData?.sourcePath, - addSystemMeta: options?.addSystemMeta, - addSourcePath: options?.addSourcePath, - resources: options?.resources, + sourcePath: pathData?.pathToFile, + addSystemMeta, + addSourcePath, + resources, systemVars, metadataVars, + additionalMetadata: resolveAdditionalMetadata(options), }); - result = await getContentWithUpdatedDynamicMetadata(result, options); + return await getContentWithUpdatedDynamicMetadata(withUpdatedStaticMeta, options); +} - return result; +type FileMetadata = { + [key: string]: unknown; + metadata?: Record[]; +}; + +type ParseExistingMetadataReturn = { + metadata: FileMetadata; + metadataStrippedContent: string; +}; + +function parseExistingMetadata(fileContent: string): ParseExistingMetadataReturn { + const matches = matchMetadata(fileContent); + + if (matches && matches.length > 0) { + const [, metadata, , metadataStrippedContent] = matches; + + return { + metadata: load(metadata) as FileMetadata, + metadataStrippedContent, + }; + } + + return { + metadata: {}, + metadataStrippedContent: fileContent, + }; +} + +function serializeMetadata(objectMetadata: FileMetadata) { + const dumped = dump(objectMetadata).trimEnd(); + + // This empty object check is a bit naive + // The other option would be to check if all own fields are `undefined`, + // since we exploit passing in `undefined` to remove a field quite a bit + if (dumped === '{}') { + return ''; + } + + return `${metadataBorder}${carriageReturn}${dumped}${carriageReturn}${metadataBorder}${carriageReturn}`; } function getContentWithUpdatedStaticMetadata({ @@ -47,6 +96,7 @@ function getContentWithUpdatedStaticMetadata({ addSourcePath, resources, systemVars, + additionalMetadata, metadataVars = [], }: { fileContent: string; @@ -55,59 +105,36 @@ function getContentWithUpdatedStaticMetadata({ addSourcePath?: boolean; resources?: Resources; systemVars?: unknown; + additionalMetadata?: Record; metadataVars?: VarsMetadata; }): string { - const newMetadatas: string[] = []; - - if ( - (!addSystemMeta || !systemVars) && - !addSourcePath && - !resources && - metadataVars.length === 0 - ) { - return fileContent; - } - - const matches = matchMetadata(fileContent); - - if (addSystemMeta && systemVars && isObject(systemVars)) { - newMetadatas.push(getSystemVarsMetadataString(systemVars)); - } - - if (resources) { - newMetadatas.push(dump(resources)); - } - - if (addSourcePath && sourcePath) { - const sourcePathMetadataString = `sourcePath: ${sourcePath}`; - newMetadatas.push(sourcePathMetadataString); - } - - if (matches && matches.length > 0) { - const [, fileMetadata, , fileMainContent] = matches; - - if (!metadataVars.length) { - return `${getUpdatedMetadataString(newMetadatas, fileMetadata)}${fileMainContent}`; - } - - const parsed = load(fileMetadata) as Record; - - if (!Array.isArray(parsed.metadata)) { - parsed.metadata = [parsed.metadata]; - } - - parsed.metadata = parsed.metadata.concat(metadataVars).filter(Boolean); - - const patchedMetada = dump(parsed); - - return `${getUpdatedMetadataString(newMetadatas, patchedMetada)}${fileMainContent}`; - } - - if (metadataVars.length) { - newMetadatas.push(dump({metadata: metadataVars})); - } + const {metadata, metadataStrippedContent} = parseExistingMetadata(fileContent); + + const mergedInnerMetadata: FileMetadata['metadata'] = [ + ...(metadata.metadata ?? []), + ...metadataVars, + ]; + + // Technically, we could use the trick of creating a property, but setting it to `undefined` + // That way js-yaml wouldn't include it in the serialized YAML + // However, that way, this would overwrite (delete) existing properties, e.g.: sourcePath + // Because of this, we spread objects to create properties if necessary + const systemVarsMetadataToSpread = + addSystemMeta && isObject(systemVars) ? {__system: JSON.stringify(systemVars)} : undefined; + const sourcePathMetadataToSpread = addSourcePath && sourcePath ? {sourcePath} : undefined; + const innerMetadataToSpread = + mergedInnerMetadata.length > 0 ? {metadata: mergedInnerMetadata} : undefined; + + const mergedMetadata: FileMetadata = { + ...metadata, + ...resources, + ...additionalMetadata, + ...systemVarsMetadataToSpread, + ...sourcePathMetadataToSpread, + ...innerMetadataToSpread, + }; - return `${getUpdatedMetadataString(newMetadatas)}${fileContent}`; + return `${serializeMetadata(mergedMetadata)}${metadataStrippedContent}`; } async function getContentWithUpdatedDynamicMetadata( @@ -154,10 +181,7 @@ async function getContentWithUpdatedDynamicMetadata( } if (!authorMetadata) { - const { - fileData: {tmpInputFilePath, inputFolderPathLength}, - } = options; - const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength); + const {pathToFile: relativeFilePath} = options.pathData; authorMetadata = await updateAuthorMetadataStringByFilePath( relativeFilePath, options.vcsConnector, @@ -199,43 +223,52 @@ function matchMetadata(fileContent: string) { // Search by format: // --- // main content 123 - const regexpFileContent = '-{3}((.*[\r?\n]*)*)'; + const regexpFileContent = '-{3}\\r?\\n((.*[\r?\n]*)*)'; const regexpParseFileContent = new RegExp(`${regexpMetadata}${regexpFileContent}`, 'gm'); return regexpParseFileContent.exec(fileContent); } +function getFileDataForContributorsService( + pathData: PathData, + fileContent: string, +): ContributorsServiceFileData { + return { + fileContent, + resolvedFilePath: pathData.resolvedPathToFile, + inputFolderPathLength: pathData.inputFolderPath.length, + }; +} + async function getContributorsMetadataString( options: MetaDataOptions, fileContent: string, ): Promise { - const {isContributorsEnabled, vcsConnector, fileData} = options; + const {isContributorsEnabled, vcsConnector, pathData} = options; if (isContributorsEnabled && vcsConnector) { - const updatedFileData = { - ...fileData, - fileContent, - }; - - return getFileContributorsMetadata(updatedFileData, vcsConnector); + return getFileContributorsMetadata( + getFileDataForContributorsService(pathData, fileContent), + vcsConnector, + ); } return undefined; } async function getModifiedTimeMetadataString(options: MetaDataOptions, fileContent: string) { - const {isContributorsEnabled, vcsConnector, fileData} = options; - - const {tmpInputFilePath, inputFolderPathLength} = fileData; + const {isContributorsEnabled, vcsConnector, pathData} = options; - const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength + 1); + const {pathToFile: relativeFilePath} = pathData; if (!isContributorsEnabled || !vcsConnector) { return undefined; } - const includedFiles = await getFileIncludes({...fileData, fileContent}); + const includedFiles = await getFileIncludes( + getFileDataForContributorsService(pathData, fileContent), + ); includedFiles.push(relativeFilePath); const tocCopyFileMap = TocService.getCopyFileMap(); @@ -256,13 +289,14 @@ async function getModifiedTimeMetadataString(options: MetaDataOptions, fileConte } function getUpdatedMetadataString(newMetadatas: string[], defaultMetadata = ''): string { - const newMetadata = newMetadatas.join(сarriage) + (newMetadatas.length ? сarriage : ''); + const newMetadata = + newMetadatas.join(carriageReturn) + (newMetadatas.length ? carriageReturn : ''); const preparedDefaultMetadata = defaultMetadata.trimRight(); - const defaultMetadataСarriage = preparedDefaultMetadata ? сarriage : ''; + const defaultMetadataСarriage = preparedDefaultMetadata ? carriageReturn : ''; const updatedMetadata = `${preparedDefaultMetadata}${defaultMetadataСarriage}${newMetadata}`; - return `${metadataBorder}${сarriage}${updatedMetadata}${metadataBorder}${ - defaultMetadata.length ? '' : сarriage + return `${metadataBorder}${carriageReturn}${updatedMetadata}${metadataBorder}${ + defaultMetadata.length ? '' : carriageReturn }`; } @@ -294,15 +328,13 @@ async function getContributorsMetadata( options: MetaDataOptions, fileContent: string, ): Promise { - const {isContributorsEnabled, vcsConnector, fileData} = options; + const {isContributorsEnabled, vcsConnector, pathData} = options; if (isContributorsEnabled && vcsConnector) { - const updatedFileData = { - ...fileData, - fileContent, - }; - - return getFileContributorsString(updatedFileData, vcsConnector); + return getFileContributorsString( + getFileDataForContributorsService(pathData, fileContent), + vcsConnector, + ); } return JSON.stringify([]); @@ -321,10 +353,6 @@ async function getAuthorMetadata( return null; } -function getSystemVarsMetadataString(systemVars: object) { - return `__system: ${JSON.stringify(systemVars)}`; -} - function getAssetsPublicPath(filePath: string) { const toc: YfmToc | null = TocService.getForPath(filePath) || null; diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index a6ba56d9..b96026fc 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -67,11 +67,7 @@ export async function processPages(outputBundlePath: string): Promise { logger.proc(pathToFile); - const metaDataOptions = getMetaDataOptions( - pathData, - inputFolderPath.length, - vcsConnector, - ); + const metaDataOptions = getMetaDataOptions(pathData, vcsConnector); await preparingPagesByOutputFormat( pathData, @@ -226,22 +222,16 @@ function savePageResultForSinglePage(pageProps: DocInnerProps, pathData: PathDat }); } -function getMetaDataOptions( - pathData: PathData, - inputFolderPathLength: number, - vcsConnector?: VCSConnector, -): MetaDataOptions { - const {contributors, addSystemMeta, resources, allowCustomResources} = ArgvService.getConfig(); +function getMetaDataOptions(pathData: PathData, vcsConnector?: VCSConnector): MetaDataOptions { + const {contributors, addSystemMeta, resources, allowCustomResources, vcs} = + ArgvService.getConfig(); const metaDataOptions: MetaDataOptions = { + pathData, vcsConnector, - fileData: { - tmpInputFilePath: pathData.resolvedPathToFile, - inputFolderPathLength, - fileContent: '', - }, isContributorsEnabled: Boolean(contributors && vcsConnector), addSystemMeta, + shouldAlwaysAddVCSPath: typeof vcs?.remoteBase === 'string' && vcs.remoteBase.length > 0, }; if (allowCustomResources && resources) { diff --git a/src/utils/markup.ts b/src/utils/markup.ts index 6b854faf..6a94fae7 100644 --- a/src/utils/markup.ts +++ b/src/utils/markup.ts @@ -13,7 +13,7 @@ import {isFileExists, resolveRelativePath} from '@diplodoc/transform/lib/utilsFS import {escape} from 'html-escaper'; -export const сarriage = platform === Platforms.WINDOWS ? '\r\n' : '\n'; +export const carriageReturn = platform === Platforms.WINDOWS ? '\r\n' : '\n'; export interface TitleMeta { title?: string; @@ -117,7 +117,7 @@ function getMetadata(metadata: VarsMetadata | undefined, restMeta: LeadingPage[' }, ''); if (args.length) { - result += `` + сarriage; + result += `` + carriageReturn; } };