feat(authors): scrape authors from github

diplodoc-platform · Mar 28, 2023 · e5c9ddf · e5c9ddf
1 parent f71c655
commit e5c9ddf
Show file tree

Hide file tree

Showing 10 changed files with 122 additions and 20 deletions.
diff --git a/src/constants.ts b/src/constants.ts
@@ -110,6 +110,7 @@ export const REGEXP_INCLUDE_FILE_PATH = /(?<=[(]).+(?=[)])/g;
 // Include example: author: authorLogin
 // Regexp result: authorLogin
 export const REGEXP_AUTHOR = /(?<=author:\s).+(?=\r?\n)/g;
+export const REGEXP_CONTRIBUTORS = /(?<=contributors:\s).+(?=\r?\n)*?/g;
 
 export const MIN_CHUNK_SIZE = Number(process.env.MIN_CHUNK_SIZE) || 1000;
 export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || (os.cpus().length - 1);

diff --git a/src/models.ts b/src/models.ts
@@ -162,6 +162,8 @@ export interface Contributor {
     login: string;
     name: string;
     url: string;
+    date?: string;
+    hash?: string;
 }
 
 export interface Contributors {

diff --git a/src/services/authors.ts b/src/services/authors.ts
@@ -1,24 +1,62 @@
-import {replaceDoubleToSingleQuotes} from '../utils';
-import {REGEXP_AUTHOR} from '../constants';
+import os from 'os';
+
+import {logger, replaceDoubleToSingleQuotes} from '../utils';
+import {REGEXP_AUTHOR, REGEXP_CONTRIBUTORS} from '../constants';
 import {VCSConnector} from '../vcs-connector/connector-models';
+import {isEarlier, parseDataFromJSONString} from './utils';
+import {Contributor} from '../models';
+
+
+export function getContributorArrayFromMetadata(metadata: string[], filePath = '') {
+    const contributors: Contributor[] = [];
+
+    for (const meta of metadata) {
+        const matchContributors = meta.match(REGEXP_CONTRIBUTORS);
+        try {
+            if (matchContributors) {
+                contributors.push(...parseDataFromJSONString(matchContributors[0]));
+            }
+        } catch (err) {
+            logger.warn(filePath, JSON.stringify(err));
+        }
+    }
+
+    return contributors;
+}
 
-async function updateAuthorMetadataString(defaultMetadata = '', vcsConnector?: VCSConnector): Promise<string> {
+
+async function updateAuthorMetadataString(
+    fileMetadata = '',
+    vcsConnector?: VCSConnector,
+    newMetadatas: string[] = [],
+): Promise<string> {
     if (!vcsConnector) {
-        return defaultMetadata;
+        return fileMetadata;
     }
 
-    const matchAuthor = defaultMetadata.match(REGEXP_AUTHOR);
+    const matchAuthor = fileMetadata.match(REGEXP_AUTHOR);
 
     if (matchAuthor && matchAuthor?.length > 0) {
         const authorLogin = matchAuthor[0];
         const user = await getAuthorDetails(vcsConnector, authorLogin);
 
         if (user) {
-            return defaultMetadata.replace(authorLogin, user);
+            return fileMetadata.replace(authorLogin, user);
+        }
+    }
+
+    const contributors = getContributorArrayFromMetadata(newMetadatas);
+
+    if (contributors.length && !matchAuthor) {
+        contributors.sort((a, b) => isEarlier(a.date, b.date) ? -1 : 1);
+        const user = await getAuthorDetails(vcsConnector, contributors[0]);
+
+        if (user) {
+            return `${fileMetadata}${os.EOL}author: ${user}`;
         }
     }
 
-    return defaultMetadata;
+    return fileMetadata;
 }
 
 async function getAuthorDetails(vcsConnector: VCSConnector, author: string | object): Promise<string | null> {

diff --git a/src/services/metadata.ts b/src/services/metadata.ts
@@ -95,7 +95,11 @@ async function getContentWithUpdatedDynamicMetadata(
         const [, fileMetadata, , fileMainContent] = matches;
         let updatedDefaultMetadata = '';
 
-        updatedDefaultMetadata = await updateAuthorMetadataString(fileMetadata, options.vcsConnector);
+        updatedDefaultMetadata = await updateAuthorMetadataString(
+            fileMetadata,
+            options.vcsConnector,
+            newMetadatas,
+        );
 
         return `${getUpdatedMetadataString(newMetadatas, updatedDefaultMetadata)}${fileMainContent}`;
     }

diff --git a/src/services/utils.ts b/src/services/utils.ts
@@ -2,6 +2,7 @@ import evalExp from '@doc-tools/transform/lib/liquid/evaluation';
 import {Filter, TextItems} from '../models';
 import liquid from '@doc-tools/transform/lib/liquid';
 import {ArgvService} from './index';
+import {replaceSingleToDoubleQuotes} from '../utils';
 
 export interface FilterFilesOptions {
     resolveConditions?: boolean;
@@ -149,3 +150,19 @@ export function liquidField(input: string, vars: Record<string, unknown>, path:
 export function isObject(o: unknown): o is object {
     return typeof o === 'object' && o !== null;
 }
+
+export function isEarlier(oldDate?: string, newDate?: string) {
+    if (!newDate) {
+        return false;
+    }
+
+    if (!oldDate) {
+        return true;
+    }
+
+    return new Date(oldDate) > new Date(newDate);
+}
+
+export function parseDataFromJSONString(dataString: string) {
+    return JSON.parse(replaceSingleToDoubleQuotes(dataString));
+}
diff --git a/src/utils/markup.ts b/src/utils/markup.ts
@@ -117,3 +117,7 @@ export function joinSinglePageResults(singlePageResults: SinglePageResult[], roo
 export function replaceDoubleToSingleQuotes(str: string): string {
     return str.replace(/"/g, '\'');
 }
+
+export function replaceSingleToDoubleQuotes(str: string): string {
+    return str.replace(/'/g, '"');
+}
diff --git a/src/vcs-connector/connector-models.ts b/src/vcs-connector/connector-models.ts
@@ -51,6 +51,7 @@ export interface GithubCommitDTO {
         author: {
             name: string;
             email: string;
+            date: string;
         };
     };
     author: {

diff --git a/src/vcs-connector/github.ts b/src/vcs-connector/github.ts
@@ -17,6 +17,7 @@ import {
 } from '../constants';
 import {addSlashPrefix, execAsync, logger} from '../utils';
 import {validateConnectorFields} from './connector-validator';
+import {isEarlier} from '../services/utils';
 
 const contributorsByPath: Map<string, FileContributors> = new Map();
 const contributorsData: Map<string, Contributor | null> = new Map();
@@ -146,6 +147,7 @@ async function getContributorDataByHashCommit(httpClientByToken: Octokit, hashCo
         login,
         name: commit.author.name,
         url,
+        date: commit.author.date,
     };
 }
 
@@ -188,18 +190,36 @@ function addContributorForPath(paths: string[], newContributor: Contributors, ha
                 contributors: newContributor,
                 hasIncludes,
             });
+
             return;
         }
 
         const oldContributors = contributorsByPath.get(normalizePath);
+        const newContributorKey = Object.keys(newContributor)?.[0];
 
-        contributorsByPath.set(normalizePath, {
-            contributors: {
+        if (newContributorKey) {
+            const contributors = {
                 ...oldContributors?.contributors,
                 ...newContributor,
-            },
-            hasIncludes,
-        });
+            };
+
+            if (!oldContributors?.contributors?.[newContributorKey]) {
+                contributorsByPath.set(normalizePath, {
+                    contributors,
+                    hasIncludes,
+                });
+            } else if (
+                isEarlier(
+                newContributor[newContributorKey]?.date,
+                oldContributors?.contributors[newContributorKey]?.date,
+                )
+            ) {
+                contributorsByPath.set(normalizePath, {
+                    contributors,
+                    hasIncludes,
+                });
+            }
+        }
     });
 }
 

diff --git a/tests/integrations/services/metadataContributors.test.ts b/tests/integrations/services/metadataContributors.test.ts
@@ -1,3 +1,4 @@
+import os from 'os';
 import {readFileSync} from 'fs';
 import {normalize} from 'path';
 import {metadataBorder} from '../../../src/constants';
@@ -85,6 +86,8 @@ describe('getContentWithUpdatedMetadata (Contributors)', () => {
                 [contributorFirst.email]: contributorFirst,
                 [contributorSecond.email]: contributorSecond,
             };
+            const expectedAuthorString: string = replaceDoubleToSingleQuotes(
+                JSON.stringify(contributorFirst));
             const expectedContributorsArray: Contributor[] = Object.values(expectedContributors);
             const expectedContributorsString: string =
                 replaceDoubleToSingleQuotes(JSON.stringify(expectedContributorsArray));
@@ -99,7 +102,7 @@ describe('getContentWithUpdatedMetadata (Contributors)', () => {
 
             const splitedFiledContent = fileContent.split(metadataBorder);
             splitedFiledContent[1] =
-                `${splitedFiledContent[1]}contributors: ${expectedContributorsString}${сarriage}`;
+                `${splitedFiledContent[1]}${os.EOL}author: ${expectedAuthorString}${os.EOL}contributors: ${expectedContributorsString}${сarriage}`;
             const expectedFileContent = splitedFiledContent.join(metadataBorder);
             expect(updatedFileContent).toEqual(expectedFileContent);
         });
@@ -116,6 +119,8 @@ describe('getContentWithUpdatedMetadata (Contributors)', () => {
             const expectedContributors: Contributors = {
                 [contributorFirst.email]: contributorFirst,
             };
+            const expectedAuthorString: string = replaceDoubleToSingleQuotes(
+                JSON.stringify(contributorFirst));
             const expectedContributorsArray: Contributor[] = Object.values(expectedContributors);
             const expectedContributorsString: string =
                 replaceDoubleToSingleQuotes(JSON.stringify(expectedContributorsArray));
@@ -130,7 +135,7 @@ describe('getContentWithUpdatedMetadata (Contributors)', () => {
 
             const splitedFiledContent = fileContent.split(metadataBorder);
             splitedFiledContent[1] =
-                `${splitedFiledContent[1]}contributors: ${expectedContributorsString}${сarriage}`;
+                `${splitedFiledContent[1]}${os.EOL}author: ${expectedAuthorString}${os.EOL}contributors: ${expectedContributorsString}${сarriage}`;
             const expectedFileContent = splitedFiledContent.join(metadataBorder);
             expect(updatedFileContent).toEqual(expectedFileContent);
         });
@@ -195,7 +200,8 @@ describe('getContentWithUpdatedMetadata (Contributors)', () => {
                 `and includes files and ${item.title}`, async () => {
                 const expectedContributorsString: string = replaceDoubleToSingleQuotes(
                     JSON.stringify(item.expectedContributorsArray));
-
+                const expectedAuthorString: string = replaceDoubleToSingleQuotes(
+                    JSON.stringify(contributorFirst));
                 metaDataOptions.vcsConnector.getContributorsByPath = (path: string) => Promise.resolve({
                     contributors: getFileContributors(path),
                     hasIncludes: item.getHasIncludes(path),
@@ -207,7 +213,7 @@ describe('getContentWithUpdatedMetadata (Contributors)', () => {
 
                 const splitedFiledContent = fileContent.split(metadataBorder);
                 splitedFiledContent[1] =
-                    `${splitedFiledContent[1]}contributors: ${expectedContributorsString}${сarriage}`;
+                    `${splitedFiledContent[1]}${os.EOL}author: ${expectedAuthorString}${os.EOL}contributors: ${expectedContributorsString}${сarriage}`;
                 const expectedFileContent = splitedFiledContent.join(metadataBorder);
                 expect(updatedFileContent).toEqual(expectedFileContent);
             });

diff --git a/tests/units/services/authors.test.ts b/tests/units/services/authors.test.ts
@@ -99,7 +99,10 @@ describe('updateAuthorMetadataString', () => {
         test('when "defaultMetadata" is empty', async () => {
             const expectedMetadata = '';
 
-            const authorDetails = await updateAuthorMetadataString(expectedMetadata, defaultVCSConnector);
+            const authorDetails = await updateAuthorMetadataString(
+                expectedMetadata, 
+                defaultVCSConnector,
+            );
 
             expect(authorDetails).toEqual(expectedMetadata);
         });
@@ -111,7 +114,10 @@ describe('updateAuthorMetadataString', () => {
 
             defaultVCSConnector.getUserByLogin = () => Promise.resolve(null);
 
-            const updatedMetadata = await updateAuthorMetadataString(expectedMetadata, defaultVCSConnector);
+            const updatedMetadata = await updateAuthorMetadataString(
+                expectedMetadata, 
+                defaultVCSConnector,
+            );
 
             expect(updatedMetadata).toEqual(expectedMetadata);
         });
@@ -125,7 +131,10 @@ describe('updateAuthorMetadataString', () => {
             const authorDetails = units.replaceDoubleToSingleQuotes(JSON.stringify(author));
             defaultVCSConnector.getUserByLogin = () => Promise.resolve(author);
 
-            const updatedMetadata = await updateAuthorMetadataString(defaultMetadata, defaultVCSConnector);
+            const updatedMetadata = await updateAuthorMetadataString(
+                defaultMetadata, 
+                defaultVCSConnector,
+            );
 
             const matchAuthor = defaultMetadata.match(REGEXP_AUTHOR);
             const expectedMetadata = defaultMetadata.replace(matchAuthor[0], authorDetails);