feat: optimize author obtaining

diplodoc-platform · Aug 29, 2023 · 324bbd1 · 324bbd1
1 parent 4d4b195
commit 324bbd1
Show file tree

Hide file tree

Showing 6 changed files with 233 additions and 166 deletions.
diff --git a/src/models.ts b/src/models.ts
@@ -240,3 +240,8 @@ export type YandexCloudTranslateGlossaryPair = {
   sourceText: string;
   translatedText: string;
 };
+
+export type CommitInfo = {
+    email: string;
+    hashCommit: string;
+};
diff --git a/src/services/authors.ts b/src/services/authors.ts
@@ -1,35 +1,40 @@
-import {replaceDoubleToSingleQuotes, сarriage} from '../utils';
-import {REGEXP_AUTHOR} from '../constants';
+import {replaceDoubleToSingleQuotes} from '../utils';
 import {VCSConnector} from '../vcs-connector/connector-models';
 
-async function updateAuthorMetadataString(
-    defaultMetadata = '',
+async function updateAuthorMetadataStringByAuthorLogin(
+    authorLogin: string,
     vcsConnector?: VCSConnector,
-    filePath?: string | null,
 ): Promise<string> {
     if (!vcsConnector) {
-        return defaultMetadata;
+        return '';
     }
 
-    const matchAuthor = defaultMetadata.match(REGEXP_AUTHOR);
+    const user = await getAuthorDetails(vcsConnector, authorLogin);
 
-    if (matchAuthor && matchAuthor?.length > 0) {
-        const authorLogin = matchAuthor[0];
-        const user = await getAuthorDetails(vcsConnector, authorLogin);
+    if (user) {
+        return user;
+    }
 
-        if (user) {
-            return defaultMetadata.replace(authorLogin, user);
-        }
-    } else if (filePath) {
-        const user = vcsConnector.getExternalAuthorByPath(filePath);
+    return '';
+}
 
-        if (user) {
-            const author = replaceDoubleToSingleQuotes(JSON.stringify(user));
-            return `${defaultMetadata}${сarriage}author: ${author}`;
-        }
+
+async function updateAuthorMetadataStringByFilePath(
+    filePath: string,
+    vcsConnector?: VCSConnector,
+): Promise<string> {
+    if (!vcsConnector) {
+        return '';
+    }
+
+    const user = vcsConnector.getExternalAuthorByPath(filePath);
+
+    if (user) {
+        const author = replaceDoubleToSingleQuotes(JSON.stringify(user));
+        return author;
     }
 
-    return defaultMetadata;
+    return '';
 }
 
 async function getAuthorDetails(vcsConnector: VCSConnector, author: string | object): Promise<string | null> {
@@ -53,6 +58,7 @@ async function getAuthorDetails(vcsConnector: VCSConnector, author: string | obj
 }
 
 export {
-    updateAuthorMetadataString,
+    updateAuthorMetadataStringByAuthorLogin,
+    updateAuthorMetadataStringByFilePath,
     getAuthorDetails,
 };
diff --git a/src/services/metadata.ts b/src/services/metadata.ts
@@ -2,11 +2,15 @@ import {dump} from 'js-yaml';
 
 import {VCSConnector} from '../vcs-connector/connector-models';
 import {Metadata, MetaDataOptions, Resources} from '../models';
-import {getAuthorDetails, updateAuthorMetadataString} from './authors';
+import {
+    getAuthorDetails,
+    updateAuthorMetadataStringByAuthorLogin,
+    updateAuthorMetadataStringByFilePath,
+} from './authors';
 import {getFileContributorsMetadata, getFileContributorsString} from './contributors';
 import {isObject} from './utils';
 import {сarriage} from '../utils';
-import {metadataBorder} from '../constants';
+import {metadataBorder, REGEXP_AUTHOR} from '../constants';
 import {dirname, relative, resolve} from 'path';
 import {ArgvService} from './index';
 
@@ -78,26 +82,54 @@ async function getContentWithUpdatedDynamicMetadata(
         return fileContent;
     }
 
+    let fileMetadata: string | undefined, fileMainContent: string | undefined;
     const matches = matchMetadata(fileContent);
+    if (matches && matches.length > 0) {
+        const [, matchedFileMetadata, , matchedFileMainContent] = matches;
+        fileMetadata = matchedFileMetadata;
+        fileMainContent = matchedFileMainContent;
+    }
+
     const newMetadatas: string[] = [];
 
     const {isContributorsEnabled} = options;
 
     if (isContributorsEnabled) {
         const contributorsMetaData = await getContributorsMetadataString(options, fileContent);
-
         if (contributorsMetaData) {
             newMetadatas.push(contributorsMetaData);
         }
+
+        let authorMetadata = '';
+        if (fileMetadata) {
+            const matchAuthor = fileMetadata.match(REGEXP_AUTHOR);
+            if (matchAuthor) {
+                const matchedAuthor = matchAuthor[0];
+                authorMetadata = await updateAuthorMetadataStringByAuthorLogin(matchedAuthor, options.vcsConnector);
+            }
+        }
+
+        if (!authorMetadata) {
+            const {fileData: {tmpInputFilePath, inputFolderPathLength}} = options;
+            const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength);
+            authorMetadata = await updateAuthorMetadataStringByFilePath(relativeFilePath, options.vcsConnector);
+        }
+
+        if (authorMetadata) {
+            newMetadatas.push(`author: ${authorMetadata}`);
+        }
     }
 
-    if (matches && matches.length > 0) {
-        const [, fileMetadata, , fileMainContent] = matches;
-        let updatedDefaultMetadata = '';
+    if (fileMetadata && fileMainContent) {
+        let updatedFileMetadata = fileMetadata;
+        const matchAuthor = fileMetadata.match(REGEXP_AUTHOR);
 
-        updatedDefaultMetadata = await getAuthorMetadataString(options, fileMetadata);
+        const isNewMetadataIncludesAuthor = newMetadatas.some((item) => /^author: /.test(item));
+        if (matchAuthor && isNewMetadataIncludesAuthor) {
+            updatedFileMetadata = updatedFileMetadata.replace(`author: ${matchAuthor[0]}`, '');
+        }
 
-        return `${getUpdatedMetadataString(newMetadatas, updatedDefaultMetadata)}${fileMainContent}`;
+        return `${getUpdatedMetadataString(newMetadatas, updatedFileMetadata)}${fileMainContent}`;
     }
 
     return `${getUpdatedMetadataString(newMetadatas)}${fileContent}`;
@@ -142,18 +174,6 @@ async function getContributorsMetadataString(
     return undefined;
 }
 
-async function getAuthorMetadataString(options: MetaDataOptions, fileMetadata: string) {
-    const {fileData: {tmpInputFilePath, inputFolderPathLength}} = options;
-
-    const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength);
-
-    return await updateAuthorMetadataString(
-        fileMetadata,
-        options.vcsConnector,
-        relativeFilePath,
-    );
-}
-
 function getUpdatedMetadataString(newMetadatas: string[], defaultMetadata = ''): string {
     const newMetadata = newMetadatas.join(сarriage) + (newMetadatas.length ? сarriage : '');
     const preparedDefaultMetadata = defaultMetadata.trimRight();

diff --git a/src/vcs-connector/github.ts b/src/vcs-connector/github.ts
@@ -1,12 +1,12 @@
 import {Octokit} from '@octokit/core';
 import {join, normalize} from 'path';
 import simpleGit, {SimpleGitOptions} from 'simple-git';
-import {asyncify, mapLimit} from 'async';
 import {minimatch} from 'minimatch';
 
 import github from './client/github';
 import {ArgvService} from '../services';
 import {
+    CommitInfo,
     Contributor,
     Contributors,
     ContributorsByPathFunction,
@@ -28,11 +28,8 @@ import {addSlashPrefix, logger} from '../utils';
 import {validateConnectorFields} from './connector-validator';
 import process from 'process';
 
-const MAX_CONCURRENCY = 99;
-
 const authorByGitEmail: Map<string, Contributor | null> = new Map();
 const authorByPath: Map<string, Contributor | null> = new Map();
-const authorAlreadyCheckedForPath: Map<string, boolean> = new Map();
 const contributorsByPath: Map<string, FileContributors> = new Map();
 const contributorsData: Map<string, Contributor | null> = new Map();
 
@@ -109,7 +106,17 @@ async function getAllContributorsTocFiles(httpClientByToken: Octokit): Promise<v
         );
         const repoLogs = fullRepoLogString.split('\n\n');
         if (process.env.ENABLE_EXPERIMANTAL_AUTHORS) {
-            await matchAuthorsForEachPath(repoLogs, httpClientByToken);
+            const fullAuthorRepoLogString = await simpleGit({
+                baseDir: join(rootInput, masterDir),
+            }).raw(
+                'log',
+                `${FIRST_COMMIT_FROM_ROBOT_IN_GITHUB}..HEAD`,
+                '--diff-filter=A',
+                '--pretty=format:%ae;%an;%H',
+                '--name-only',
+            );
+            const authorRepoLog = fullAuthorRepoLogString.split('\n\n');
+            await matchAuthorsForEachPath(authorRepoLog, httpClientByToken);
         }
         await matchContributionsForEachPath(repoLogs, httpClientByToken);
     } finally {
@@ -160,21 +167,21 @@ async function matchContributionsForEachPath(repoLogs: string[], httpClientByTok
     }
 }
 
-async function matchAuthorsForEachPath(repoLogs: string[], httpClientByToken: Octokit) {
-    for (const repoLog of repoLogs) {
+async function matchAuthorsForEachPath(authorRepoLogs: string[], httpClientByToken: Octokit) {
+    for (const repoLog of authorRepoLogs) {
         if (!repoLog) {
             continue;
         }
 
         const dataArray = repoLog.split('\n');
         const [userData, ...paths] = dataArray;
-        const [email, name] = userData.split(', ');
+        const [email, name, hashCommit] = userData.split(';');
 
         if (shouldAuthorBeIgnored({email, name})) {
             continue;
         }
 
-        await getAuthorByPaths(paths, httpClientByToken);
+        await getAuthorByPaths({email, hashCommit}, paths, httpClientByToken);
     }
 }
 
@@ -203,19 +210,13 @@ async function getContributorDataByHashCommit(httpClientByToken: Octokit, hashCo
     };
 }
 
-async function getAuthorByPaths(paths: string[], httpClientByToken: Octokit) {
-    const externalCommits = (await mapLimit(
-        paths,
-        MAX_CONCURRENCY,
-        asyncify(getAuthorForPath) as typeof getAuthorForPath,
-    )).filter(Boolean);
-
-    for (const externalCommit of externalCommits) {
-        if (!externalCommit) {
+async function getAuthorByPaths(commitInfo: CommitInfo, paths: string[], httpClientByToken: Octokit) {
+    for (const path of paths) {
+        if (!path) {
             continue;
         }
-
-        const {hashCommit, normalizePath, email} = externalCommit;
+        const normalizePath = normalize(addSlashPrefix(path));
+        const {email, hashCommit} = commitInfo;
 
         let authorToReturn = authorByGitEmail.get(email) || null;
 
@@ -243,9 +244,7 @@ async function getAuthorByPaths(paths: string[], httpClientByToken: Octokit) {
             authorByGitEmail.set(email, authorToReturn);
         }
 
-        if (authorToReturn) {
-            authorByPath.set(normalizePath, authorToReturn);
-        }
+        authorByPath.set(normalizePath, authorToReturn);
     }
 }
 
@@ -303,46 +302,6 @@ function addContributorForPath(paths: string[], newContributor: Contributors, ha
     });
 }
 
-async function getAuthorForPath(path: string) {
-    if (!path) {
-        return null;
-    }
-
-    const {rootInput} = ArgvService.getConfig();
-    const masterDir = './_yfm-master';
-    const options: Partial<SimpleGitOptions> = {
-        baseDir: join(rootInput, masterDir),
-    };
-
-    const normalizePath = normalize(addSlashPrefix(path));
-
-    if (authorAlreadyCheckedForPath.has(normalizePath)) {
-        return null;
-    }
-
-    const commitData = await simpleGit(options).raw(
-        'log',
-        `${FIRST_COMMIT_FROM_ROBOT_IN_GITHUB}..HEAD`,
-        '--diff-filter=A',
-        '--pretty=format:%ae;%an;%H',
-        '--',
-        path,
-    );
-
-    const [email, name, hashCommit] = commitData.split(';');
-    if (!(email && hashCommit)) {
-        return null;
-    }
-
-    authorAlreadyCheckedForPath.set(normalizePath, true);
-
-    if (shouldAuthorBeIgnored({email, name})) {
-        return null;
-    }
-
-    return {hashCommit, normalizePath, email};
-}
-
 type ShouldAuthorBeIgnoredArgs = {
     email?: string;
     name?: string;

diff --git a/tests/integrations/services/metadataAuthors.test.ts b/tests/integrations/services/metadataAuthors.test.ts
@@ -1,6 +1,6 @@
 import {readFileSync} from 'fs';
 import {REGEXP_AUTHOR} from '../../../src/constants';
-import {replaceDoubleToSingleQuotes} from '../../../src/utils/markup';
+import {replaceDoubleToSingleQuotes, сarriage} from '../../../src/utils/markup';
 import {MetaDataOptions} from 'models';
 import {getContentWithUpdatedMetadata} from 'services/metadata';
 import {VCSConnector} from 'vcs-connector/connector-models';
@@ -103,6 +103,7 @@ describe('getContentWithUpdatedMetadata (Authors)', () => {
             metaDataOptions.vcsConnector = {
                 ...defaultVCSConnector,
                 getUserByLogin: () => Promise.resolve(null),
+                getExternalAuthorByPath: () => null,
             };
             const fileContent = readFileSync(authorAliasInMetadataFilePath, 'utf8');
 
@@ -117,18 +118,11 @@ describe('getContentWithUpdatedMetadata (Authors)', () => {
             const fileContent = readFileSync(simpleMetadataFilePath, 'utf8');
 
             const updatedFileContent = await getContentWithUpdatedMetadata(fileContent, metaDataOptions);
+            const lastMetadataRow = 'editable: false';
+            const expectedFileContent = fileContent
+                .replace(lastMetadataRow, replaceDoubleToSingleQuotes(`${lastMetadataRow}${сarriage}author: ${replaceDoubleToSingleQuotes(JSON.stringify(expectedAuthorData))}`));
 
-            expect(updatedFileContent).toEqual(fileContent);
-        });
-
-        test('if metadata does not have author', async () => {
-            metaDataOptions.isContributorsEnabled = true;
-            metaDataOptions.vcsConnector = defaultVCSConnector;
-            const fileContent = readFileSync(simpleMetadataFilePath, 'utf8');
-
-            const updatedFileContent = await getContentWithUpdatedMetadata(fileContent, metaDataOptions);
-
-            expect(updatedFileContent).toEqual(fileContent);
+            expect(updatedFileContent).toEqual(expectedFileContent);
         });
     });
 });