Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: optimize author obtaining #534

Merged
merged 1 commit into from
Sep 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -240,3 +240,8 @@ export type YandexCloudTranslateGlossaryPair = {
sourceText: string;
translatedText: string;
};

export type CommitInfo = {
email: string;
hashCommit: string;
};
48 changes: 27 additions & 21 deletions src/services/authors.ts
Original file line number Diff line number Diff line change
@@ -1,35 +1,40 @@
import {replaceDoubleToSingleQuotes, сarriage} from '../utils';
import {REGEXP_AUTHOR} from '../constants';
import {replaceDoubleToSingleQuotes} from '../utils';
import {VCSConnector} from '../vcs-connector/connector-models';

async function updateAuthorMetadataString(
defaultMetadata = '',
async function updateAuthorMetadataStringByAuthorLogin(
authorLogin: string,
vcsConnector?: VCSConnector,
filePath?: string | null,
): Promise<string> {
if (!vcsConnector) {
return defaultMetadata;
return '';
}

const matchAuthor = defaultMetadata.match(REGEXP_AUTHOR);
const user = await getAuthorDetails(vcsConnector, authorLogin);

if (matchAuthor && matchAuthor?.length > 0) {
const authorLogin = matchAuthor[0];
const user = await getAuthorDetails(vcsConnector, authorLogin);
if (user) {
return user;
}

if (user) {
return defaultMetadata.replace(authorLogin, user);
}
} else if (filePath) {
const user = vcsConnector.getExternalAuthorByPath(filePath);
return '';
}

if (user) {
const author = replaceDoubleToSingleQuotes(JSON.stringify(user));
return `${defaultMetadata}${сarriage}author: ${author}`;
}

async function updateAuthorMetadataStringByFilePath(
filePath: string,
vcsConnector?: VCSConnector,
): Promise<string> {
if (!vcsConnector) {
return '';
}

const user = vcsConnector.getExternalAuthorByPath(filePath);

if (user) {
const author = replaceDoubleToSingleQuotes(JSON.stringify(user));
return author;
}

return defaultMetadata;
return '';
}

async function getAuthorDetails(vcsConnector: VCSConnector, author: string | object): Promise<string | null> {
Expand All @@ -53,6 +58,7 @@ async function getAuthorDetails(vcsConnector: VCSConnector, author: string | obj
}

export {
updateAuthorMetadataString,
updateAuthorMetadataStringByAuthorLogin,
updateAuthorMetadataStringByFilePath,
getAuthorDetails,
};
60 changes: 40 additions & 20 deletions src/services/metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,15 @@ import {dump} from 'js-yaml';

import {VCSConnector} from '../vcs-connector/connector-models';
import {Metadata, MetaDataOptions, Resources} from '../models';
import {getAuthorDetails, updateAuthorMetadataString} from './authors';
import {
getAuthorDetails,
updateAuthorMetadataStringByAuthorLogin,
updateAuthorMetadataStringByFilePath,
} from './authors';
import {getFileContributorsMetadata, getFileContributorsString} from './contributors';
import {isObject} from './utils';
import {сarriage} from '../utils';
import {metadataBorder} from '../constants';
import {metadataBorder, REGEXP_AUTHOR} from '../constants';
import {dirname, relative, resolve} from 'path';
import {ArgvService} from './index';

Expand Down Expand Up @@ -78,26 +82,54 @@ async function getContentWithUpdatedDynamicMetadata(
return fileContent;
}

let fileMetadata: string | undefined, fileMainContent: string | undefined;
const matches = matchMetadata(fileContent);
if (matches && matches.length > 0) {
const [, matchedFileMetadata, , matchedFileMainContent] = matches;
fileMetadata = matchedFileMetadata;
fileMainContent = matchedFileMainContent;
}

const newMetadatas: string[] = [];

const {isContributorsEnabled} = options;

if (isContributorsEnabled) {
const contributorsMetaData = await getContributorsMetadataString(options, fileContent);

if (contributorsMetaData) {
newMetadatas.push(contributorsMetaData);
}

let authorMetadata = '';
if (fileMetadata) {
const matchAuthor = fileMetadata.match(REGEXP_AUTHOR);
if (matchAuthor) {
const matchedAuthor = matchAuthor[0];
authorMetadata = await updateAuthorMetadataStringByAuthorLogin(matchedAuthor, options.vcsConnector);
}
}

if (!authorMetadata) {
const {fileData: {tmpInputFilePath, inputFolderPathLength}} = options;
const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength);
authorMetadata = await updateAuthorMetadataStringByFilePath(relativeFilePath, options.vcsConnector);
}

if (authorMetadata) {
newMetadatas.push(`author: ${authorMetadata}`);
}
}

if (matches && matches.length > 0) {
const [, fileMetadata, , fileMainContent] = matches;
let updatedDefaultMetadata = '';
if (fileMetadata && fileMainContent) {
let updatedFileMetadata = fileMetadata;
const matchAuthor = fileMetadata.match(REGEXP_AUTHOR);

updatedDefaultMetadata = await getAuthorMetadataString(options, fileMetadata);
const isNewMetadataIncludesAuthor = newMetadatas.some((item) => /^author: /.test(item));
if (matchAuthor && isNewMetadataIncludesAuthor) {
updatedFileMetadata = updatedFileMetadata.replace(`author: ${matchAuthor[0]}`, '');
}

return `${getUpdatedMetadataString(newMetadatas, updatedDefaultMetadata)}${fileMainContent}`;
return `${getUpdatedMetadataString(newMetadatas, updatedFileMetadata)}${fileMainContent}`;
}

return `${getUpdatedMetadataString(newMetadatas)}${fileContent}`;
Expand Down Expand Up @@ -142,18 +174,6 @@ async function getContributorsMetadataString(
return undefined;
}

async function getAuthorMetadataString(options: MetaDataOptions, fileMetadata: string) {
const {fileData: {tmpInputFilePath, inputFolderPathLength}} = options;

const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength);

return await updateAuthorMetadataString(
fileMetadata,
options.vcsConnector,
relativeFilePath,
);
}

function getUpdatedMetadataString(newMetadatas: string[], defaultMetadata = ''): string {
const newMetadata = newMetadatas.join(сarriage) + (newMetadatas.length ? сarriage : '');
const preparedDefaultMetadata = defaultMetadata.trimRight();
Expand Down
85 changes: 22 additions & 63 deletions src/vcs-connector/github.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import {Octokit} from '@octokit/core';
import {join, normalize} from 'path';
import simpleGit, {SimpleGitOptions} from 'simple-git';
import {asyncify, mapLimit} from 'async';
import {minimatch} from 'minimatch';

import github from './client/github';
import {ArgvService} from '../services';
import {
CommitInfo,
Contributor,
Contributors,
ContributorsByPathFunction,
Expand All @@ -28,11 +28,8 @@ import {addSlashPrefix, logger} from '../utils';
import {validateConnectorFields} from './connector-validator';
import process from 'process';

const MAX_CONCURRENCY = 99;

const authorByGitEmail: Map<string, Contributor | null> = new Map();
const authorByPath: Map<string, Contributor | null> = new Map();
const authorAlreadyCheckedForPath: Map<string, boolean> = new Map();
const contributorsByPath: Map<string, FileContributors> = new Map();
const contributorsData: Map<string, Contributor | null> = new Map();

Expand Down Expand Up @@ -109,7 +106,17 @@ async function getAllContributorsTocFiles(httpClientByToken: Octokit): Promise<v
);
const repoLogs = fullRepoLogString.split('\n\n');
if (process.env.ENABLE_EXPERIMANTAL_AUTHORS) {
await matchAuthorsForEachPath(repoLogs, httpClientByToken);
const fullAuthorRepoLogString = await simpleGit({
baseDir: join(rootInput, masterDir),
}).raw(
'log',
`${FIRST_COMMIT_FROM_ROBOT_IN_GITHUB}..HEAD`,
'--diff-filter=A',
'--pretty=format:%ae;%an;%H',
'--name-only',
);
const authorRepoLog = fullAuthorRepoLogString.split('\n\n');
await matchAuthorsForEachPath(authorRepoLog, httpClientByToken);
}
await matchContributionsForEachPath(repoLogs, httpClientByToken);
} finally {
Expand Down Expand Up @@ -160,21 +167,21 @@ async function matchContributionsForEachPath(repoLogs: string[], httpClientByTok
}
}

async function matchAuthorsForEachPath(repoLogs: string[], httpClientByToken: Octokit) {
for (const repoLog of repoLogs) {
async function matchAuthorsForEachPath(authorRepoLogs: string[], httpClientByToken: Octokit) {
for (const repoLog of authorRepoLogs) {
if (!repoLog) {
continue;
}

const dataArray = repoLog.split('\n');
const [userData, ...paths] = dataArray;
const [email, name] = userData.split(', ');
const [email, name, hashCommit] = userData.split(';');

if (shouldAuthorBeIgnored({email, name})) {
continue;
}

await getAuthorByPaths(paths, httpClientByToken);
await getAuthorByPaths({email, hashCommit}, paths, httpClientByToken);
}
}

Expand Down Expand Up @@ -203,19 +210,13 @@ async function getContributorDataByHashCommit(httpClientByToken: Octokit, hashCo
};
}

async function getAuthorByPaths(paths: string[], httpClientByToken: Octokit) {
const externalCommits = (await mapLimit(
paths,
MAX_CONCURRENCY,
asyncify(getAuthorForPath) as typeof getAuthorForPath,
)).filter(Boolean);

for (const externalCommit of externalCommits) {
if (!externalCommit) {
async function getAuthorByPaths(commitInfo: CommitInfo, paths: string[], httpClientByToken: Octokit) {
for (const path of paths) {
if (!path) {
continue;
}

const {hashCommit, normalizePath, email} = externalCommit;
const normalizePath = normalize(addSlashPrefix(path));
const {email, hashCommit} = commitInfo;

let authorToReturn = authorByGitEmail.get(email) || null;

Expand Down Expand Up @@ -243,9 +244,7 @@ async function getAuthorByPaths(paths: string[], httpClientByToken: Octokit) {
authorByGitEmail.set(email, authorToReturn);
}

if (authorToReturn) {
authorByPath.set(normalizePath, authorToReturn);
}
authorByPath.set(normalizePath, authorToReturn);
}
}

Expand Down Expand Up @@ -303,46 +302,6 @@ function addContributorForPath(paths: string[], newContributor: Contributors, ha
});
}

async function getAuthorForPath(path: string) {
if (!path) {
return null;
}

const {rootInput} = ArgvService.getConfig();
const masterDir = './_yfm-master';
const options: Partial<SimpleGitOptions> = {
baseDir: join(rootInput, masterDir),
};

const normalizePath = normalize(addSlashPrefix(path));

if (authorAlreadyCheckedForPath.has(normalizePath)) {
return null;
}

const commitData = await simpleGit(options).raw(
'log',
`${FIRST_COMMIT_FROM_ROBOT_IN_GITHUB}..HEAD`,
'--diff-filter=A',
'--pretty=format:%ae;%an;%H',
'--',
path,
);

const [email, name, hashCommit] = commitData.split(';');
if (!(email && hashCommit)) {
return null;
}

authorAlreadyCheckedForPath.set(normalizePath, true);

if (shouldAuthorBeIgnored({email, name})) {
return null;
}

return {hashCommit, normalizePath, email};
}

type ShouldAuthorBeIgnoredArgs = {
email?: string;
name?: string;
Expand Down
18 changes: 6 additions & 12 deletions tests/integrations/services/metadataAuthors.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {readFileSync} from 'fs';
import {REGEXP_AUTHOR} from '../../../src/constants';
import {replaceDoubleToSingleQuotes} from '../../../src/utils/markup';
import {replaceDoubleToSingleQuotes, сarriage} from '../../../src/utils/markup';
import {MetaDataOptions} from 'models';
import {getContentWithUpdatedMetadata} from 'services/metadata';
import {VCSConnector} from 'vcs-connector/connector-models';
Expand Down Expand Up @@ -103,6 +103,7 @@ describe('getContentWithUpdatedMetadata (Authors)', () => {
metaDataOptions.vcsConnector = {
...defaultVCSConnector,
getUserByLogin: () => Promise.resolve(null),
getExternalAuthorByPath: () => null,
};
const fileContent = readFileSync(authorAliasInMetadataFilePath, 'utf8');

Expand All @@ -117,18 +118,11 @@ describe('getContentWithUpdatedMetadata (Authors)', () => {
const fileContent = readFileSync(simpleMetadataFilePath, 'utf8');

const updatedFileContent = await getContentWithUpdatedMetadata(fileContent, metaDataOptions);
const lastMetadataRow = 'editable: false';
const expectedFileContent = fileContent
.replace(lastMetadataRow, replaceDoubleToSingleQuotes(`${lastMetadataRow}${сarriage}author: ${replaceDoubleToSingleQuotes(JSON.stringify(expectedAuthorData))}`));

expect(updatedFileContent).toEqual(fileContent);
});

test('if metadata does not have author', async () => {
metaDataOptions.isContributorsEnabled = true;
metaDataOptions.vcsConnector = defaultVCSConnector;
const fileContent = readFileSync(simpleMetadataFilePath, 'utf8');

const updatedFileContent = await getContentWithUpdatedMetadata(fileContent, metaDataOptions);

expect(updatedFileContent).toEqual(fileContent);
expect(updatedFileContent).toEqual(expectedFileContent);
});
});
});
Loading
Loading