Skip to content

Commit

Permalink
feat: new build pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
@makamekm
makamekm committed Oct 30, 2024
1 parent dd0e1c9 commit 618c3af
Show file tree
Hide file tree
Showing 29 changed files with 1,228 additions and 298 deletions.
176 changes: 124 additions & 52 deletions src/cmd/build/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,31 @@
import glob from 'glob';
import {Arguments, Argv} from 'yargs';
import {join, resolve} from 'path';
import shell from 'shelljs';
import glob from 'glob';

import OpenapiIncluder from '@diplodoc/openapi-extension/includer';

import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants';
import {argvValidator} from '../../validator';
import {ArgvService, Includers, SearchService} from '../../services';
import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '~/constants';
import {argvValidator} from '~/validator';
import {Includer, YfmArgv} from '~/models';
import {ArgvService, Includers, SearchService, TocService} from '~/services';
import {
initLinterWorkers,
finishProcessPages,
getLintFn,
getProcessPageFn,
processAssets,
processChangelogs,
processExcludedFiles,
processLinter,
processLogs,
processPages,
processServiceFiles,
} from '../../steps';
import {prepareMapFile} from '../../steps/processMapFile';
import {copyFiles, logger} from '../../utils';
import {upload as publishFilesToS3} from '../../commands/publish/upload';
} from '~/steps';
import {prepareMapFile} from '~/steps/processMapFile';
import {copyFiles, logger} from '~/utils';
import {upload as publishFilesToS3} from '~/commands/publish/upload';
import {RevisionContext, makeRevisionContext, setRevisionContext} from '~/context/context';
import {FsContextCli} from '~/context/fs';
import {DependencyContextCli} from '~/context/dependency';
import {FileQueueProcessor} from '~/context/processor';

export const build = {
command: ['build', '$0'],
Expand All @@ -43,6 +48,24 @@ function builder<T>(argv: Argv<T>) {
type: 'string',
group: 'Build options:',
})
.option('plugins', {
alias: 'p',
describe: 'Path to plugins js file',
type: 'string',
group: 'Build options:',
})
.option('cached', {
default: false,
describe: 'Use cache from revision meta file',
type: 'boolean',
group: 'Build options:',
})
.option('clean', {
default: false,
describe: 'Remove output folder before build',
type: 'boolean',
group: 'Build options:',
})
.option('varsPreset', {
default: 'default',
describe: 'Target vars preset of documentation <external|internal>',
Expand Down Expand Up @@ -175,24 +198,30 @@ function builder<T>(argv: Argv<T>) {
);
}

async function handler(args: Arguments<any>) {
const userOutputFolder = resolve(args.output);
const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);

async function handler(args: Arguments<YfmArgv>) {
if (typeof VERSION !== 'undefined') {
console.log(`Using v${VERSION} version`);
}

shell.config.silent = true;

let hasError = false;

const userInputFolder = resolve(args.input);
const userOutputFolder = resolve(args.output);
const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);

try {
// Init singletone services
ArgvService.init({
...args,
rootInput: args.input,
rootInput: userInputFolder,
input: tmpInputFolder,
output: tmpOutputFolder,
});
SearchService.init();
Includers.init([OpenapiIncluder as any]);
Includers.init([OpenapiIncluder as Includer]);

const {
output: outputFolderPath,
Expand All @@ -203,41 +232,77 @@ async function handler(args: Arguments<any>) {
addMapFile,
} = ArgvService.getConfig();

preparingTemporaryFolders(userOutputFolder);
const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);

if (args.clean) {
await clearTemporaryFolders(userOutputFolder);
}

// Create build context that stores the information about the current build
const context = await makeRevisionContext(
args.cached,
userInputFolder,
userOutputFolder,
tmpInputFolder,
tmpOutputFolder,
outputBundlePath,
);

const fs = new FsContextCli(context);
const deps = new DependencyContextCli(context);

await processServiceFiles();
processExcludedFiles();
// Creating temp .input & .output folder
await preparingTemporaryFolders(context);

// Read and prepare Preset & Toc data
await processServiceFiles(context, fs);

// Removes all content files that unspecified in toc files or ignored.
await processExcludedFiles();

// Write files.json
if (addMapFile) {
prepareMapFile();
await prepareMapFile();
}

const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);
// Collect navigation paths as entry files
const navigationPaths = TocService.getNavigationPaths();

// 1. Linting
if (!lintDisabled) {
/* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */
await initLinterWorkers();
const pageLintProcessor = new FileQueueProcessor(context, deps);
pageLintProcessor.setNavigationPaths(navigationPaths);

const processLintPageFn = await getLintFn(context);
await pageLintProcessor.processQueue(processLintPageFn);
}

const processes = [
!lintDisabled && processLinter(),
!buildDisabled && processPages(outputBundlePath),
].filter(Boolean) as Promise<void>[];
// 2. Building
if (!buildDisabled) {
const pageProcessor = new FileQueueProcessor(context, deps);
pageProcessor.setNavigationPaths(navigationPaths);

const processPageFn = await getProcessPageFn(fs, deps, context, outputBundlePath);
await pageProcessor.processQueue(processPageFn);

await Promise.all(processes);
// Save single pages & redirects
await finishProcessPages(fs);

if (!buildDisabled) {
// process additional files
processAssets({
// Process asset files
await processAssets({
args,
outputFormat,
outputBundlePath,
tmpOutputFolder,
userOutputFolder,
context,
fs,
});

// Process changelogs
await processChangelogs();

// Finish search service processing
await SearchService.release();

// Copy all generated files to user' output folder
Expand All @@ -246,6 +311,7 @@ async function handler(args: Arguments<any>) {
shell.cp('-r', join(tmpOutputFolder, '.*'), userOutputFolder);
}

// Upload the files to S3
if (publish) {
const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? '';
const {
Expand All @@ -269,35 +335,41 @@ async function handler(args: Arguments<any>) {
secretAccessKey,
});
}

// Save .revision.meta.json file for the future processing
await setRevisionContext(context);
}
} catch (err) {
logger.error('', err.message);

hasError = true;
} finally {
// Print logs
processLogs(tmpInputFolder);

shell.rm('-rf', tmpInputFolder, tmpOutputFolder);
}
}

function preparingTemporaryFolders(userOutputFolder: string) {
const args = ArgvService.getConfig();
// If build has some errors, then exit with error code 1
if (hasError) {
process.exit(1);
}
}

shell.mkdir('-p', userOutputFolder);
// Creating temp .input & .output folder
async function preparingTemporaryFolders(context: RevisionContext) {
shell.mkdir('-p', context.userOutputFolder);

// Create temporary input/output folders
shell.rm('-rf', args.input, args.output);
shell.mkdir(args.input, args.output);

copyFiles(
args.rootInput,
args.input,
glob.sync('**', {
cwd: args.rootInput,
nodir: true,
follow: true,
ignore: ['node_modules/**', '*/node_modules/**'],
}),
);

shell.chmod('-R', 'u+w', args.input);
shell.rm('-rf', context.tmpInputFolder, context.tmpOutputFolder);
shell.mkdir(context.tmpInputFolder, context.tmpOutputFolder);

await copyFiles(context.userInputFolder, context.tmpInputFolder, context.files, context.meta);

shell.chmod('-R', 'u+w', context.tmpInputFolder);
}

// Clear output folder folders
async function clearTemporaryFolders(userOutputFolder: string) {
shell.rm('-rf', userOutputFolder);
}
58 changes: 58 additions & 0 deletions src/context/context.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import {
RevisionContext as RevisionContextTransfrom,
RevisionMeta,
} from '@diplodoc/transform/lib/typings';
import glob from 'glob';
import {getMetaFile, makeMetaFile, updateChangedMetaFile, updateMetaFile} from '~/utils/meta';

export interface RevisionContext extends RevisionContextTransfrom {
userInputFolder: string;
userOutputFolder: string;
tmpInputFolder: string;
tmpOutputFolder: string;
outputBundlePath: string;
}

export async function makeRevisionContext(
cached: boolean,
userInputFolder: string,
userOutputFolder: string,
tmpInputFolder: string,
tmpOutputFolder: string,
outputBundlePath: string,
): Promise<RevisionContext> {
const files = glob.sync('**', {
cwd: userInputFolder,
nodir: true,
follow: true,
ignore: ['node_modules/**', '*/node_modules/**'],
});

const meta = normalizeMeta(await getMetaFile(userOutputFolder));

await updateMetaFile(cached, userInputFolder, meta.files, files);

await updateChangedMetaFile(cached, userInputFolder, meta.files);

return {
userInputFolder,
userOutputFolder,
tmpInputFolder,
tmpOutputFolder,
outputBundlePath,
files,
meta,
};
}

function normalizeMeta(meta?: RevisionMeta | undefined | null) {
const metaSafe: RevisionMeta = meta ?? {
files: {},
};
metaSafe.files = metaSafe.files ?? {};
return metaSafe;
}

export async function setRevisionContext(context: RevisionContext): Promise<void> {
await makeMetaFile(context.userOutputFolder, context.files, context.meta);
}
62 changes: 62 additions & 0 deletions src/context/dependency.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import {resolve} from 'path';
import {DependencyContext} from '@diplodoc/transform/lib/typings';
import {RevisionContext} from './context';

export class DependencyContextCli implements DependencyContext {
private context: RevisionContext;

constructor(context: RevisionContext) {
this.context = context;
}

getAssetPath(path: string) {
const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/');
if (isFromTmpInputFolder) {
const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', '');
return assetPath;
}

const isFromInputFolder = path.startsWith(resolve(this.context.userInputFolder) + '/');
if (isFromInputFolder) {
const assetPath = path.replace(resolve(this.context.userInputFolder) + '/', '');
return assetPath;
}

return path;
}

markDep(path: string, dependencyPath: string, type?: string): void {
type = type ?? 'include';

const assetPath = this.getAssetPath(path);
const depAssetPath = this.getAssetPath(dependencyPath);

if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) {
const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? [];
const array = [...dependencies, depAssetPath];
this.context.meta.files[assetPath].dependencies[type] = [...new Set(array)];
}
}

unmarkDep(path: string, dependencyPath: string, type?: string): void {
type = type ?? 'include';

const assetPath = this.getAssetPath(path);
const depAssetPath = this.getAssetPath(dependencyPath);

if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) {
const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? [];
this.context.meta.files[assetPath].dependencies[type] = dependencies.filter(
(file) => file !== depAssetPath,
);
}
}

resetDeps(path: string): void {
const assetPath = this.getAssetPath(path);

if (assetPath && this.context?.meta?.files?.[assetPath]) {
this.context.meta.files[assetPath].dependencies = {};
}
}
}
Loading

0 comments on commit 618c3af

Please sign in to comment.