feat: new build pipeline

src/cmd/build/index.ts

@@ -1,26 +1,31 @@
-import glob from 'glob';
 import {Arguments, Argv} from 'yargs';
 import {join, resolve} from 'path';
 import shell from 'shelljs';
+import glob from 'glob';
 
 import OpenapiIncluder from '@diplodoc/openapi-extension/includer';
 
-import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants';
-import {argvValidator} from '../../validator';
-import {ArgvService, Includers, SearchService} from '../../services';
+import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '~/constants';
+import {argvValidator} from '~/validator';
+import {Includer, YfmArgv} from '~/models';
+import {ArgvService, Includers, SearchService, TocService} from '~/services';
 import {
-    initLinterWorkers,
+    finishProcessPages,
+    getLintFn,
+    getProcessPageFn,
     processAssets,
     processChangelogs,
     processExcludedFiles,
-    processLinter,
     processLogs,
-    processPages,
     processServiceFiles,
-} from '../../steps';
-import {prepareMapFile} from '../../steps/processMapFile';
-import {copyFiles, logger} from '../../utils';
-import {upload as publishFilesToS3} from '../../commands/publish/upload';
+} from '~/steps';
+import {prepareMapFile} from '~/steps/processMapFile';
+import {copyFiles, logger} from '~/utils';
+import {upload as publishFilesToS3} from '~/commands/publish/upload';
+import {RevisionContext, makeRevisionContext, setRevisionContext} from '~/context/context';
+import {FsContextCli} from '~/context/fs';
+import {DependencyContextCli} from '~/context/dependency';
+import {FileQueueProcessor} from '~/context/processor';
 
 export const build = {
     command: ['build', '$0'],
@@ -43,6 +48,24 @@ function builder<T>(argv: Argv<T>) {
             type: 'string',
             group: 'Build options:',
         })
+        .option('plugins', {
+            alias: 'p',
+            describe: 'Path to plugins js file',
+            type: 'string',
+            group: 'Build options:',
+        })
+        .option('cached', {
+            default: false,
+            describe: 'Use cache from revision meta file',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('clean', {
+            default: false,
+            describe: 'Remove output folder before build',
+            type: 'boolean',
+            group: 'Build options:',
+        })
         .option('varsPreset', {
             default: 'default',
             describe: 'Target vars preset of documentation <external|internal>',
@@ -175,24 +198,30 @@ function builder<T>(argv: Argv<T>) {
         );
 }
 
-async function handler(args: Arguments<any>) {
-    const userOutputFolder = resolve(args.output);
-    const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
-    const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
-
+async function handler(args: Arguments<YfmArgv>) {
     if (typeof VERSION !== 'undefined') {
         console.log(`Using v${VERSION} version`);
     }
 
+    shell.config.silent = true;
+
+    let hasError = false;
+
+    const userInputFolder = resolve(args.input);
+    const userOutputFolder = resolve(args.output);
+    const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
+    const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
+
     try {
+        // Init singletone services
         ArgvService.init({
             ...args,
-            rootInput: args.input,
+            rootInput: userInputFolder,
             input: tmpInputFolder,
             output: tmpOutputFolder,
         });
         SearchService.init();
-        Includers.init([OpenapiIncluder as any]);
+        Includers.init([OpenapiIncluder as Includer]);
 
         const {
             output: outputFolderPath,
@@ -203,41 +232,77 @@ async function handler(args: Arguments<any>) {
             addMapFile,
         } = ArgvService.getConfig();
 
-        preparingTemporaryFolders(userOutputFolder);
+        const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);
+
+        if (args.clean) {
+            await clearTemporaryFolders(userOutputFolder);
+        }
+
+        // Create build context that stores the information about the current build
+        const context = await makeRevisionContext(
+            args.cached,
+            userInputFolder,
+            userOutputFolder,
+            tmpInputFolder,
+            tmpOutputFolder,
+            outputBundlePath,
+        );
+
+        const fs = new FsContextCli(context);
+        const deps = new DependencyContextCli(context);
 
-        await processServiceFiles();
-        processExcludedFiles();
+        // Creating temp .input & .output folder
+        await preparingTemporaryFolders(context);
 
+        // Read and prepare Preset & Toc data
+        await processServiceFiles(context, fs);
+
+        // Removes all content files that unspecified in toc files or ignored.
+        await processExcludedFiles();
+
+        // Write files.json
         if (addMapFile) {
-            prepareMapFile();
+            await prepareMapFile();
         }
 
-        const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);
+        // Collect navigation paths as entry files
+        const navigationPaths = TocService.getNavigationPaths();
 
+        // 1. Linting
         if (!lintDisabled) {
-            /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */
-            await initLinterWorkers();
+            const pageLintProcessor = new FileQueueProcessor(context, deps);
+            pageLintProcessor.setNavigationPaths(navigationPaths);
+
+            const processLintPageFn = await getLintFn(context);
+            await pageLintProcessor.processQueue(processLintPageFn);
         }
 
-        const processes = [
-            !lintDisabled && processLinter(),
-            !buildDisabled && processPages(outputBundlePath),
-        ].filter(Boolean) as Promise<void>[];
+        // 2. Building
+        if (!buildDisabled) {
+            const pageProcessor = new FileQueueProcessor(context, deps);
+            pageProcessor.setNavigationPaths(navigationPaths);
+
+            const processPageFn = await getProcessPageFn(fs, deps, context, outputBundlePath);
+            await pageProcessor.processQueue(processPageFn);
 
-        await Promise.all(processes);
+            // Save single pages & redirects
+            await finishProcessPages(fs);
 
-        if (!buildDisabled) {
-            // process additional files
-            processAssets({
+            // Process asset files
+            await processAssets({
                 args,
                 outputFormat,
                 outputBundlePath,
                 tmpOutputFolder,
                 userOutputFolder,
+                context,
+                fs,
             });
 
+            // Process changelogs
             await processChangelogs();
 
+            // Finish search service processing
             await SearchService.release();
 
             // Copy all generated files to user' output folder
@@ -246,6 +311,7 @@ async function handler(args: Arguments<any>) {
                 shell.cp('-r', join(tmpOutputFolder, '.*'), userOutputFolder);
             }
 
+            // Upload the files to S3
             if (publish) {
                 const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? '';
                 const {
@@ -269,35 +335,41 @@ async function handler(args: Arguments<any>) {
                     secretAccessKey,
                 });
             }
+
+            // Save .revision.meta.json file for the future processing
+            await setRevisionContext(context);
         }
     } catch (err) {
         logger.error('', err.message);
+
+        hasError = true;
     } finally {
+        // Print logs
         processLogs(tmpInputFolder);
 
         shell.rm('-rf', tmpInputFolder, tmpOutputFolder);
     }
-}
 
-function preparingTemporaryFolders(userOutputFolder: string) {
-    const args = ArgvService.getConfig();
+    // If build has some errors, then exit with error code 1
+    if (hasError) {
+        process.exit(1);
+    }
+}
 
-    shell.mkdir('-p', userOutputFolder);
+// Creating temp .input & .output folder
+async function preparingTemporaryFolders(context: RevisionContext) {
+    shell.mkdir('-p', context.userOutputFolder);
 
     // Create temporary input/output folders
-    shell.rm('-rf', args.input, args.output);
-    shell.mkdir(args.input, args.output);
-
-    copyFiles(
-        args.rootInput,
-        args.input,
-        glob.sync('**', {
-            cwd: args.rootInput,
-            nodir: true,
-            follow: true,
-            ignore: ['node_modules/**', '*/node_modules/**'],
-        }),
-    );
-
-    shell.chmod('-R', 'u+w', args.input);
+    shell.rm('-rf', context.tmpInputFolder, context.tmpOutputFolder);
+    shell.mkdir(context.tmpInputFolder, context.tmpOutputFolder);
+
+    await copyFiles(context.userInputFolder, context.tmpInputFolder, context.files, context.meta);
+
+    shell.chmod('-R', 'u+w', context.tmpInputFolder);
+}
+
+// Clear output folder folders
+async function clearTemporaryFolders(userOutputFolder: string) {
+    shell.rm('-rf', userOutputFolder);
 }

src/context/context.ts

@@ -0,0 +1,58 @@
+import {
+    RevisionContext as RevisionContextTransfrom,
+    RevisionMeta,
+} from '@diplodoc/transform/lib/typings';
+import glob from 'glob';
+import {getMetaFile, makeMetaFile, updateChangedMetaFile, updateMetaFile} from '~/utils/meta';
+
+export interface RevisionContext extends RevisionContextTransfrom {
+    userInputFolder: string;
+    userOutputFolder: string;
+    tmpInputFolder: string;
+    tmpOutputFolder: string;
+    outputBundlePath: string;
+}
+
+export async function makeRevisionContext(
+    cached: boolean,
+    userInputFolder: string,
+    userOutputFolder: string,
+    tmpInputFolder: string,
+    tmpOutputFolder: string,
+    outputBundlePath: string,
+): Promise<RevisionContext> {
+    const files = glob.sync('**', {
+        cwd: userInputFolder,
+        nodir: true,
+        follow: true,
+        ignore: ['node_modules/**', '*/node_modules/**'],
+    });
+
+    const meta = normalizeMeta(await getMetaFile(userOutputFolder));
+
+    await updateMetaFile(cached, userInputFolder, meta.files, files);
+
+    await updateChangedMetaFile(cached, userInputFolder, meta.files);
+
+    return {
+        userInputFolder,
+        userOutputFolder,
+        tmpInputFolder,
+        tmpOutputFolder,
+        outputBundlePath,
+        files,
+        meta,
+    };
+}
+
+function normalizeMeta(meta?: RevisionMeta | undefined | null) {
+    const metaSafe: RevisionMeta = meta ?? {
+        files: {},
+    };
+    metaSafe.files = metaSafe.files ?? {};
+    return metaSafe;
+}
+
+export async function setRevisionContext(context: RevisionContext): Promise<void> {
+    await makeMetaFile(context.userOutputFolder, context.files, context.meta);
+}

src/context/dependency.ts

@@ -0,0 +1,62 @@
+import {resolve} from 'path';
+import {DependencyContext} from '@diplodoc/transform/lib/typings';
+import {RevisionContext} from './context';
+
+export class DependencyContextCli implements DependencyContext {
+    private context: RevisionContext;
+
+    constructor(context: RevisionContext) {
+        this.context = context;
+    }
+
+    getAssetPath(path: string) {
+        const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/');
+        if (isFromTmpInputFolder) {
+            const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', '');
+            return assetPath;
+        }
+
+        const isFromInputFolder = path.startsWith(resolve(this.context.userInputFolder) + '/');
+        if (isFromInputFolder) {
+            const assetPath = path.replace(resolve(this.context.userInputFolder) + '/', '');
+            return assetPath;
+        }
+
+        return path;
+    }
+
+    markDep(path: string, dependencyPath: string, type?: string): void {
+        type = type ?? 'include';
+
+        const assetPath = this.getAssetPath(path);
+        const depAssetPath = this.getAssetPath(dependencyPath);
+
+        if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) {
+            const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? [];
+            const array = [...dependencies, depAssetPath];
+            this.context.meta.files[assetPath].dependencies[type] = [...new Set(array)];
+        }
+    }
+
+    unmarkDep(path: string, dependencyPath: string, type?: string): void {
+        type = type ?? 'include';
+
+        const assetPath = this.getAssetPath(path);
+        const depAssetPath = this.getAssetPath(dependencyPath);
+
+        if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) {
+            const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? [];
+            this.context.meta.files[assetPath].dependencies[type] = dependencies.filter(
+                (file) => file !== depAssetPath,
+            );
+        }
+    }
+
+    resetDeps(path: string): void {
+        const assetPath = this.getAssetPath(path);
+
+        if (assetPath && this.context?.meta?.files?.[assetPath]) {
+            this.context.meta.files[assetPath].dependencies = {};
+        }
+    }
+}