feat: new build pipeline

package.json

@@ -68,7 +68,7 @@
     "@diplodoc/mermaid-extension": "^1.3.1",
     "@diplodoc/openapi-extension": "^2.4.1",
     "@diplodoc/prettier-config": "^2.0.0",
-    "@diplodoc/transform": "^4.32.3",
+    "@diplodoc/transform": "0.0.0-rc-add-file-meta-202410220915",
     "@diplodoc/tsconfig": "^1.0.2",
     "@octokit/core": "4.2.4",
     "@types/async": "^3.2.15",

src/cmd/build/index.ts

@@ -1,26 +1,29 @@
-import glob from 'glob';
 import {Arguments, Argv} from 'yargs';
 import {join, resolve} from 'path';
 import shell from 'shelljs';
 
 import OpenapiIncluder from '@diplodoc/openapi-extension/includer';
 
-import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants';
-import {argvValidator} from '../../validator';
-import {ArgvService, Includers, SearchService} from '../../services';
+import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '~/constants';
+import {argvValidator} from '~/validator';
+import {ArgvService, Includers, SearchService, TocService} from '~/services';
 import {
-    initLinterWorkers,
+    finishProcessPages,
+    getLintFn,
+    getProcessPageFn,
     processAssets,
     processChangelogs,
     processExcludedFiles,
-    processLinter,
     processLogs,
-    processPages,
     processServiceFiles,
-} from '../../steps';
-import {prepareMapFile} from '../../steps/processMapFile';
-import {copyFiles, logger} from '../../utils';
-import {upload as publishFilesToS3} from '../../commands/publish/upload';
+} from '~/steps';
+import {prepareMapFile} from '~/steps/processMapFile';
+import {copyFiles, logger} from '~/utils';
+import {upload as publishFilesToS3} from '~/commands/publish/upload';
+import {RevisionContext, makeRevisionContext, setRevisionContext} from '~/context/context';
+import {FsContextCli} from '~/context/fs';
+import {DependencyContextCli} from '~/context/dependency';
+import {FileQueueProcessor} from '~/context/processor';
 
 export const build = {
     command: ['build', '$0'],
@@ -37,12 +40,30 @@ function builder<T>(argv: Argv<T>) {
             type: 'string',
             group: 'Build options:',
         })
+        .option('debug', {
+            alias: 'd',
+            describe: 'Debug mode for development',
+            type: 'string',
+            group: 'Build options:',
+        })
         .option('output', {
             alias: 'o',
             describe: 'Path to output folder',
             type: 'string',
             group: 'Build options:',
         })
+        .option('plugins', {
+            alias: 'p',
+            describe: 'Path to plugins js file',
+            type: 'string',
+            group: 'Build options:',
+        })
+        .option('cached', {
+            default: false,
+            describe: 'Use cache from revision meta file',
+            type: 'boolean',
+            group: 'Build options:',
+        })
         .option('varsPreset', {
             default: 'default',
             describe: 'Target vars preset of documentation <external|internal>',
@@ -176,6 +197,8 @@ function builder<T>(argv: Argv<T>) {
 }
 
 async function handler(args: Arguments<any>) {
+    let hasError = false;
+
     const userOutputFolder = resolve(args.output);
     const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
     const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
@@ -203,37 +226,61 @@ async function handler(args: Arguments<any>) {
             addMapFile,
         } = ArgvService.getConfig();
 
-        preparingTemporaryFolders(userOutputFolder);
+        const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);
+
+        const context = await makeRevisionContext(
+            userOutputFolder,
+            tmpInputFolder,
+            tmpOutputFolder,
+            outputBundlePath,
+        );
+
+        const fs = new FsContextCli(context);
+        const deps = new DependencyContextCli(context);
+        const pageProcessor = new FileQueueProcessor(context, deps);
+        const pageLintProcessor = new FileQueueProcessor(context, deps);
 
-        await processServiceFiles();
-        processExcludedFiles();
+        await preparingTemporaryFolders(context);
+        await processServiceFiles(context, fs);
+        await processExcludedFiles();
 
         if (addMapFile) {
             prepareMapFile();
         }
 
-        const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);
+        const navigationPaths = TocService.getNavigationPaths();
+
+        pageProcessor.setNavigationPaths(navigationPaths);
+        pageLintProcessor.setNavigationPaths(navigationPaths);
 
         if (!lintDisabled) {
-            /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */
-            await initLinterWorkers();
+            const filesToProcess = pageLintProcessor.getFilesToProcess();
+
+            const processLintPageFn = await getLintFn(context);
+
+            await pageLintProcessor.processQueue(processLintPageFn, filesToProcess);
         }
 
-        const processes = [
-            !lintDisabled && processLinter(),
-            !buildDisabled && processPages(outputBundlePath),
-        ].filter(Boolean) as Promise<void>[];
+        if (!buildDisabled) {
+            const filesToProcess = pageProcessor.getFilesToProcess();
+
+            const processPageFn = await getProcessPageFn(fs, deps, context, outputBundlePath);
+
+            await pageProcessor.processQueue(processPageFn, filesToProcess);
 
-        await Promise.all(processes);
+            await finishProcessPages(fs);
+        }
 
         if (!buildDisabled) {
             // process additional files
-            processAssets({
+            await processAssets({
                 args,
                 outputFormat,
                 outputBundlePath,
                 tmpOutputFolder,
                 userOutputFolder,
+                context,
+                fs,
             });
 
             await processChangelogs();
@@ -271,34 +318,37 @@ async function handler(args: Arguments<any>) {
                 });
             }
         }
+
+        await setRevisionContext(context);
     } catch (err) {
+        if (args.debug) {
+            console.error(err);
+        }
+
         logger.error('', err.message);
+
+        hasError = true;
     } finally {
         processLogs(tmpInputFolder);
 
         shell.rm('-rf', tmpInputFolder, tmpOutputFolder);
     }
+
+    if (hasError) {
+        process.exit(1);
+    }
 }
 
-function preparingTemporaryFolders(userOutputFolder: string) {
+async function preparingTemporaryFolders(revisionContext: RevisionContext) {
     const args = ArgvService.getConfig();
 
-    shell.mkdir('-p', userOutputFolder);
+    shell.mkdir('-p', revisionContext.userOutputFolder);
 
     // Create temporary input/output folders
     shell.rm('-rf', args.input, args.output);
     shell.mkdir(args.input, args.output);
 
-    copyFiles(
-        args.rootInput,
-        args.input,
-        glob.sync('**', {
-            cwd: args.rootInput,
-            nodir: true,
-            follow: true,
-            ignore: ['node_modules/**', '*/node_modules/**'],
-        }),
-    );
+    await copyFiles(args.rootInput, args.input, revisionContext.files, revisionContext.meta);
 
     shell.chmod('-R', 'u+w', args.input);
 }

src/commands/publish/upload.ts

@@ -1,14 +1,15 @@
 import type {Run} from './run';
 import {join} from 'path';
 import {asyncify, mapLimit} from 'async';
-import walkSync from 'walk-sync';
 import mime from 'mime-types';
 import {LogLevel} from '~/logger';
+import {walk} from '~/utils';
 
 export async function upload(run: Run): Promise<void> {
     const {input, endpoint, bucket, prefix, hidden = []} = run.config;
     const logUpload = run.logger.topic(LogLevel.INFO, 'UPLOAD');
-    const filesToPublish: string[] = walkSync(run.root, {
+    const filesToPublish: string[] = walk({
+        folder: run.root,
         directories: false,
         includeBasePath: false,
         ignore: hidden,

src/context/context.ts

@@ -0,0 +1,59 @@
+import {
+    RevisionContext as RevisionContextTransfrom,
+    RevisionMeta,
+} from '@diplodoc/transform/lib/typings';
+import glob from 'glob';
+import {ArgvService} from '~/services';
+import {getMetaFile, makeMetaFile, updateChangedMetaFile, updateMetaFile} from '~/utils/meta';
+
+export interface RevisionContext extends RevisionContextTransfrom {
+    userInputFolder: string;
+    userOutputFolder: string;
+    tmpInputFolder: string;
+    tmpOutputFolder: string;
+    outputBundlePath: string;
+}
+
+export async function makeRevisionContext(
+    userOutputFolder: string,
+    tmpInputFolder: string,
+    tmpOutputFolder: string,
+    outputBundlePath: string,
+): Promise<RevisionContext> {
+    const args = ArgvService.getConfig();
+
+    const files = glob.sync('**', {
+        cwd: args.rootInput,
+        nodir: true,
+        follow: true,
+        ignore: ['node_modules/**', '*/node_modules/**'],
+    });
+
+    const meta = normalizeMeta(await getMetaFile(userOutputFolder));
+
+    await updateMetaFile(args.cached, args.rootInput, meta.files, files);
+
+    await updateChangedMetaFile(args.cached, args.rootInput, meta.files);
+
+    return {
+        userInputFolder: args.rootInput,
+        userOutputFolder,
+        tmpInputFolder,
+        tmpOutputFolder,
+        outputBundlePath,
+        files,
+        meta,
+    };
+}
+
+function normalizeMeta(meta?: RevisionMeta | undefined | null) {
+    const metaSafe: RevisionMeta = meta ?? {
+        files: {},
+    };
+    metaSafe.files = metaSafe.files ?? {};
+    return metaSafe;
+}
+
+export async function setRevisionContext(context: RevisionContext): Promise<void> {
+    await makeMetaFile(context.userOutputFolder, context.files, context.meta);
+}

src/context/dependency.ts

@@ -0,0 +1,62 @@
+import {resolve} from 'path';
+import {DependencyContext} from '@diplodoc/transform/lib/typings';
+import {RevisionContext} from './context';
+
+export class DependencyContextCli implements DependencyContext {
+    private context: RevisionContext;
+
+    constructor(context: RevisionContext) {
+        this.context = context;
+    }
+
+    getAssetPath(path: string) {
+        const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/');
+        if (isFromTmpInputFolder) {
+            const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', '');
+            return assetPath;
+        }
+
+        const isFromInputFolder = path.startsWith(resolve(this.context.userInputFolder) + '/');
+        if (isFromInputFolder) {
+            const assetPath = path.replace(resolve(this.context.userInputFolder) + '/', '');
+            return assetPath;
+        }
+
+        return path;
+    }
+
+    markDep(path: string, dependencyPath: string, type?: string): void {
+        type = type ?? 'include';
+
+        const assetPath = this.getAssetPath(path);
+        const depAssetPath = this.getAssetPath(dependencyPath);
+
+        if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) {
+            const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? [];
+            const array = [...dependencies, depAssetPath];
+            this.context.meta.files[assetPath].dependencies[type] = [...new Set(array)];
+        }
+    }
+
+    unmarkDep(path: string, dependencyPath: string, type?: string): void {
+        type = type ?? 'include';
+
+        const assetPath = this.getAssetPath(path);
+        const depAssetPath = this.getAssetPath(dependencyPath);
+
+        if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) {
+            const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? [];
+            this.context.meta.files[assetPath].dependencies[type] = dependencies.filter(
+                (file) => file !== depAssetPath,
+            );
+        }
+    }
+
+    resetDeps(path: string): void {
+        const assetPath = this.getAssetPath(path);
+
+        if (assetPath && this.context?.meta?.files?.[assetPath]) {
+            this.context.meta.files[assetPath].dependencies = {};
+        }
+    }
+}