From 4afddce89d5f0a302ba694331159dc1a08d5b393 Mon Sep 17 00:00:00 2001 From: Maxim Karpov Date: Mon, 28 Oct 2024 17:25:50 +0300 Subject: [PATCH] feat: new build pipeline --- src/cmd/build/index.ts | 176 ++++++++++++----- src/context/context.ts | 58 ++++++ src/context/dependency.ts | 62 ++++++ src/context/fs.ts | 107 ++++++++++ src/context/processor.ts | 107 ++++++++++ src/models.ts | 22 ++- src/resolvers/lintPage.ts | 52 +++-- src/resolvers/md2html.ts | 70 ++++--- src/resolvers/md2md.ts | 62 +++--- src/services/leading.ts | 8 +- src/services/plugins.ts | 23 ++- src/services/tocs.ts | 41 ++-- src/steps/processAssets.ts | 85 ++++---- src/steps/processExcludedFiles.ts | 4 +- src/steps/processLinter.ts | 76 ++++++-- src/steps/processMapFile.ts | 6 +- src/steps/processPages.ts | 182 ++++++++++++++---- src/steps/processServiceFiles.ts | 37 ++-- src/utils/common.ts | 11 +- src/utils/file.ts | 77 ++++++-- src/utils/logger.ts | 7 +- src/utils/meta.ts | 119 ++++++++++++ src/utils/queue.ts | 102 ++++++++++ src/workers/linter/index.ts | 20 +- .../__snapshots__/include-toc.test.ts.snap | 4 + .../load-custom-resources.spec.ts.snap | 3 + tests/e2e/__snapshots__/metadata.spec.ts.snap | 4 +- tests/e2e/__snapshots__/rtl.spec.ts.snap | 2 + tests/utils.ts | 5 +- 29 files changed, 1231 insertions(+), 301 deletions(-) create mode 100644 src/context/context.ts create mode 100644 src/context/dependency.ts create mode 100644 src/context/fs.ts create mode 100644 src/context/processor.ts create mode 100644 src/utils/meta.ts create mode 100644 src/utils/queue.ts diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts index 2c9f052e..fe782796 100644 --- a/src/cmd/build/index.ts +++ b/src/cmd/build/index.ts @@ -1,26 +1,31 @@ -import glob from 'glob'; import {Arguments, Argv} from 'yargs'; import {join, resolve} from 'path'; import shell from 'shelljs'; +import glob from 'glob'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; -import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants'; -import {argvValidator} from '../../validator'; -import {ArgvService, Includers, SearchService} from '../../services'; +import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '~/constants'; +import {argvValidator} from '~/validator'; +import {Includer, YfmArgv} from '~/models'; +import {ArgvService, Includers, SearchService, TocService} from '~/services'; import { - initLinterWorkers, + finishProcessPages, + getLintFn, + getProcessPageFn, processAssets, processChangelogs, processExcludedFiles, - processLinter, processLogs, - processPages, processServiceFiles, -} from '../../steps'; -import {prepareMapFile} from '../../steps/processMapFile'; -import {copyFiles, logger} from '../../utils'; -import {upload as publishFilesToS3} from '../../commands/publish/upload'; +} from '~/steps'; +import {prepareMapFile} from '~/steps/processMapFile'; +import {copyFiles, logger} from '~/utils'; +import {upload as publishFilesToS3} from '~/commands/publish/upload'; +import {RevisionContext, makeRevisionContext, setRevisionContext} from '~/context/context'; +import {FsContextCli} from '~/context/fs'; +import {DependencyContextCli} from '~/context/dependency'; +import {FileQueueProcessor} from '~/context/processor'; export const build = { command: ['build', '$0'], @@ -43,6 +48,24 @@ function builder(argv: Argv) { type: 'string', group: 'Build options:', }) + .option('plugins', { + alias: 'p', + describe: 'Path to plugins js file', + type: 'string', + group: 'Build options:', + }) + .option('cached', { + default: false, + describe: 'Use cache from revision meta file', + type: 'boolean', + group: 'Build options:', + }) + .option('clean', { + default: false, + describe: 'Remove output folder before build', + type: 'boolean', + group: 'Build options:', + }) .option('varsPreset', { default: 'default', describe: 'Target vars preset of documentation ', @@ -175,24 +198,30 @@ function builder(argv: Argv) { ); } -async function handler(args: Arguments) { - const userOutputFolder = resolve(args.output); - const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER); - const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER); - +async function handler(args: Arguments) { if (typeof VERSION !== 'undefined') { console.log(`Using v${VERSION} version`); } + shell.config.silent = true; + + let hasError = false; + + const userInputFolder = resolve(args.input); + const userOutputFolder = resolve(args.output); + const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER); + const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER); + try { + // Init singletone services ArgvService.init({ ...args, - rootInput: args.input, + rootInput: userInputFolder, input: tmpInputFolder, output: tmpOutputFolder, }); SearchService.init(); - Includers.init([OpenapiIncluder as any]); + Includers.init([OpenapiIncluder as Includer]); const { output: outputFolderPath, @@ -203,41 +232,77 @@ async function handler(args: Arguments) { addMapFile, } = ArgvService.getConfig(); - preparingTemporaryFolders(userOutputFolder); + const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); + + if (args.clean) { + await clearTemporaryFolders(userOutputFolder); + } + + // Create build context that stores the information about the current build + const context = await makeRevisionContext( + args.cached, + userInputFolder, + userOutputFolder, + tmpInputFolder, + tmpOutputFolder, + outputBundlePath, + ); + + const fs = new FsContextCli(context); + const deps = new DependencyContextCli(context); - await processServiceFiles(); - processExcludedFiles(); + // Creating temp .input & .output folder + await preparingTemporaryFolders(context); + // Read and prepare Preset & Toc data + await processServiceFiles(context, fs); + + // Removes all content files that unspecified in toc files or ignored. + await processExcludedFiles(); + + // Write files.json if (addMapFile) { - prepareMapFile(); + await prepareMapFile(); } - const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); + // Collect navigation paths as entry files + const navigationPaths = TocService.getNavigationPaths(); + // 1. Linting if (!lintDisabled) { - /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */ - await initLinterWorkers(); + const pageLintProcessor = new FileQueueProcessor(context, deps); + pageLintProcessor.setNavigationPaths(navigationPaths); + + const processLintPageFn = await getLintFn(context); + await pageLintProcessor.processQueue(processLintPageFn); } - const processes = [ - !lintDisabled && processLinter(), - !buildDisabled && processPages(outputBundlePath), - ].filter(Boolean) as Promise[]; + // 2. Building + if (!buildDisabled) { + const pageProcessor = new FileQueueProcessor(context, deps); + pageProcessor.setNavigationPaths(navigationPaths); + + const processPageFn = await getProcessPageFn(fs, deps, context, outputBundlePath); + await pageProcessor.processQueue(processPageFn); - await Promise.all(processes); + // Save single pages & redirects + await finishProcessPages(fs); - if (!buildDisabled) { - // process additional files - processAssets({ + // Process asset files + await processAssets({ args, outputFormat, outputBundlePath, tmpOutputFolder, userOutputFolder, + context, + fs, }); + // Process changelogs await processChangelogs(); + // Finish search service processing await SearchService.release(); // Copy all generated files to user' output folder @@ -246,6 +311,7 @@ async function handler(args: Arguments) { shell.cp('-r', join(tmpOutputFolder, '.*'), userOutputFolder); } + // Upload the files to S3 if (publish) { const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? ''; const { @@ -269,35 +335,41 @@ async function handler(args: Arguments) { secretAccessKey, }); } + + // Save .revision.meta.json file for the future processing + await setRevisionContext(context); } } catch (err) { logger.error('', err.message); + + hasError = true; } finally { + // Print logs processLogs(tmpInputFolder); shell.rm('-rf', tmpInputFolder, tmpOutputFolder); } -} -function preparingTemporaryFolders(userOutputFolder: string) { - const args = ArgvService.getConfig(); + // If build has some errors, then exit with error code 1 + if (hasError) { + process.exit(1); + } +} - shell.mkdir('-p', userOutputFolder); +// Creating temp .input & .output folder +async function preparingTemporaryFolders(context: RevisionContext) { + shell.mkdir('-p', context.userOutputFolder); // Create temporary input/output folders - shell.rm('-rf', args.input, args.output); - shell.mkdir(args.input, args.output); - - copyFiles( - args.rootInput, - args.input, - glob.sync('**', { - cwd: args.rootInput, - nodir: true, - follow: true, - ignore: ['node_modules/**', '*/node_modules/**'], - }), - ); - - shell.chmod('-R', 'u+w', args.input); + shell.rm('-rf', context.tmpInputFolder, context.tmpOutputFolder); + shell.mkdir(context.tmpInputFolder, context.tmpOutputFolder); + + await copyFiles(context.userInputFolder, context.tmpInputFolder, context.files, context.meta); + + shell.chmod('-R', 'u+w', context.tmpInputFolder); +} + +// Clear output folder folders +async function clearTemporaryFolders(userOutputFolder: string) { + shell.rm('-rf', userOutputFolder); } diff --git a/src/context/context.ts b/src/context/context.ts new file mode 100644 index 00000000..949ea336 --- /dev/null +++ b/src/context/context.ts @@ -0,0 +1,58 @@ +import { + RevisionContext as RevisionContextTransfrom, + RevisionMeta, +} from '@diplodoc/transform/lib/typings'; +import glob from 'glob'; +import {getMetaFile, makeMetaFile, updateChangedMetaFile, updateMetaFile} from '~/utils/meta'; + +export interface RevisionContext extends RevisionContextTransfrom { + userInputFolder: string; + userOutputFolder: string; + tmpInputFolder: string; + tmpOutputFolder: string; + outputBundlePath: string; +} + +export async function makeRevisionContext( + cached: boolean, + userInputFolder: string, + userOutputFolder: string, + tmpInputFolder: string, + tmpOutputFolder: string, + outputBundlePath: string, +): Promise { + const files = glob.sync('**', { + cwd: userInputFolder, + nodir: true, + follow: true, + ignore: ['node_modules/**', '*/node_modules/**'], + }); + + const meta = normalizeMeta(await getMetaFile(userOutputFolder)); + + await updateMetaFile(cached, userInputFolder, meta.files, files); + + await updateChangedMetaFile(cached, userInputFolder, meta.files); + + return { + userInputFolder, + userOutputFolder, + tmpInputFolder, + tmpOutputFolder, + outputBundlePath, + files, + meta, + }; +} + +function normalizeMeta(meta?: RevisionMeta | undefined | null) { + const metaSafe: RevisionMeta = meta ?? { + files: {}, + }; + metaSafe.files = metaSafe.files ?? {}; + return metaSafe; +} + +export async function setRevisionContext(context: RevisionContext): Promise { + await makeMetaFile(context.userOutputFolder, context.files, context.meta); +} diff --git a/src/context/dependency.ts b/src/context/dependency.ts new file mode 100644 index 00000000..d28d0a23 --- /dev/null +++ b/src/context/dependency.ts @@ -0,0 +1,62 @@ +import {resolve} from 'path'; +import {DependencyContext} from '@diplodoc/transform/lib/typings'; +import {RevisionContext} from './context'; + +export class DependencyContextCli implements DependencyContext { + private context: RevisionContext; + + constructor(context: RevisionContext) { + this.context = context; + } + + getAssetPath(path: string) { + const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/'); + if (isFromTmpInputFolder) { + const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', ''); + return assetPath; + } + + const isFromInputFolder = path.startsWith(resolve(this.context.userInputFolder) + '/'); + if (isFromInputFolder) { + const assetPath = path.replace(resolve(this.context.userInputFolder) + '/', ''); + return assetPath; + } + + return path; + } + + markDep(path: string, dependencyPath: string, type?: string): void { + type = type ?? 'include'; + + const assetPath = this.getAssetPath(path); + const depAssetPath = this.getAssetPath(dependencyPath); + + if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) { + const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? []; + const array = [...dependencies, depAssetPath]; + this.context.meta.files[assetPath].dependencies[type] = [...new Set(array)]; + } + } + + unmarkDep(path: string, dependencyPath: string, type?: string): void { + type = type ?? 'include'; + + const assetPath = this.getAssetPath(path); + const depAssetPath = this.getAssetPath(dependencyPath); + + if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) { + const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? []; + this.context.meta.files[assetPath].dependencies[type] = dependencies.filter( + (file) => file !== depAssetPath, + ); + } + } + + resetDeps(path: string): void { + const assetPath = this.getAssetPath(path); + + if (assetPath && this.context?.meta?.files?.[assetPath]) { + this.context.meta.files[assetPath].dependencies = {}; + } + } +} diff --git a/src/context/fs.ts b/src/context/fs.ts new file mode 100644 index 00000000..24065e3c --- /dev/null +++ b/src/context/fs.ts @@ -0,0 +1,107 @@ +import {readFileSync, statSync, writeFileSync} from 'fs'; +import {readFile, stat, writeFile} from 'fs/promises'; +import {resolve} from 'path'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {RevisionContext} from './context'; + +export function isFileExists(file: string) { + try { + const stats = statSync(file); + + return stats.isFile(); + } catch (e) { + return false; + } +} + +export async function isFileExistsAsync(file: string) { + try { + const stats = await stat(file); + + return stats.isFile(); + } catch (e) { + return false; + } +} + +export class FsContextCli implements FsContext { + private context: RevisionContext; + + constructor(context: RevisionContext) { + this.context = context; + } + + getPaths(path: string) { + const arr = [path]; + + const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/'); + if (isFromTmpInputFolder) { + const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', ''); + const originPath = resolve(this.context.userInputFolder, assetPath); + + arr.unshift(originPath); + } + + return arr; + } + + exist(path: string): boolean { + const paths = this.getPaths(path); + + for (const path of paths) { + if (isFileExists(path)) { + return true; + } + } + + return false; + } + + read(path: string): string { + const paths = this.getPaths(path); + + for (const path of paths) { + if (isFileExists(path)) { + return readFileSync(path, 'utf8'); + } + } + + throw Error(`File has not been found at: ${path}`); + } + + write(path: string, content: string): void { + writeFileSync(path, content, { + encoding: 'utf8', + }); + } + + async existAsync(path: string): Promise { + const paths = this.getPaths(path); + + for (const path of paths) { + if (await isFileExistsAsync(path)) { + return true; + } + } + + return false; + } + + async readAsync(path: string): Promise { + const paths = this.getPaths(path); + + for (const path of paths) { + if (await isFileExistsAsync(path)) { + return await readFile(path, 'utf8'); + } + } + + throw Error(`File has not been found at: ${path}`); + } + + async writeAsync(path: string, content: string): Promise { + await writeFile(path, content, { + encoding: 'utf8', + }); + } +} diff --git a/src/context/processor.ts b/src/context/processor.ts new file mode 100644 index 00000000..7f9f74df --- /dev/null +++ b/src/context/processor.ts @@ -0,0 +1,107 @@ +import {DependencyContext} from '@diplodoc/transform/lib/typings'; +import {logger} from '~/utils/logger'; +import {Queue} from '~/utils/queue'; +import {RevisionContext} from './context'; + +const PAGES_ACTIVE_QUEUE_LENGTH = 200; + +type FileQueueProcessorFn = (path: string) => Promise | void; + +// Processor allows to process files in parallel via PAGES_ACTIVE_QUEUE_LENGTH limit +// - it uses Queue engine as processor. +// - it has white queue to collect dependencies and puts them in the stack to avoid dead lock by the queue limit +export class FileQueueProcessor { + private context: RevisionContext; + private deps: DependencyContext; + + private processed = new Set(); + private navigationPaths = new Set(); + + constructor(context: RevisionContext, deps: DependencyContext) { + this.context = context; + this.deps = deps; + } + + // Set entry files + setNavigationPaths(navigationPaths: string[]) { + this.navigationPaths = new Set(navigationPaths); + } + + // Without 'cached' option all the files are changed + isChanged(path: string) { + return this.context.meta?.files?.[path]?.changed !== false; + } + + // Processable file is the entry file + isProcessable(pattern: string) { + return this.navigationPaths.has(pattern); + } + + // Main process function + async processQueue(fn: FileQueueProcessorFn) { + const files = this.getFilesToProcess(); + + if (files.length > 0) { + let index = 0; + const queue = new Queue( + async (file: string) => { + if (!this.processed.has(file)) { + this.processed.add(file); + this.deps.resetDeps?.(file); + + // Check that the file is the entry file + if (this.isProcessable(file)) { + logger.prog(index, this.navigationPaths.size, file); + index++; + + await fn(file); + } + + this.addDepsToQueue(file, queue.add); + } + }, + PAGES_ACTIVE_QUEUE_LENGTH, + (error, file) => logger.error(file, error.message), + ); + + files.forEach(queue.add); + await queue.loop(); + } + } + + // Get initial file queue + private getFilesToProcess() { + const files = new Set( + Object.keys(this.context.meta?.files || {}).filter((path) => this.isChanged(path)), + ); + + for (const path of this.navigationPaths) { + if (this.isChanged(path)) { + files.add(path); + } + } + + return [...files]; + } + + // Find the dependency file and add them to queue + private addDepsToQueue(path: string, add: (path: string) => void) { + const dependencies = Object.keys(this.context.meta?.files || {}).filter((file) => { + const dependencies = this.context.meta?.files?.[file]?.dependencies; + return ( + dependencies?.['include']?.includes(path) || + dependencies?.['toc']?.includes(path) || + dependencies?.['presets']?.includes(path) + ); + }); + + for (const file of dependencies) { + if (!this.processed.has(file)) { + if (this.context.meta?.files?.[file]) { + this.context.meta.files[file].changed = true; + } + add(file); + } + } + } +} diff --git a/src/models.ts b/src/models.ts index 7b447965..97a76e6c 100644 --- a/src/models.ts +++ b/src/models.ts @@ -6,6 +6,8 @@ import {LintConfig} from '@diplodoc/transform/lib/yfmlint'; import {IncludeMode, Lang, ResourceType, Stage} from './constants'; import {FileContributors, VCSConnector, VCSConnectorConfig} from './vcs-connector/connector-models'; +import {RevisionContext} from './context/context'; +import {DependencyContext, FsContext} from '@diplodoc/transform/lib/typings'; export type VarsPreset = 'internal' | 'external'; @@ -26,7 +28,10 @@ export type NestedContributorsForPathFunction = ( nestedContributors: Contributors, ) => void; export type UserByLoginFunction = (login: string) => Promise; -export type CollectionOfPluginsFunction = (output: string, options: PluginOptions) => string; +export type CollectionOfPluginsFunction = ( + output: string, + options: PluginOptions, +) => Promise; export type GetModifiedTimeByPathFunction = (filepath: string) => number | undefined; /** @@ -58,6 +63,8 @@ interface YfmConfig { varsPreset: VarsPreset; ignore: string[]; outputFormat: string; + cached: boolean; + plugins: string; allowHTML: boolean; vars: Record; applyPresets: boolean; @@ -96,6 +103,7 @@ interface YfmConfig { export interface YfmArgv extends YfmConfig { rootInput: string; input: string; + config: string; output: string; quiet: string; publish: boolean; @@ -111,6 +119,7 @@ export interface YfmArgv extends YfmConfig { addMapFile: boolean; allowCustomResources: boolean; staticContent: boolean; + clean: boolean; } export type DocPreset = { @@ -257,16 +266,22 @@ export interface PluginOptions { changelogs?: ChangelogItem[]; extractChangelogs?: boolean; included?: boolean; + context: RevisionContext; + fs?: FsContext; + deps?: DependencyContext; } export interface Plugin { - collect: (input: string, options: PluginOptions) => string | void; + collect: (input: string, options: PluginOptions) => Promise; } export interface ResolveMd2MdOptions { inputPath: string; outputPath: string; metadata: MetaDataOptions; + context: RevisionContext; + fs: FsContext; + deps?: DependencyContext; } export interface ResolverOptions { @@ -278,6 +293,9 @@ export interface ResolverOptions { outputPath: string; outputBundlePath: string; metadata?: MetaDataOptions; + context: RevisionContext; + fs: FsContext; + deps?: DependencyContext; } export interface PathData { diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts index a98ed0b7..cd15cda0 100644 --- a/src/resolvers/lintPage.ts +++ b/src/resolvers/lintPage.ts @@ -10,23 +10,27 @@ import {isLocalUrl} from '@diplodoc/transform/lib/utils'; import {getLogLevel} from '@diplodoc/transform/lib/yfmlint/utils'; import {LINK_KEYS} from '@diplodoc/client/ssr'; -import {readFileSync} from 'fs'; import {bold} from 'chalk'; -import {ArgvService, PluginService} from '../services'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {ArgvService, PluginService} from '~/services'; +import {RevisionContext} from '~/context/context'; +import {FsContextCli} from '~/context/fs'; import { checkPathExists, findAllValuesByKeys, getLinksWithExtension, getVarsPerFile, getVarsPerRelativeFile, -} from '../utils'; +} from '~/utils'; import {liquidMd2Html} from './md2html'; import {liquidMd2Md} from './md2md'; interface FileTransformOptions { path: string; root?: string; + context: RevisionContext; + fs: FsContext; } const FileLinter: Record = { @@ -38,22 +42,24 @@ export interface ResolverLintOptions { inputPath: string; fileExtension: string; onFinish?: () => void; + context: RevisionContext; } -export function lintPage(options: ResolverLintOptions) { - const {inputPath, fileExtension, onFinish} = options; +export async function lintPage(options: ResolverLintOptions) { + const {inputPath, fileExtension, onFinish, context} = options; const {input} = ArgvService.getConfig(); const resolvedPath: string = resolve(input, inputPath); + const fs = new FsContextCli(context); try { - const content: string = readFileSync(resolvedPath, 'utf8'); + const content: string = await fs.readAsync(resolvedPath); const lintFn: Function = FileLinter[fileExtension]; if (!lintFn) { return; } - lintFn(content, {path: inputPath}); + await lintFn(content, {path: inputPath, fs, context}); } catch (e) { const message = `No such file or has no access to ${bold(resolvedPath)}`; console.error(message, e); @@ -65,7 +71,7 @@ export function lintPage(options: ResolverLintOptions) { } } -function YamlFileLinter(content: string, lintOptions: FileTransformOptions): void { +async function YamlFileLinter(content: string, lintOptions: FileTransformOptions): Promise { const {input, lintConfig} = ArgvService.getConfig(); const {path: filePath} = lintOptions; const currentFilePath: string = resolve(input, filePath); @@ -76,21 +82,24 @@ function YamlFileLinter(content: string, lintOptions: FileTransformOptions): voi defaultLevel: log.LogLevels.ERROR, }); - const contentLinks = findAllValuesByKeys(load(content), LINK_KEYS); + const data = load(content) as object; + const contentLinks: string[] = findAllValuesByKeys(data, LINK_KEYS); const localLinks = contentLinks.filter( (link) => getLinksWithExtension(link) && isLocalUrl(link), ); - return localLinks.forEach( - (link) => - checkPathExists(link, currentFilePath) || - log[logLevel](`Link is unreachable: ${bold(link)} in ${bold(currentFilePath)}`), + await Promise.all( + localLinks.map( + async (link) => + (await checkPathExists(lintOptions.fs, link, currentFilePath)) || + log[logLevel](`Link is unreachable: ${bold(link)} in ${bold(currentFilePath)}`), + ), ); } -function MdFileLinter(content: string, lintOptions: FileTransformOptions): void { +async function MdFileLinter(content: string, lintOptions: FileTransformOptions): Promise { const {input, lintConfig, disableLiquid, outputFormat, ...options} = ArgvService.getConfig(); - const {path: filePath} = lintOptions; + const {path: filePath, fs} = lintOptions; const plugins = outputFormat === 'md' ? [] : PluginService.getPlugins(); const vars = getVarsPerFile(filePath); @@ -101,7 +110,7 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void /* Relative path from folder of .md file to root of user' output folder */ const assetsPublicPath = relative(dirname(path), root); - const lintMarkdown = function lintMarkdown(opts: LintMarkdownFunctionOptions) { + async function lintMarkdown(opts: LintMarkdownFunctionOptions) { const {input: localInput, path: localPath, sourceMap} = opts; const pluginOptions: PluginOptions = { @@ -114,9 +123,10 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void disableLiquid, log, getVarsPerFile: getVarsPerRelativeFile, + fs, }; - yfmlint({ + await yfmlint({ input: localInput, lintConfig, pluginOptions, @@ -125,22 +135,22 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void customLintRules: PluginService.getCustomLintRules(), sourceMap, }); - }; + } let sourceMap; if (!disableLiquid) { let liquidResult; if (outputFormat === 'md') { - liquidResult = liquidMd2Md(content, vars, path); + liquidResult = await liquidMd2Md(content, vars, path); } else { - liquidResult = liquidMd2Html(content, vars, path); + liquidResult = await liquidMd2Html(content, vars, path); } preparedContent = liquidResult.output; sourceMap = liquidResult.sourceMap; } - lintMarkdown({ + await lintMarkdown({ input: preparedContent, path, sourceMap, diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts index 4df66f56..3fc2d307 100644 --- a/src/resolvers/md2html.ts +++ b/src/resolvers/md2html.ts @@ -1,21 +1,19 @@ -import type {DocInnerProps} from '@diplodoc/client'; - -import {readFileSync, writeFileSync} from 'fs'; import {basename, dirname, join, resolve, sep} from 'path'; -import {LINK_KEYS, preprocess} from '@diplodoc/client/ssr'; import {isString} from 'lodash'; +import yaml from 'js-yaml'; +import type {DocInnerProps} from '@diplodoc/client'; +import {LINK_KEYS, preprocess} from '@diplodoc/client/ssr'; import transform, {Output} from '@diplodoc/transform'; import liquid from '@diplodoc/transform/lib/liquid'; import log from '@diplodoc/transform/lib/log'; import {MarkdownItPluginCb} from '@diplodoc/transform/lib/plugins/typings'; import {getPublicPath, isFileExists} from '@diplodoc/transform/lib/utilsFS'; -import yaml from 'js-yaml'; -import {Lang, PROCESSING_FINISHED} from '../constants'; -import {LeadingPage, ResolverOptions, YfmToc} from '../models'; -import {ArgvService, PluginService, SearchService, TocService} from '../services'; -import {getAssetsPublicPath, getAssetsRootPath, getVCSMetadata} from '../services/metadata'; +import {Lang, PROCESSING_FINISHED} from '~/constants'; +import {LeadingPage, ResolverOptions, YfmToc} from '~/models'; +import {ArgvService, PluginService, SearchService, TocService} from '~/services'; +import {getAssetsPublicPath, getAssetsRootPath, getVCSMetadata} from '~/services/metadata'; import { getLinksWithContentExtersion, getVarsPerFile, @@ -24,11 +22,17 @@ import { modifyValuesByKeys, transformToc, } from '../utils'; -import {generateStaticMarkup} from '../pages'; +import {RevisionContext} from '~/context/context'; +import {DependencyContext, FsContext} from '@diplodoc/transform/lib/typings'; +import {generateStaticMarkup} from '~/pages'; export interface FileTransformOptions { + lang: string; path: string; root?: string; + fs: FsContext; + context: RevisionContext; + deps: DependencyContext; } const FileTransformer: Record = { @@ -40,14 +44,22 @@ const fixRelativePath = (relativeTo: string) => (path: string) => { return join(getAssetsPublicPath(relativeTo), path); }; -const getFileMeta = async ({fileExtension, metadata, inputPath}: ResolverOptions) => { +async function getFileMeta({ + fileExtension, + metadata, + inputPath, + context, + fs, + deps, +}: ResolverOptions) { const {input, allowCustomResources} = ArgvService.getConfig(); const resolvedPath: string = resolve(input, inputPath); - const content: string = readFileSync(resolvedPath, 'utf8'); + const content: string = await fs.readAsync(resolvedPath); const transformFn: Function = FileTransformer[fileExtension]; - const {result} = transformFn(content, {path: inputPath}); + + const {result} = await transformFn(content, {path: inputPath, context, fs, deps}); const vars = getVarsPerFile(inputPath); const updatedMetadata = metadata?.isContributorsEnabled @@ -73,9 +85,9 @@ const getFileMeta = async ({fileExtension, metadata, inputPath}: ResolverOptions } return {...result, meta: fileMeta}; -}; +} -const getFileProps = async (options: ResolverOptions) => { +async function getFileProps(options: ResolverOptions) { const {inputPath, outputPath} = options; const pathToDir: string = dirname(inputPath); @@ -117,14 +129,14 @@ const getFileProps = async (options: ResolverOptions) => { }; return props; -}; +} export async function resolveMd2HTML(options: ResolverOptions): Promise { - const {outputPath, inputPath, deep, deepBase} = options; + const {outputPath, inputPath, deep, deepBase, fs} = options; const props = await getFileProps(options); const outputFileContent = generateStaticMarkup(props, deepBase, deep); - writeFileSync(outputPath, outputFileContent); + await fs.writeAsync(outputPath, outputFileContent); logger.info(inputPath, PROCESSING_FINISHED); return props; @@ -159,7 +171,10 @@ function getHref(path: string, href: string) { return href; } -function YamlFileTransformer(content: string, transformOptions: FileTransformOptions): Object { +async function YamlFileTransformer( + content: string, + transformOptions: FileTransformOptions, +): Object { let data: LeadingPage | null = null; try { @@ -184,7 +199,7 @@ function YamlFileTransformer(content: string, transformOptions: FileTransformOpt const {path, lang} = transformOptions; const transformFn: Function = FileTransformer['.md']; - data = preprocess(data, {lang}, (lang, content) => { + data = await preprocess(data, {lang}, (lang, content) => { const {result} = transformFn(content, {path}); return result?.html; }); @@ -210,28 +225,28 @@ function YamlFileTransformer(content: string, transformOptions: FileTransformOpt }; } -export function liquidMd2Html(input: string, vars: Record, path: string) { +export async function liquidMd2Html(input: string, vars: Record, path: string) { const {conditionsInCode, useLegacyConditions} = ArgvService.getConfig(); - return liquid(input, vars, path, { + return await liquid(input, vars, path, { conditionsInCode, withSourceMap: true, useLegacyConditions, }); } -function MdFileTransformer(content: string, transformOptions: FileTransformOptions): Output { +async function MdFileTransformer(content: string, transformOptions: FileTransformOptions): Output { const {input, ...options} = ArgvService.getConfig(); - const {path: filePath} = transformOptions; + const {path: filePath, context, fs, deps} = transformOptions; const plugins = PluginService.getPlugins(); const vars = getVarsPerFile(filePath); const root = resolve(input); const path: string = resolve(input, filePath); - return transform(content, { + return await transform(content, { ...options, - plugins: plugins as MarkdownItPluginCb[], + plugins: plugins as MarkdownItPluginCb[], vars, root, path, @@ -240,5 +255,8 @@ function MdFileTransformer(content: string, transformOptions: FileTransformOptio getVarsPerFile: getVarsPerRelativeFile, getPublicPath, extractTitle: true, + context, + fs, + deps, }); } diff --git a/src/resolvers/md2md.ts b/src/resolvers/md2md.ts index 300913f3..7971cdd2 100644 --- a/src/resolvers/md2md.ts +++ b/src/resolvers/md2md.ts @@ -1,25 +1,24 @@ -import {readFileSync, writeFileSync} from 'fs'; import {basename, dirname, extname, join, resolve} from 'path'; import shell from 'shelljs'; import log from '@diplodoc/transform/lib/log'; import liquid from '@diplodoc/transform/lib/liquid'; +import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types'; import {ArgvService, PluginService} from '../services'; import {getVarsPerFile, logger} from '../utils'; import {PluginOptions, ResolveMd2MdOptions} from '../models'; import {PROCESSING_FINISHED} from '../constants'; -import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types'; import {enrichWithFrontMatter} from '../services/metadata'; export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise { - const {inputPath, outputPath, metadata: metadataOptions} = options; + const {inputPath, outputPath, metadata: metadataOptions, fs} = options; const {input, output, changelogs: changelogsSetting, included} = ArgvService.getConfig(); const resolvedInputPath = resolve(input, inputPath); const vars = getVarsPerFile(inputPath); const content = await enrichWithFrontMatter({ - fileContent: readFileSync(resolvedInputPath, 'utf8'), + fileContent: await fs.readAsync(resolvedInputPath), metadataOptions, resolvedFrontMatterVars: { systemVars: vars.__system as unknown, @@ -27,7 +26,21 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise }, }); - const {result, changelogs} = transformMd2Md(content, { + async function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) { + shell.mkdir('-p', dirname(targetDestPath)); + + if (options) { + const sourceIncludeContent = await fs.readAsync(targetPath); + const {result} = await transformMd2Md(sourceIncludeContent, options); + + await fs.writeAsync(targetDestPath, result); + } else { + shell.cp(targetPath, targetDestPath); + } + } + + const {result, changelogs} = await transformMd2Md(content, { + ...options, path: resolvedInputPath, destPath: outputPath, root: resolve(input), @@ -35,16 +48,21 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise collectOfPlugins: PluginService.getCollectOfPlugins(), vars: vars, log, - copyFile, included, + copyFile, + context: options.context, + fs: options.fs, + deps: options.deps, }); - writeFileSync(outputPath, result); + await fs.writeAsync(outputPath, result); if (changelogsSetting && changelogs?.length) { const mdFilename = basename(outputPath, extname(outputPath)); const outputDir = dirname(outputPath); - changelogs.forEach((changes, index) => { + + let index = 0; + for (const changes of changelogs) { let changesName; const changesDate = changes.date as string | undefined; const changesIdx = changes.index as number | undefined; @@ -63,14 +81,16 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise const changesPath = join(outputDir, `__changes-${changesName}.json`); - writeFileSync( + await fs.writeAsync( changesPath, JSON.stringify({ ...changes, source: mdFilename, }), ); - }); + + index++; + } } logger.info(inputPath, PROCESSING_FINISHED); @@ -78,23 +98,11 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise return undefined; } -function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) { - shell.mkdir('-p', dirname(targetDestPath)); - - if (options) { - const sourceIncludeContent = readFileSync(targetPath, 'utf8'); - const {result} = transformMd2Md(sourceIncludeContent, options); - writeFileSync(targetDestPath, result); - } else { - shell.cp(targetPath, targetDestPath); - } -} - -export function liquidMd2Md(input: string, vars: Record, path: string) { +export async function liquidMd2Md(input: string, vars: Record, path: string) { const {applyPresets, resolveConditions, conditionsInCode, useLegacyConditions} = ArgvService.getConfig(); - return liquid(input, vars, path, { + return await liquid(input, vars, path, { conditions: resolveConditions, substitutions: applyPresets, conditionsInCode, @@ -104,7 +112,7 @@ export function liquidMd2Md(input: string, vars: Record, path: }); } -function transformMd2Md(input: string, options: PluginOptions) { +async function transformMd2Md(input: string, options: PluginOptions) { const {disableLiquid, changelogs: changelogsSetting} = ArgvService.getConfig(); const {vars = {}, path, collectOfPlugins, log: pluginLog} = options; @@ -112,13 +120,13 @@ function transformMd2Md(input: string, options: PluginOptions) { const changelogs: ChangelogItem[] = []; if (!disableLiquid) { - const liquidResult = liquidMd2Md(input, vars, path); + const liquidResult = await liquidMd2Md(input, vars, path); output = liquidResult.output; } if (collectOfPlugins) { - output = collectOfPlugins(output, { + output = await collectOfPlugins(output, { ...options, vars, path, diff --git a/src/services/leading.ts b/src/services/leading.ts index d785a996..a8366c20 100644 --- a/src/services/leading.ts +++ b/src/services/leading.ts @@ -1,5 +1,5 @@ import {dirname, resolve} from 'path'; -import {readFileSync, writeFileSync} from 'fs'; +import {readFile, writeFile} from 'fs/promises'; import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; @@ -13,12 +13,12 @@ import { liquidFields, } from './utils'; -function filterFile(path: string) { +async function filterFile(path: string) { const {input: inputFolderPath, vars} = ArgvService.getConfig(); const pathToDir = dirname(path); const filePath = resolve(inputFolderPath, path); - const content = readFileSync(filePath, 'utf8'); + const content = await readFile(filePath, 'utf8'); const parsedIndex = load(content) as LeadingPage; const combinedVars = { @@ -74,7 +74,7 @@ function filterFile(path: string) { } }); - writeFileSync(filePath, dump(parsedIndex)); + await writeFile(filePath, dump(parsedIndex)); } catch (error) { log.error(`Error while filtering index file: ${path}. Error message: ${error}`); } diff --git a/src/services/plugins.ts b/src/services/plugins.ts index b941a0ab..b623b7c0 100644 --- a/src/services/plugins.ts +++ b/src/services/plugins.ts @@ -1,7 +1,10 @@ import {LintConfig, LintRule} from '@diplodoc/transform/lib/yfmlint'; +import {existsSync} from 'fs'; +import {resolve} from 'path'; import {CollectionOfPluginsFunction, Plugin, PluginOptions} from '../models'; import {YFM_PLUGINS} from '../constants'; +import {ArgvService} from '.'; let plugins: Function[] | Plugin[]; let collectionOfPlugins: CollectionOfPluginsFunction; @@ -24,22 +27,22 @@ function makeCollectOfPlugins(): CollectionOfPluginsFunction { return typeof plugin.collect === 'function'; }); - return (output: string, options: PluginOptions) => { + return async (output: string, options: PluginOptions) => { let collectsOutput = output; - pluginsWithCollect.forEach((plugin: Plugin) => { - const collectOutput = plugin.collect(collectsOutput, options); - + for (const plugin of pluginsWithCollect) { + const collectOutput = await plugin.collect(collectsOutput, options); collectsOutput = typeof collectOutput === 'string' ? collectOutput : collectsOutput; - }); + } return collectsOutput; }; } function getAllPlugins(): Function[] { + const argsPlugins = getArgsPlugins(); const customPlugins = getCustomPlugins(); - return [...YFM_PLUGINS, ...customPlugins]; + return [...YFM_PLUGINS, ...argsPlugins, ...customPlugins]; } function getCustomPlugins(): Function[] { @@ -51,6 +54,14 @@ function getCustomPlugins(): Function[] { } } +function getArgsPlugins(): Function[] { + const {plugins: pluginsFile} = ArgvService.getConfig(); + if (pluginsFile && existsSync(resolve(pluginsFile))) { + return require(resolve(pluginsFile)); + } + return []; +} + export function getHeadContent(): string { try { return require(require.resolve('./head-content.js')); diff --git a/src/services/tocs.ts b/src/services/tocs.ts index d4bbcd98..04d93fa0 100644 --- a/src/services/tocs.ts +++ b/src/services/tocs.ts @@ -1,16 +1,15 @@ import {dirname, extname, join, normalize, parse, relative, resolve, sep} from 'path'; -import {existsSync, readFileSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import shell from 'shelljs'; -import walkSync from 'walk-sync'; import liquid from '@diplodoc/transform/lib/liquid'; import log from '@diplodoc/transform/lib/log'; +import {FsContext} from '@diplodoc/transform/lib/typings'; import {bold} from 'chalk'; import {ArgvService, PresetService} from './index'; import {YfmToc} from '../models'; import {IncludeMode, Stage} from '../constants'; -import {isExternalHref, logger} from '../utils'; +import {isExternalHref, logger, walk} from '../utils'; import {filterFiles, firstFilterItem, firstFilterTextItems, liquidField} from './utils'; import {IncludersError, applyIncluders} from './includers'; import {addSourcePath} from './metadata'; @@ -22,13 +21,16 @@ export interface TocServiceData { includedTocPaths: Set; } +let fsContext: FsContext; const storage: TocServiceData['storage'] = new Map(); const tocs: TocServiceData['tocs'] = new Map(); let navigationPaths: TocServiceData['navigationPaths'] = []; const includedTocPaths: TocServiceData['includedTocPaths'] = new Set(); const tocFileCopyMap = new Map(); -async function init(tocFilePaths: string[]) { +async function init(fs: FsContext, tocFilePaths: string[]) { + fsContext = fs; + for (const path of tocFilePaths) { logger.proc(path); @@ -58,7 +60,7 @@ async function add(path: string) { } = ArgvService.getConfig(); const pathToDir = dirname(path); - const content = readFileSync(resolve(inputFolderPath, path), 'utf8'); + const content = await fsContext.readAsync(resolve(inputFolderPath, path)); const parsedToc = load(content) as YfmToc; // Should ignore toc with specified stage. @@ -114,7 +116,7 @@ async function add(path: string) { const outputPath = resolve(outputFolderPath, path); const outputToc = dump(parsedToc); shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputToc); + await fsContext.writeAsync(outputPath, outputToc); } } @@ -258,17 +260,18 @@ function _normalizeHref(href: string): string { * @return * @private */ -function _copyTocDir(tocPath: string, destDir: string) { +async function _copyTocDir(tocPath: string, destDir: string) { const {input: inputFolderPath} = ArgvService.getConfig(); const {dir: tocDir} = parse(tocPath); - const files: string[] = walkSync(tocDir, { + const files: string[] = walk({ + folder: tocDir, globs: ['**/*.*'], ignore: ['**/toc.yaml'], directories: false, }); - files.forEach((relPath) => { + for (const relPath of files) { const from = resolve(tocDir, relPath); const to = resolve(destDir, relPath); const fileExtension = extname(relPath); @@ -277,11 +280,11 @@ function _copyTocDir(tocPath: string, destDir: string) { shell.mkdir('-p', parse(to).dir); if (isMdFile) { - const fileContent = readFileSync(from, 'utf8'); + const fileContent = await fsContext.readAsync(from); const sourcePath = relative(inputFolderPath, from); const updatedFileContent = addSourcePath(fileContent, sourcePath); - writeFileSync(to, updatedFileContent); + await fsContext.writeAsync(to, updatedFileContent); } else { shell.cp(from, to); } @@ -289,7 +292,7 @@ function _copyTocDir(tocPath: string, destDir: string) { const relFrom = relative(inputFolderPath, from); const relTo = relative(inputFolderPath, to); tocFileCopyMap.set(relTo, relFrom); - }); + } } /** @@ -392,7 +395,7 @@ async function _replaceIncludes( const includeTocDir = dirname(includeTocPath); try { - const includeToc = load(readFileSync(includeTocPath, 'utf8')) as YfmToc; + const includeToc = load(await fsContext.readAsync(includeTocPath)) as YfmToc; // Should ignore included toc with tech-preview stage. if (includeToc.stage === Stage.TECH_PREVIEW) { @@ -400,7 +403,7 @@ async function _replaceIncludes( } if (mode === IncludeMode.MERGE || mode === IncludeMode.ROOT_MERGE) { - _copyTocDir(includeTocPath, tocDir); + await _copyTocDir(includeTocPath, tocDir); } /* Save the path to exclude toc from the output directory in the next step */ @@ -457,21 +460,23 @@ async function _replaceIncludes( return result; } -function getTocDir(pagePath: string): string { +async function getTocDir(pagePath: string, pageBasePath?: string): Promise { + pageBasePath = pageBasePath ?? pagePath; + const {input: inputFolderPath} = ArgvService.getConfig(); const tocDir = dirname(pagePath); const tocPath = resolve(tocDir, 'toc.yaml'); if (!tocDir.includes(inputFolderPath)) { - throw new Error('Error while finding toc dir'); + throw new Error(`Error while finding toc dir for "${pageBasePath}"`); } - if (existsSync(tocPath)) { + if (await fsContext.existAsync(tocPath)) { return tocDir; } - return getTocDir(tocDir); + return await getTocDir(tocDir, pageBasePath); } function setNavigationPaths(paths: TocServiceData['navigationPaths']) { diff --git a/src/steps/processAssets.ts b/src/steps/processAssets.ts index f7337228..5ebe3001 100644 --- a/src/steps/processAssets.ts +++ b/src/steps/processAssets.ts @@ -1,23 +1,23 @@ import walkSync from 'walk-sync'; import {load} from 'js-yaml'; -import {readFileSync} from 'fs'; import shell from 'shelljs'; import {join, resolve, sep} from 'path'; -import {ArgvService, TocService} from '../services'; -import {checkPathExists, copyFiles, findAllValuesByKeys} from '../utils'; - import {LINK_KEYS} from '@diplodoc/client/ssr'; import {isLocalUrl} from '@diplodoc/transform/lib/utils'; +import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {FsContext} from '@diplodoc/transform/lib/typings'; import { ASSETS_FOLDER, LINT_CONFIG_FILENAME, REDIRECTS_FILENAME, YFM_CONFIG_FILENAME, -} from '../constants'; -import {Resources} from '../models'; -import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +} from '~/constants'; +import {ArgvService, TocService} from '~/services'; +import {checkPathExists, copyFiles, findAllValuesByKeys} from '~/utils'; +import {Resources, YfmArgv} from '~/models'; +import {RevisionContext} from '~/context/context'; /** * @param {Array} args @@ -28,26 +28,30 @@ import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; */ type Props = { - args: string[]; + args: YfmArgv; outputBundlePath: string; outputFormat: string; tmpOutputFolder: string; + userOutputFolder: string; + context: RevisionContext; + fs: FsContext; }; + /* * Processes assets files (everything except .md files) */ -export function processAssets({args, outputFormat, outputBundlePath, tmpOutputFolder}: Props) { - switch (outputFormat) { +export async function processAssets(props: Props) { + switch (props.outputFormat) { case 'html': - processAssetsHtmlRun({outputBundlePath}); + await processAssetsHtmlRun(props); break; case 'md': - processAssetsMdRun({args, tmpOutputFolder}); + await processAssetsMdRun(props); break; } } -function processAssetsHtmlRun({outputBundlePath}) { +async function processAssetsHtmlRun({outputBundlePath, context}: Props) { const {input: inputFolderPath, output: outputFolderPath} = ArgvService.getConfig(); const documentationAssetFilePath: string[] = walkSync(inputFolderPath, { @@ -56,17 +60,17 @@ function processAssetsHtmlRun({outputBundlePath}) { ignore: ['**/*.yaml', '**/*.md'], }); - copyFiles(inputFolderPath, outputFolderPath, documentationAssetFilePath); + await copyFiles(inputFolderPath, outputFolderPath, documentationAssetFilePath, context.meta); const bundleAssetFilePath: string[] = walkSync(ASSETS_FOLDER, { directories: false, includeBasePath: false, }); - copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath); + await copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath, context.meta); } -function processAssetsMdRun({args, tmpOutputFolder}) { +async function processAssetsMdRun({args, tmpOutputFolder, context, fs}: Props) { const {input: inputFolderPath, allowCustomResources, resources} = ArgvService.getConfig(); const pathToConfig = args.config || join(args.input, YFM_CONFIG_FILENAME); @@ -85,11 +89,11 @@ function processAssetsMdRun({args, tmpOutputFolder}) { resources[type as keyof Resources]?.forEach((path: string) => resourcePaths.push(path)), ); - //copy resources - copyFiles(args.input, tmpOutputFolder, resourcePaths); + // copy resources + await copyFiles(args.input, tmpOutputFolder, resourcePaths, context.meta); } - const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { + const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { if (file.endsWith('.yaml')) { const resolvedPathToFile = resolve(inputFolderPath, file); @@ -98,32 +102,33 @@ function processAssetsMdRun({args, tmpOutputFolder}) { return acc; }, []); - tocYamlFiles.forEach((yamlFile) => { - const content = load(readFileSync(yamlFile, 'utf8')); + for (const yamlFile of tocYamlFiles) { + const content = load(await fs.readAsync(yamlFile)) as object; if (!Object.prototype.hasOwnProperty.call(content, 'blocks')) { return; } const contentLinks = findAllValuesByKeys(content, LINK_KEYS); - const localMediaLinks = contentLinks.reduce( - (acc, link) => { - const linkHasMediaExt = new RegExp( - /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, - ).test(link); - - if (linkHasMediaExt && isLocalUrl(link) && checkPathExists(link, yamlFile)) { - const linkAbsolutePath = resolveRelativePath(yamlFile, link); - const linkRootPath = linkAbsolutePath.replace(`${inputFolderPath}${sep}`, ''); - - acc.push(linkRootPath); - } - return acc; - }, - - [], - ); + const localMediaLinks = []; + + for (const link of contentLinks) { + const linkHasMediaExt = new RegExp( + /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, + ).test(link); + + if ( + linkHasMediaExt && + isLocalUrl(link) && + (await checkPathExists(fs, link, yamlFile)) + ) { + const linkAbsolutePath = resolveRelativePath(yamlFile, link); + const linkRootPath = linkAbsolutePath.replace(`${inputFolderPath}${sep}`, ''); + + localMediaLinks.push(linkRootPath); + } + } - copyFiles(args.input, tmpOutputFolder, localMediaLinks); - }); + await copyFiles(args.input, tmpOutputFolder, localMediaLinks, context.meta); + } } diff --git a/src/steps/processExcludedFiles.ts b/src/steps/processExcludedFiles.ts index f34f8420..9a404709 100644 --- a/src/steps/processExcludedFiles.ts +++ b/src/steps/processExcludedFiles.ts @@ -25,7 +25,7 @@ export function processExcludedFiles() { const tocSpecifiedFiles = new Set(navigationPaths); const excludedFiles = allContentFiles.filter((filePath) => !tocSpecifiedFiles.has(filePath)); - if (excludedFiles.length) { + if (excludedFiles?.length) { shell.rm('-f', excludedFiles); } @@ -36,7 +36,7 @@ export function processExcludedFiles() { return convertBackSlashToSlash(destTocPath); }); - if (includedTocPaths.length) { + if (includedTocPaths?.length) { shell.rm('-rf', includedTocPaths); } } diff --git a/src/steps/processLinter.ts b/src/steps/processLinter.ts index 13c3c050..3e5b26d7 100644 --- a/src/steps/processLinter.ts +++ b/src/steps/processLinter.ts @@ -2,23 +2,31 @@ import log from '@diplodoc/transform/lib/log'; import {Thread, Worker, spawn} from 'threads'; import {extname} from 'path'; -import {ArgvService, PluginService, PresetService, TocService} from '../services'; -import {ProcessLinterWorker} from '../workers/linter'; -import {logger} from '../utils'; -import {LINTING_FINISHED, MIN_CHUNK_SIZE, WORKERS_COUNT} from '../constants'; -import {lintPage} from '../resolvers'; -import {splitOnChunks} from '../utils/worker'; +import {LINTING_FINISHED, MIN_CHUNK_SIZE, WORKERS_COUNT} from '~/constants'; +import {ArgvService, PluginService, PresetService} from '~/services'; +import {ProcessLinterWorker} from '~/workers/linter'; +import {logger} from '~/utils'; +import {lintPage} from '~/resolvers'; +import {splitOnChunks} from '~/utils/worker'; +import {RevisionContext} from '~/context/context'; let processLinterWorkers: (ProcessLinterWorker & Thread)[]; -let navigationPathsChunks: string[][]; +let filesToProcessChunks: string[][]; -export async function processLinter(): Promise { +export async function processLinter( + context: RevisionContext, + filesToProcess: string[], +): Promise { const argvConfig = ArgvService.getConfig(); - const navigationPaths = TocService.getNavigationPaths(); - if (!processLinterWorkers) { - lintPagesFallback(navigationPaths); + await lintPagesFallback(filesToProcess, context); + + const {error} = log.get(); + + if (error.length > 0) { + throw Error('Linting the project has failed'); + } return; } @@ -35,20 +43,27 @@ export async function processLinter(): Promise { /* Run processing the linter */ await Promise.all( processLinterWorkers.map((worker, i) => { - const navigationPathsChunk = navigationPathsChunks[i]; + const navigationPathsChunk = filesToProcessChunks[i]; return worker.run({ argvConfig, presetStorage, navigationPaths: navigationPathsChunk, + context, }); }), ); + let isSuccess = true; + /* Unsubscribe from workers */ await Promise.all( processLinterWorkers.map((worker) => { return worker.finish().then((logs) => { + if (logs.error?.length > 0) { + isSuccess = false; + } + log.add(logs); }); }), @@ -60,19 +75,22 @@ export async function processLinter(): Promise { return Thread.terminate(worker); }), ); + + if (!isSuccess) { + throw Error('Linting the project has failed'); + } } -export async function initLinterWorkers() { - const navigationPaths = TocService.getNavigationPaths(); - const chunkSize = getChunkSize(navigationPaths); +export async function initLinterWorkers(filesToProcess: string[]) { + const chunkSize = getChunkSize(filesToProcess); if (process.env.DISABLE_PARALLEL_BUILD || chunkSize < MIN_CHUNK_SIZE || WORKERS_COUNT <= 0) { return; } - navigationPathsChunks = splitOnChunks(navigationPaths, chunkSize).filter((arr) => arr.length); + filesToProcessChunks = splitOnChunks(filesToProcess, chunkSize).filter((arr) => arr.length); - const workersCount = navigationPathsChunks.length; + const workersCount = filesToProcessChunks.length; processLinterWorkers = await Promise.all( new Array(workersCount).fill(null).map(() => { @@ -86,16 +104,32 @@ function getChunkSize(arr: string[]) { return Math.ceil(arr.length / WORKERS_COUNT); } -function lintPagesFallback(navigationPaths: string[]) { +async function lintPagesFallback(filesToProcess: string[], context: RevisionContext) { PluginService.setPlugins(); - navigationPaths.forEach((pathToFile) => { - lintPage({ + for (const pathToFile of filesToProcess) { + await lintPage({ inputPath: pathToFile, fileExtension: extname(pathToFile), onFinish: () => { logger.info(pathToFile, LINTING_FINISHED); }, + context, }); - }); + } +} + +export async function getLintFn(context: RevisionContext) { + PluginService.setPlugins(); + + return async (pathToFile: string) => { + await lintPage({ + inputPath: pathToFile, + fileExtension: extname(pathToFile), + onFinish: () => { + logger.info(pathToFile, LINTING_FINISHED); + }, + context, + }); + }; } diff --git a/src/steps/processMapFile.ts b/src/steps/processMapFile.ts index 5dc9c7d6..c69d7b1a 100644 --- a/src/steps/processMapFile.ts +++ b/src/steps/processMapFile.ts @@ -1,4 +1,4 @@ -import {writeFileSync} from 'fs'; +import {writeFile} from 'fs/promises'; import {extname, join} from 'path'; import {ArgvService, TocService} from '../services'; @@ -12,7 +12,7 @@ type TocItem = { type TocItems = TocItem[]; -export function prepareMapFile(): void { +export async function prepareMapFile(): void { const {output: outputFolderPath} = ArgvService.getConfig(); const navigationPathsWithoutExtensions = TocService.getNavigationPaths().map((path) => { @@ -28,5 +28,5 @@ export function prepareMapFile(): void { const filesMapBuffer = Buffer.from(JSON.stringify(navigationPaths, null, '\t'), 'utf8'); const mapFile = join(outputFolderPath, 'files.json'); - writeFileSync(mapFile, filesMapBuffer); + await writeFile(mapFile, filesMapBuffer); } diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index 965c3e1f..ce8d1359 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -1,6 +1,5 @@ import type {DocInnerProps} from '@diplodoc/client'; -import {basename, dirname, extname, join, relative, resolve} from 'path'; -import {existsSync, readFileSync, writeFileSync} from 'fs'; +import {basename, dirname, extname, join, relative, resolve, sep} from 'path'; import log from '@diplodoc/transform/lib/log'; import {asyncify, mapLimit} from 'async'; import {bold} from 'chalk'; @@ -13,7 +12,7 @@ import { ResourceType, SINGLE_PAGE_DATA_FILENAME, SINGLE_PAGE_FILENAME, -} from '../constants'; +} from '~/constants'; import { LeadingPage, MetaDataOptions, @@ -21,20 +20,26 @@ import { Resources, SinglePageResult, YfmToc, -} from '../models'; -import {resolveMd2HTML, resolveMd2Md} from '../resolvers'; -import {ArgvService, LeadingService, PluginService, SearchService, TocService} from '../services'; -import {generateStaticMarkup} from '~/pages/document'; -import {generateStaticRedirect} from '~/pages/redirect'; -import {joinSinglePageResults, logger, transformTocForSinglePage} from '../utils'; -import {getVCSConnector} from '../vcs-connector'; -import {VCSConnector} from '../vcs-connector/connector-models'; +} from '~/models'; +import {resolveMd2HTML, resolveMd2Md} from '~/resolvers'; +import {ArgvService, LeadingService, PluginService, SearchService, TocService} from '~/services'; +import {joinSinglePageResults, logger, transformTocForSinglePage} from '~/utils'; +import {generateStaticMarkup, generateStaticRedirect} from '~/pages'; +import {getVCSConnector} from '~/vcs-connector'; +import {VCSConnector} from '~/vcs-connector/connector-models'; +import {RevisionContext} from '~/context/context'; +import {DependencyContext, FsContext} from '@diplodoc/transform/lib/typings'; const singlePageResults: Record = {}; const singlePagePaths: Record> = {}; // Processes files of documentation (like index.yaml, *.md) -export async function processPages(outputBundlePath: string): Promise { +export async function processPages( + fs: FsContext, + deps: DependencyContext, + outputBundlePath: string, + context: RevisionContext, +): Promise { const { input: inputFolderPath, output: outputFolderPath, @@ -53,7 +58,7 @@ export async function processPages(outputBundlePath: string): Promise { navigationPaths, PAGE_PROCESS_CONCURRENCY, asyncify(async (pathToFile: string) => { - const pathData = getPathData( + const pathData = await getPathData( pathToFile, inputFolderPath, outputFolderPath, @@ -66,30 +71,88 @@ export async function processPages(outputBundlePath: string): Promise { const metaDataOptions = getMetaDataOptions(pathData, vcsConnector); await preparingPagesByOutputFormat( + fs, + deps, pathData, metaDataOptions, resolveConditions, singlePage, + context, ); }), ); if (singlePage) { - await saveSinglePages(); + await saveSinglePages(fs); } if (outputFormat === 'html') { - saveRedirectPage(outputFolderPath); + await saveRedirectPage(fs, outputFolderPath); } } -function getPathData( +export const getProcessPageFn = async ( + fs: FsContext, + deps: DependencyContext, + context: RevisionContext, + outputBundlePath: string, +) => { + const { + input: inputFolderPath, + output: outputFolderPath, + outputFormat, + singlePage, + resolveConditions, + } = ArgvService.getConfig(); + + const vcsConnector = await getVCSConnector(); + + PluginService.setPlugins(); + + return async (pathToFile: string) => { + const pathData = await getPathData( + pathToFile, + inputFolderPath, + outputFolderPath, + outputFormat, + outputBundlePath, + ); + + logger.proc(pathToFile); + + const metaDataOptions = getMetaDataOptions(pathData, vcsConnector); + + await preparingPagesByOutputFormat( + fs, + deps, + pathData, + metaDataOptions, + resolveConditions, + singlePage, + context, + ); + }; +}; + +export const finishProcessPages = async (fs: FsContext) => { + const {output: outputFolderPath, outputFormat, singlePage} = ArgvService.getConfig(); + + if (singlePage) { + await saveSinglePages(fs); + } + + if (outputFormat === 'html') { + saveRedirectPage(fs, outputFolderPath); + } +}; + +async function getPathData( pathToFile: string, inputFolderPath: string, outputFolderPath: string, outputFormat: string, outputBundlePath: string, -): PathData { +): Promise { const pathToDir: string = dirname(pathToFile); const filename: string = basename(pathToFile); const fileExtension: string = extname(pathToFile); @@ -98,7 +161,8 @@ function getPathData( const outputFileName = `${fileBaseName}.${outputFormat}`; const outputPath = resolve(outputDir, outputFileName); const resolvedPathToFile = resolve(inputFolderPath, pathToFile); - const outputTocDir = TocService.getTocDir(resolvedPathToFile); + + const outputTocDir = await TocService.getTocDir(resolvedPathToFile); const pathData: PathData = { pathToFile, @@ -118,7 +182,7 @@ function getPathData( return pathData; } -async function saveSinglePages() { +async function saveSinglePages(fs: FsContext) { const { input: inputFolderPath, lang: configLang, @@ -171,8 +235,8 @@ async function saveSinglePages() { toc?.root?.deepBase || toc?.deepBase || 0, ); - writeFileSync(singlePageFn, singlePageContent); - writeFileSync(singlePageDataFn, JSON.stringify(pageData)); + await fs.writeAsync(singlePageFn, singlePageContent); + await fs.writeAsync(singlePageDataFn, JSON.stringify(pageData)); }), ); } catch (error) { @@ -180,7 +244,7 @@ async function saveSinglePages() { } } -function saveRedirectPage(outputDir: string): void { +async function saveRedirectPage(fs: FsContext, outputDir: string): Promise { const {lang, langs} = ArgvService.getConfig(); const redirectLang = lang || langs?.[0] || Lang.RU; @@ -189,9 +253,9 @@ function saveRedirectPage(outputDir: string): void { const redirectPagePath = join(outputDir, 'index.html'); const redirectLangPath = join(outputDir, redirectLangRelativePath); - if (!existsSync(redirectPagePath) && existsSync(redirectLangPath)) { + if (!(await fs.existAsync(redirectPagePath)) && (await fs.existAsync(redirectLangPath))) { const content = generateStaticRedirect(redirectLang, redirectLangRelativePath); - writeFileSync(redirectPagePath, content); + await fs.writeAsync(redirectPagePath, content); } } @@ -245,10 +309,13 @@ function getMetaDataOptions(pathData: PathData, vcsConnector?: VCSConnector): Me } async function preparingPagesByOutputFormat( + fs: FsContext, + deps: DependencyContext, path: PathData, metaDataOptions: MetaDataOptions, resolveConditions: boolean, singlePage: boolean, + context: RevisionContext, ): Promise { const { filename, @@ -267,11 +334,11 @@ async function preparingPagesByOutputFormat( const isYamlFileExtension = fileExtension === '.yaml'; if (resolveConditions && fileBaseName === 'index' && isYamlFileExtension) { - LeadingService.filterFile(pathToFile); + await LeadingService.filterFile(pathToFile); } if (outputFormat === 'md' && isYamlFileExtension && allowCustomResources) { - processingYamlFile(path, metaDataOptions); + await processingYamlFile(fs, path, metaDataOptions); return; } @@ -279,16 +346,24 @@ async function preparingPagesByOutputFormat( (outputFormat === 'md' && isYamlFileExtension) || (outputFormat === 'html' && !isYamlFileExtension && fileExtension !== '.md') ) { - copyFileWithoutChanges(resolvedPathToFile, outputDir, filename); + await copyFileWithoutChanges(resolvedPathToFile, outputDir, filename); return; } + await addTocPresetsDeps(path, fs, deps); + switch (outputFormat) { case 'md': - await processingFileToMd(path, metaDataOptions); + await processingFileToMd(path, metaDataOptions, context, fs, deps); return; case 'html': { - const resolvedFileProps = await processingFileToHtml(path, metaDataOptions); + const resolvedFileProps = await processingFileToHtml( + path, + metaDataOptions, + context, + fs, + deps, + ); SearchService.add(resolvedFileProps); @@ -305,45 +380,77 @@ async function preparingPagesByOutputFormat( log.error(message); } } -//@ts-ignore -function processingYamlFile(path: PathData, metaDataOptions: MetaDataOptions) { + +async function addTocPresetsDeps(path: PathData, fs: FsContext, deps: DependencyContext) { + const {pathToFile} = path; + + const names = pathToFile.split(sep).filter((file) => !file.includes('.')); + + for (let index = names.length; index >= 1; index--) { + const dirs = names.slice(0, index); + const tocPath = resolve(...dirs, 'toc.yaml'); + const presetsPath = resolve(...dirs, 'presets.yaml'); + + if (await fs.existAsync(tocPath)) { + deps.markDep?.(pathToFile, tocPath, 'toc'); + } + + if (await fs.existAsync(presetsPath)) { + deps.markDep?.(pathToFile, presetsPath, 'presets'); + } + } +} + +async function processingYamlFile(fs: FsContext, path: PathData, metaDataOptions: MetaDataOptions) { const {pathToFile, outputFolderPath, inputFolderPath} = path; const filePath = resolve(inputFolderPath, pathToFile); - const content = readFileSync(filePath, 'utf8'); + const content = await fs.readAsync(filePath); const parsedContent = load(content) as LeadingPage; if (metaDataOptions.resources) { parsedContent.meta = {...parsedContent.meta, ...metaDataOptions.resources}; } - writeFileSync(resolve(outputFolderPath, pathToFile), dump(parsedContent)); + await fs.writeAsync(resolve(outputFolderPath, pathToFile), dump(parsedContent)); } -function copyFileWithoutChanges( +async function copyFileWithoutChanges( resolvedPathToFile: string, outputDir: string, filename: string, -): void { +) { const from = resolvedPathToFile; const to = resolve(outputDir, filename); shell.cp(from, to); } -async function processingFileToMd(path: PathData, metaDataOptions: MetaDataOptions): Promise { +async function processingFileToMd( + path: PathData, + metaDataOptions: MetaDataOptions, + context: RevisionContext, + fs: FsContext, + deps: DependencyContext, +): Promise { const {outputPath, pathToFile} = path; await resolveMd2Md({ inputPath: pathToFile, outputPath, metadata: metaDataOptions, + context, + fs, + deps, }); } async function processingFileToHtml( path: PathData, metaDataOptions: MetaDataOptions, + context: RevisionContext, + fs: FsContext, + deps: DependencyContext, ): Promise { const {outputBundlePath, filename, fileExtension, outputPath, pathToFile} = path; const {deepBase, deep} = TocService.getDeepForPath(pathToFile); @@ -357,5 +464,8 @@ async function processingFileToHtml( metadata: metaDataOptions, deep, deepBase, + context, + fs, + deps, }); } diff --git a/src/steps/processServiceFiles.ts b/src/steps/processServiceFiles.ts index a9aa7fcc..52dd7ddb 100644 --- a/src/steps/processServiceFiles.ts +++ b/src/steps/processServiceFiles.ts @@ -1,21 +1,22 @@ import {dirname, resolve} from 'path'; -import walkSync from 'walk-sync'; -import {readFileSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; import {ArgvService, PresetService, TocService} from '../services'; -import {logger} from '../utils'; +import {logger, walk} from '../utils'; import {DocPreset} from '../models'; import shell from 'shelljs'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {RevisionContext} from '~/context/context'; type GetFilePathsByGlobalsFunction = (globs: string[]) => string[]; -export async function processServiceFiles(): Promise { - const {input: inputFolderPath, ignore = []} = ArgvService.getConfig(); +export async function processServiceFiles(context: RevisionContext, fs: FsContext): Promise { + const {ignore} = ArgvService.getConfig(); const getFilePathsByGlobals = (globs: string[]): string[] => { - return walkSync(inputFolderPath, { + return walk({ + folder: [context.tmpInputFolder, context.userInputFolder], directories: false, includeBasePath: false, globs, @@ -23,11 +24,14 @@ export async function processServiceFiles(): Promise { }); }; - preparingPresetFiles(getFilePathsByGlobals); - await preparingTocFiles(getFilePathsByGlobals); + await preparingPresetFiles(fs, getFilePathsByGlobals); + await preparingTocFiles(fs, getFilePathsByGlobals); } -function preparingPresetFiles(getFilePathsByGlobals: GetFilePathsByGlobalsFunction): void { +async function preparingPresetFiles( + fs: FsContext, + getFilePathsByGlobals: GetFilePathsByGlobalsFunction, +) { const { input: inputFolderPath, varsPreset = '', @@ -43,14 +47,14 @@ function preparingPresetFiles(getFilePathsByGlobals: GetFilePathsByGlobalsFuncti logger.proc(path); const pathToPresetFile = resolve(inputFolderPath, path); - const content = readFileSync(pathToPresetFile, 'utf8'); + const content = await fs.readAsync(pathToPresetFile); const parsedPreset = load(content) as DocPreset; PresetService.add(parsedPreset, path, varsPreset); if (outputFormat === 'md' && (!applyPresets || !resolveConditions)) { // Should save filtered presets.yaml only when --apply-presets=false or --resolve-conditions=false - saveFilteredPresets(path, parsedPreset); + await saveFilteredPresets(fs, path, parsedPreset); } } } catch (error) { @@ -59,7 +63,11 @@ function preparingPresetFiles(getFilePathsByGlobals: GetFilePathsByGlobalsFuncti } } -function saveFilteredPresets(path: string, parsedPreset: DocPreset): void { +async function saveFilteredPresets( + fs: FsContext, + path: string, + parsedPreset: DocPreset, +): Promise { const {output: outputFolderPath, varsPreset = ''} = ArgvService.getConfig(); const outputPath = resolve(outputFolderPath, path); @@ -76,15 +84,16 @@ function saveFilteredPresets(path: string, parsedPreset: DocPreset): void { }); shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputPreset); + await fs.writeAsync(outputPath, outputPreset); } async function preparingTocFiles( + fs: FsContext, getFilePathsByGlobals: GetFilePathsByGlobalsFunction, ): Promise { try { const tocFilePaths = getFilePathsByGlobals(['**/toc.yaml']); - await TocService.init(tocFilePaths); + await TocService.init(fs, tocFilePaths); } catch (error) { log.error(`Preparing toc.yaml files failed. Error: ${error}`); throw error; diff --git a/src/utils/common.ts b/src/utils/common.ts index c2c3782c..e634c0c4 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -1,5 +1,6 @@ import {cloneDeepWith, flatMapDeep, isArray, isObject, isString} from 'lodash'; -import {isFileExists, resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {FsContext} from '@diplodoc/transform/lib/typings'; export function findAllValuesByKeys(obj: object, keysToFind: string[]) { return flatMapDeep(obj, (value: string | string[], key: string) => { @@ -21,10 +22,10 @@ export function findAllValuesByKeys(obj: object, keysToFind: string[]) { export function modifyValuesByKeys( originalObj: object, keysToFind: string[], - modifyFn: (value: string) => string, + modifyFn: (value: string) => string | undefined, ) { // Clone the object deeply with a customizer function that modifies matching keys - return cloneDeepWith(originalObj, function (value: unknown, key) { + return cloneDeepWith(originalObj, (value: unknown, key) => { if (keysToFind.includes(key as string) && isString(value)) { return modifyFn(value); } @@ -45,8 +46,8 @@ export function getLinksWithExtension(link: string) { return oneLineWithExtension.test(link); } -export function checkPathExists(path: string, parentFilePath: string) { +export async function checkPathExists(fs: FsContext, path: string, parentFilePath: string) { const includePath = resolveRelativePath(parentFilePath, path); - return isFileExists(includePath); + return fs.existAsync(includePath); } diff --git a/src/utils/file.ts b/src/utils/file.ts index c599f84f..d4a91ca7 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -1,26 +1,77 @@ import {dirname, resolve} from 'path'; +import {copyFile} from 'node:fs/promises'; import shell from 'shelljs'; +import walkSync from 'walk-sync'; +import {RevisionMeta} from '@diplodoc/transform/lib/typings'; import {logger} from './logger'; +import {Queue} from './queue'; -export function copyFiles( +const COPY_FILES_ACTIVE_QUEUE_LENGTH = 50; + +export async function copyFiles( inputFolderPath: string, outputFolderPath: string, files: string[], -): void { + meta?: RevisionMeta | null, +) { + if (files.length === 0) { + return; + } + const dirs = new Set(); - files.forEach((pathToAsset) => { - const outputDir = resolve(outputFolderPath, dirname(pathToAsset)); - const from = resolve(inputFolderPath, pathToAsset); - const to = resolve(outputFolderPath, pathToAsset); + const queue = new Queue( + async (pathToAsset: string) => { + const from = resolve(inputFolderPath, pathToAsset); + const to = resolve(outputFolderPath, pathToAsset); + const isChanged = meta?.files?.[pathToAsset]?.changed !== false; + + if (isChanged) { + const outputDir = resolve(outputFolderPath, dirname(pathToAsset)); + + if (!dirs.has(outputDir)) { + dirs.add(outputDir); + shell.mkdir('-p', outputDir); + } + + await copyFile(from, to); + logger.copy(pathToAsset); + } + }, + COPY_FILES_ACTIVE_QUEUE_LENGTH, + (error, pathToAsset) => logger.error(pathToAsset, error.message), + ); + + files.forEach(queue.add); + await queue.loop(); +} - if (!dirs.has(outputDir)) { - dirs.add(outputDir); - shell.mkdir('-p', outputDir); - } +export function walk({ + folder, + globs, + ignore, + directories, + includeBasePath, +}: { + folder?: string | string[]; + globs?: string[]; + ignore?: string[]; + directories?: boolean; + includeBasePath?: boolean; +}) { + if (!Array.isArray(folder) && folder) { + folder = [folder]; + } - shell.cp(from, to); + const dirs = [...(folder || [])].filter(Boolean) as string[]; + const files = dirs.map((folder) => + walkSync(folder as string, { + directories, + includeBasePath, + globs, + ignore, + }), + ); - logger.copy(pathToAsset); - }); + return [...new Set(files.flat())]; } diff --git a/src/utils/logger.ts b/src/utils/logger.ts index a94c3488..081f6840 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -1,5 +1,5 @@ import log from '@diplodoc/transform/lib/log'; -import {blue, green, grey, red, yellow} from 'chalk'; +import {blue, cyan, green, grey, red, yellow} from 'chalk'; import {ArgvService} from '../services'; function writeLog(msg: string, fatal = false) { @@ -16,6 +16,11 @@ export const logger = { info: function (pathToFile: string, extraMessage?: string) { writeLog(`${grey('INFO')} ${extraMessage} ${pathToFile}`); }, + prog: function (current: number, total: number, pathToFile: string) { + writeLog( + `${cyan('PROG')} Processing ${((current / total) * 100).toFixed()}% (${current} of ${total} files) for ${pathToFile}`, + ); + }, proc: function (pathToFile: string) { writeLog(`${blue('PROC')} Processing file ${pathToFile}`); }, diff --git a/src/utils/meta.ts b/src/utils/meta.ts new file mode 100644 index 00000000..901829a6 --- /dev/null +++ b/src/utils/meta.ts @@ -0,0 +1,119 @@ +import {resolve} from 'path'; +import {readFile, stat, unlink, writeFile} from 'node:fs/promises'; +import {logger} from './logger'; +import {Queue} from './queue'; +import {RevisionMeta} from '@diplodoc/transform/lib/typings'; + +const FILE_META_NAME = '.revision.meta.json'; +const META_ACTIVE_QUEUE_LENGTH = 50; + +export async function makeMetaFile(userOutputFolder: string, files: string[], meta: RevisionMeta) { + if (meta.files) { + for (const file of Object.keys(meta.files)) { + if (!files.includes(file)) { + delete meta.files[file]; + } + } + } + + const outputFile = resolve(userOutputFolder, FILE_META_NAME); + + try { + await unlink(outputFile); + } catch (error) { + // ignore + } + + await writeFile(outputFile, JSON.stringify(meta, null, 4), {encoding: 'utf8'}); +} + +export async function getMetaFile(userOutputFolder: string): Promise { + const outputFile = resolve(userOutputFolder, FILE_META_NAME); + + try { + return JSON.parse(await readFile(outputFile, 'utf8')); + } catch (_) { + return null; + } +} + +export async function updateMetaFile( + cached: boolean, + outputFolderPath: string, + metaFiles: RevisionMeta['files'], + files: string[], +) { + if (files.length) { + const queue = new Queue( + async (pathToAsset: string) => { + const from = resolve(outputFolderPath, pathToAsset); + + try { + const changed = !cached || !metaFiles[pathToAsset]; + const modDate = Number((await stat(from)).mtime); + metaFiles[pathToAsset] = { + modifyedDate: changed + ? modDate + : (metaFiles[pathToAsset]?.modifyedDate ?? modDate), + dependencies: metaFiles[pathToAsset]?.dependencies || {}, + changed, + }; + } catch (error) { + // ignore + } + }, + META_ACTIVE_QUEUE_LENGTH, + (error, pathToAsset) => logger.error(pathToAsset, error.message), + ); + + files.forEach(queue.add); + + await queue.loop(); + } +} + +export async function updateChangedMetaFile( + cached: boolean, + inputFolderPath: string, + metaFiles: RevisionMeta['files'], +) { + const files = Object.keys(metaFiles); + + if (files.length) { + const queue = new Queue( + async (pathToAsset: string) => { + if (metaFiles[pathToAsset] && !metaFiles[pathToAsset].changed) { + const from = resolve(inputFolderPath, pathToAsset); + const modDateNullable = await getFileModifiedDate(from); + const modDate = modDateNullable ?? metaFiles[pathToAsset].modifyedDate; + + metaFiles[pathToAsset].changed = + !cached || + !modDateNullable || + isFileModified(modDate, metaFiles[pathToAsset].modifyedDate); + metaFiles[pathToAsset].modifyedDate = modDate; + } + }, + META_ACTIVE_QUEUE_LENGTH, + (error, pathToAsset) => logger.error(pathToAsset, error.message), + ); + + files.forEach(queue.add); + + await queue.loop(); + } +} + +async function getFileModifiedDate(from: string) { + try { + const data = await stat(from); + const folderLMM = Number(data.mtime); + return folderLMM; + } catch (_) { + return null; + } +} + +function isFileModified(newModDate: number, oldModDate: number) { + return Math.abs(newModDate - oldModDate) > 1000; +} diff --git a/src/utils/queue.ts b/src/utils/queue.ts new file mode 100644 index 00000000..f19311b1 --- /dev/null +++ b/src/utils/queue.ts @@ -0,0 +1,102 @@ +// Queue is the system that processes async tasks in parallel but limits them by paralleledTasks. +// For example, we have 120 files to process. If the limit is 50, and time to process 1 file takes 1 minute, then the total time will take 3 minutes. +// The files will be groupped by 50 tasks in memory, so 50 + 50 + 20 ~> 1 + 1 + 1 = 3 minutes. + +/* eslint-disable @typescript-eslint/no-explicit-any */ +export class Queue { + stack = new Set(); + queue: unknown[][] = []; + promise: Promise | null = null; + promiseResolve: (() => void) | null = null; + promiseFinish: Promise | null = null; + promiseFinishResolve: (() => void) | null = null; + + processTask: (...args: unknown[]) => Promise; + paralleledTasks: number; + whenEmpty?: () => void; + whenError?: (e: Error, ...args: unknown[]) => void; + + constructor( + processTask: (...args: any[]) => Promise, + paralleledTasks: number, + whenError?: (error: Error, ...args: any[]) => void, + whenEmpty?: () => void, + ) { + this.processTask = processTask; + this.paralleledTasks = paralleledTasks; + this.whenEmpty = whenEmpty; + this.whenError = whenError; + } + + add = (...args: unknown[]) => { + this.queue.push(args); + }; + + loop = async () => { + for await (const _ of this.getTasks()) { + // Process tasks + } + await this.promiseFinish; + this.whenEmpty?.(); + }; + + private canDryStack() { + return this.stack.size < this.paralleledTasks; + } + + private canFinish() { + return this.stack.size === 0 && this.queue.length === 0; + } + + private createPromiseForStack() { + if (!this.canDryStack()) { + if (!this.promise) { + this.promise = new Promise((r) => { + this.promiseResolve = r; + }); + } + } + return this.promise; + } + + private createPromiseFinish() { + if (!this.promiseFinish) { + this.promiseFinish = new Promise((r) => { + this.promiseFinishResolve = r; + }); + } + } + + private checkStack() { + if (this.promise && this.canDryStack()) { + this.promiseResolve?.(); + this.promise = null; + } + + if (this.promiseFinish && this.canFinish()) { + this.promiseFinishResolve?.(); + this.promiseFinish = null; + } + } + + private async *getTasks() { + this.createPromiseFinish(); + while (this.queue.length > 0) { + const task = this.queue.shift(); + if (task !== null && task !== undefined) { + this.stack.add(task); + try { + this.processTask(...task) + .catch(this.whenError) + .finally(() => { + this.stack.delete(task); + this.checkStack(); + }); + } catch (error) { + this.whenError?.(error as Error, ...task); + } + yield await this.createPromiseForStack(); + } + } + } +} diff --git a/src/workers/linter/index.ts b/src/workers/linter/index.ts index 969ee487..9d681949 100644 --- a/src/workers/linter/index.ts +++ b/src/workers/linter/index.ts @@ -3,11 +3,12 @@ import {extname} from 'path'; import {Observable, Subject} from 'threads/observable'; import {expose} from 'threads'; -import {ArgvService, PluginService, PresetService, TocService} from '../../services'; -import {TocServiceData} from '../../services/tocs'; -import {PresetStorage} from '../../services/preset'; -import {YfmArgv} from '../../models'; -import {lintPage} from '../../resolvers'; +import {ArgvService, PluginService, PresetService, TocService} from '~/services'; +import {TocServiceData} from '~/services/tocs'; +import {PresetStorage} from '~/services/preset'; +import {YfmArgv} from '~/models'; +import {lintPage} from '~/resolvers'; +import {RevisionContext} from '~/context/context'; let processedPages = new Subject(); @@ -15,9 +16,15 @@ interface ProcessLinterWorkerOptions { argvConfig: YfmArgv; navigationPaths: TocServiceData['navigationPaths']; presetStorage: PresetStorage; + context: RevisionContext; } -async function run({argvConfig, presetStorage, navigationPaths}: ProcessLinterWorkerOptions) { +async function run({ + argvConfig, + presetStorage, + navigationPaths, + context, +}: ProcessLinterWorkerOptions) { ArgvService.set(argvConfig); PresetService.setPresetStorage(presetStorage); TocService.setNavigationPaths(navigationPaths); @@ -27,6 +34,7 @@ async function run({argvConfig, presetStorage, navigationPaths}: ProcessLinterWo lintPage({ inputPath: pathToFile, fileExtension: extname(pathToFile), + context, onFinish: () => { processedPages.next(pathToFile); }, diff --git a/tests/e2e/__snapshots__/include-toc.test.ts.snap b/tests/e2e/__snapshots__/include-toc.test.ts.snap index ec4d957b..cb02164f 100644 --- a/tests/e2e/__snapshots__/include-toc.test.ts.snap +++ b/tests/e2e/__snapshots__/include-toc.test.ts.snap @@ -2,6 +2,7 @@ exports[`Include toc Nested toc inclusions with mixed including modes 1`] = ` "[ + ".revision.meta.json", "product1/_includes/inc.md", "product1/article1.md", "product1/toc.yaml", @@ -128,6 +129,7 @@ deepBase: 1 exports[`Include toc Toc is included in link mode 1`] = ` "[ + ".revision.meta.json", "a1.md", "folder1/a1.md", "folder1/folder2/a1.md", @@ -169,6 +171,7 @@ deepBase: 0 exports[`Include toc Toc is included inline, not as a new section 1`] = ` "[ + ".revision.meta.json", ".yfm", "file1.md", "fileA.md", @@ -298,6 +301,7 @@ deepBase: 0 exports[`Include toc Toc with expressions 1`] = ` "[ + ".revision.meta.json", "a1.md", "index.yaml", "toc.yaml" diff --git a/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap b/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap index ccd07f04..70320f6e 100644 --- a/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap +++ b/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap @@ -2,6 +2,7 @@ exports[`Allow load custom resources md2html single page with custom resources 1`] = ` "[ + ".revision.meta.json", "_assets/script/test1.js", "_assets/style/test.css", "_bundle/search-async-0", @@ -306,6 +307,7 @@ exports[`Allow load custom resources md2html single page with custom resources 6 exports[`Allow load custom resources md2html with custom resources 1`] = ` "[ + ".revision.meta.json", "_assets/script/test1.js", "_assets/style/test.css", "_bundle/search-async-0", @@ -543,6 +545,7 @@ exports[`Allow load custom resources md2html with custom resources 4`] = ` exports[`Allow load custom resources md2md with custom resources 1`] = ` "[ + ".revision.meta.json", ".yfm", "_assets/script/test1.js", "_assets/style/test.css", diff --git a/tests/e2e/__snapshots__/metadata.spec.ts.snap b/tests/e2e/__snapshots__/metadata.spec.ts.snap index f831258c..19636778 100644 --- a/tests/e2e/__snapshots__/metadata.spec.ts.snap +++ b/tests/e2e/__snapshots__/metadata.spec.ts.snap @@ -2,10 +2,10 @@ exports[`Allow load custom resources md2html with metadata 1`] = ` "[ + ".revision.meta.json", "_bundle/search-async-0", "_bundle/app-css-1", "_bundle/app-js-1", - "_bundle/search-css-2", "_bundle/search-js-0", "_bundle/search-js-1", "_bundle/search/index.js", @@ -41,7 +41,6 @@ exports[`Allow load custom resources md2html with metadata 1`] = ` "_bundle/search/langs/vi.js", "_bundle/search-css-0", "_bundle/search-js-2", - "_bundle/search-css-1", "index.html", "page.html", "project/config.html" @@ -237,6 +236,7 @@ exports[`Allow load custom resources md2html with metadata 4`] = ` exports[`Allow load custom resources md2md with metadata 1`] = ` "[ + ".revision.meta.json", "index.yaml", "page.md", "project/config.md", diff --git a/tests/e2e/__snapshots__/rtl.spec.ts.snap b/tests/e2e/__snapshots__/rtl.spec.ts.snap index c96770fd..dd105b2d 100644 --- a/tests/e2e/__snapshots__/rtl.spec.ts.snap +++ b/tests/e2e/__snapshots__/rtl.spec.ts.snap @@ -2,6 +2,7 @@ exports[`Generate html document with correct lang and dir attributes. Load correct bundles. documentation with only one rtl lang 1`] = ` "[ + ".revision.meta.json", "_bundle/search-async-0", "_bundle/app-css-1", "_bundle/app-js-1", @@ -153,6 +154,7 @@ exports[`Generate html document with correct lang and dir attributes. Load corre exports[`Generate html document with correct lang and dir attributes. Load correct bundles. documentation with rtl and ltr langs 1`] = ` "[ + ".revision.meta.json", "_bundle/search-async-0", "_bundle/app-css-1", "_bundle/app-js-1", diff --git a/tests/utils.ts b/tests/utils.ts index beba3524..7c483e0b 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -33,13 +33,14 @@ export function getFileContent(filePath: string) { return bundleless(platformless(readFileSync(filePath, 'utf8'))); } -const uselessFile = (file) => !['_bundle/', '_assets/'].some(part => file.includes(part)); +const uselessFile = (file) => !['_bundle/', '_assets/', '.revision.meta.json'].some(part => file.includes(part)); export function compareDirectories(outputPath: string) { const filesFromOutput = walkSync(outputPath, { directories: false, includeBasePath: false, - }); + }) + .sort(); expect(bundleless(JSON.stringify(filesFromOutput, null, 2))).toMatchSnapshot();