From 3c3af094bd59f1ab98fd86267c7c403966de55da Mon Sep 17 00:00:00 2001 From: Maxim Karpov Date: Wed, 23 Oct 2024 10:45:03 +0300 Subject: [PATCH] feat: new build pipeline --- package-lock.json | 371 ++++++++++++------ package.json | 2 +- src/cmd/build/index.ts | 120 ++++-- src/commands/publish/upload.ts | 5 +- src/context/context.ts | 59 +++ src/context/dependency.ts | 62 +++ src/context/fs.ts | 107 +++++ src/context/processor.ts | 89 +++++ src/models.ts | 22 +- src/pages/document.ts | 69 +++- src/resolvers/lintPage.ts | 46 ++- src/resolvers/md2html.ts | 49 ++- src/resolvers/md2md.ts | 66 ++-- src/services/plugins.ts | 23 +- src/services/tocs.ts | 33 +- src/steps/processAssets.ts | 102 ++--- src/steps/processExcludedFiles.ts | 14 +- src/steps/processLinter.ts | 72 +++- src/steps/processPages.ts | 165 ++++++-- src/steps/processServiceFiles.ts | 33 +- src/utils/common.ts | 11 +- src/utils/file.ts | 77 +++- src/utils/meta.ts | 117 ++++++ src/utils/queue.ts | 94 +++++ src/workers/linter/index.ts | 20 +- .../__snapshots__/include-toc.test.ts.snap | 129 +----- .../load-custom-resources.spec.ts.snap | 39 +- tests/e2e/__snapshots__/metadata.spec.ts.snap | 4 +- tests/e2e/__snapshots__/rtl.spec.ts.snap | 2 + tests/e2e/include-toc.test.ts | 10 +- tests/utils.ts | 10 +- 31 files changed, 1477 insertions(+), 545 deletions(-) create mode 100644 src/context/context.ts create mode 100644 src/context/dependency.ts create mode 100644 src/context/fs.ts create mode 100644 src/context/processor.ts create mode 100644 src/utils/meta.ts create mode 100644 src/utils/queue.ts diff --git a/package-lock.json b/package-lock.json index 69fa5950..a365fbc5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,7 +28,7 @@ "@diplodoc/mermaid-extension": "^1.3.1", "@diplodoc/openapi-extension": "^2.4.1", "@diplodoc/prettier-config": "^2.0.0", - "@diplodoc/transform": "^4.32.3", + "@diplodoc/transform": "0.0.0-rc-add-file-meta-202410220915", "@diplodoc/tsconfig": "^1.0.2", "@octokit/core": "4.2.4", "@types/async": "^3.2.15", @@ -1730,12 +1730,54 @@ "npm": ">=9.*" } }, + "node_modules/@diplodoc/client/node_modules/@diplodoc/transform": { + "version": "4.32.4", + "resolved": "https://registry.npmjs.org/@diplodoc/transform/-/transform-4.32.4.tgz", + "integrity": "sha512-hhR7KsvXoxhH2TtN4J1vdLf57vs74ea17WPMdqmDRKLTsWP9ooISXieJSSESt51mHde7BX3CsHTmzkE9GMtPWw==", + "dependencies": { + "@diplodoc/cut-extension": "^0.3.0", + "@diplodoc/tabs-extension": "^3.5.0", + "chalk": "^4.1.2", + "cheerio": "^1.0.0", + "css": "^3.0.0", + "cssfilter": "0.0.10", + "get-root-node-polyfill": "1.0.0", + "github-slugger": "^1.5.0", + "js-yaml": "^4.1.0", + "lodash": "4.17.21", + "markdown-it": "^13.0.2", + "markdown-it-attrs": "^4.2.0", + "markdown-it-deflist": "2.1.0", + "markdown-it-meta": "0.0.1", + "markdown-it-sup": "1.0.0", + "markdownlint": "^0.32.1", + "markdownlint-rule-helpers": "0.17.2", + "sanitize-html": "^2.11.0", + "slugify": "1.6.5", + "svgo": "^3.2.0" + }, + "peerDependencies": { + "highlight.js": "^10.0.3 || ^11" + }, + "peerDependenciesMeta": { + "highlight.js": { + "optional": true + } + } + }, + "node_modules/@diplodoc/client/node_modules/slugify": { + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.5.tgz", + "integrity": "sha512-8mo9bslnBO3tr5PEVFzMPIWwWnipGS0xVbYf65zxDqfNwmzYn1LpiKNrR6DlClusuvo+hDHd1zKpmfAe83NQSQ==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@diplodoc/components": { "version": "4.15.4", "resolved": "https://registry.npmjs.org/@diplodoc/components/-/components-4.15.4.tgz", "integrity": "sha512-i15G2jfmEqRNOXuoiQ6wuUftN4yhOcMDmxTLuh2HYX65/3nh6ByKcQgV2XIETjJorgv/syFA+2mZfWJIszvQaQ==", "hasInstallScript": true, - "license": "MIT", "peer": true, "dependencies": { "@gravity-ui/components": "^3.6.0", @@ -1765,6 +1807,121 @@ "react-dom": ">=16.8.0 || >=17.0.0 || >=18.0.0" } }, + "node_modules/@diplodoc/components/node_modules/@diplodoc/transform": { + "version": "4.32.4", + "resolved": "https://registry.npmjs.org/@diplodoc/transform/-/transform-4.32.4.tgz", + "integrity": "sha512-hhR7KsvXoxhH2TtN4J1vdLf57vs74ea17WPMdqmDRKLTsWP9ooISXieJSSESt51mHde7BX3CsHTmzkE9GMtPWw==", + "peer": true, + "dependencies": { + "@diplodoc/cut-extension": "^0.3.0", + "@diplodoc/tabs-extension": "^3.5.0", + "chalk": "^4.1.2", + "cheerio": "^1.0.0", + "css": "^3.0.0", + "cssfilter": "0.0.10", + "get-root-node-polyfill": "1.0.0", + "github-slugger": "^1.5.0", + "js-yaml": "^4.1.0", + "lodash": "4.17.21", + "markdown-it": "^13.0.2", + "markdown-it-attrs": "^4.2.0", + "markdown-it-deflist": "2.1.0", + "markdown-it-meta": "0.0.1", + "markdown-it-sup": "1.0.0", + "markdownlint": "^0.32.1", + "markdownlint-rule-helpers": "0.17.2", + "sanitize-html": "^2.11.0", + "slugify": "1.6.5", + "svgo": "^3.2.0" + }, + "peerDependencies": { + "highlight.js": "^10.0.3 || ^11" + }, + "peerDependenciesMeta": { + "highlight.js": { + "optional": true + } + } + }, + "node_modules/@diplodoc/components/node_modules/@gravity-ui/page-constructor": { + "version": "5.28.1", + "resolved": "https://registry.npmjs.org/@gravity-ui/page-constructor/-/page-constructor-5.28.1.tgz", + "integrity": "sha512-MNnkJYIcrPI0AmPVTA2rlcMowjLERnBqGTEkrSOvZHSaiixyPX5lgGD/gCGXAvkwXqJChg4Y6WoEOcV22Fsqvg==", + "peer": true, + "dependencies": { + "@bem-react/classname": "^1.6.0", + "@gravity-ui/components": "^3.8.0", + "@gravity-ui/dynamic-forms": "^4.11.0", + "@gravity-ui/i18n": "^1.3.0", + "@react-spring/web": "^9.7.3", + "ajv": "^8.12.0", + "ajv-keywords": "^5.1.0", + "final-form": "^4.20.9", + "github-buttons": "2.23.0", + "js-yaml-source-map": "^0.2.2", + "lodash": "^4.17.21", + "monaco-editor": "^0.38.0", + "react-final-form": "^6.5.9", + "react-monaco-editor": "^0.53.0", + "react-player": "^2.9.0", + "react-slick": "^0.29.0", + "react-transition-group": "^4.4.2", + "react-waypoint": "^10.1.0", + "sanitize-html": "2.12.1", + "snakecase-keys": "^5.1.0", + "swiper": "^6.8.4", + "typograf": "^6.14.0", + "utility-types": "^3.10.0", + "uuid": "^9.0.0" + }, + "peerDependencies": { + "@diplodoc/transform": "^4.10.4", + "@gravity-ui/uikit": "^6.0.0", + "react": "^16.0.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/@diplodoc/components/node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "peer": true, + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/@diplodoc/components/node_modules/sanitize-html": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.12.1.tgz", + "integrity": "sha512-Plh+JAn0UVDpBRP/xEjsk+xDCoOvMBwQUf/K+/cBAVuTbtX8bj2VB7S1sL1dssVpykqp0/KPSesHrqXtokVBpA==", + "peer": true, + "dependencies": { + "deepmerge": "^4.2.2", + "escape-string-regexp": "^4.0.0", + "htmlparser2": "^8.0.0", + "is-plain-object": "^5.0.0", + "parse-srcset": "^1.0.2", + "postcss": "^8.3.11" + } + }, + "node_modules/@diplodoc/components/node_modules/slugify": { + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.5.tgz", + "integrity": "sha512-8mo9bslnBO3tr5PEVFzMPIWwWnipGS0xVbYf65zxDqfNwmzYn1LpiKNrR6DlClusuvo+hDHd1zKpmfAe83NQSQ==", + "peer": true, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@diplodoc/cut-extension": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@diplodoc/cut-extension/-/cut-extension-0.3.0.tgz", @@ -2352,6 +2509,51 @@ "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/@diplodoc/openapi-extension/node_modules/@diplodoc/transform": { + "version": "4.32.4", + "resolved": "https://registry.npmjs.org/@diplodoc/transform/-/transform-4.32.4.tgz", + "integrity": "sha512-hhR7KsvXoxhH2TtN4J1vdLf57vs74ea17WPMdqmDRKLTsWP9ooISXieJSSESt51mHde7BX3CsHTmzkE9GMtPWw==", + "dev": true, + "dependencies": { + "@diplodoc/cut-extension": "^0.3.0", + "@diplodoc/tabs-extension": "^3.5.0", + "chalk": "^4.1.2", + "cheerio": "^1.0.0", + "css": "^3.0.0", + "cssfilter": "0.0.10", + "get-root-node-polyfill": "1.0.0", + "github-slugger": "^1.5.0", + "js-yaml": "^4.1.0", + "lodash": "4.17.21", + "markdown-it": "^13.0.2", + "markdown-it-attrs": "^4.2.0", + "markdown-it-deflist": "2.1.0", + "markdown-it-meta": "0.0.1", + "markdown-it-sup": "1.0.0", + "markdownlint": "^0.32.1", + "markdownlint-rule-helpers": "0.17.2", + "sanitize-html": "^2.11.0", + "slugify": "1.6.5", + "svgo": "^3.2.0" + }, + "peerDependencies": { + "highlight.js": "^10.0.3 || ^11" + }, + "peerDependenciesMeta": { + "highlight.js": { + "optional": true + } + } + }, + "node_modules/@diplodoc/openapi-extension/node_modules/@diplodoc/transform/node_modules/slugify": { + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.5.tgz", + "integrity": "sha512-8mo9bslnBO3tr5PEVFzMPIWwWnipGS0xVbYf65zxDqfNwmzYn1LpiKNrR6DlClusuvo+hDHd1zKpmfAe83NQSQ==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@diplodoc/prettier-config": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@diplodoc/prettier-config/-/prettier-config-2.0.0.tgz", @@ -2400,9 +2602,10 @@ } }, "node_modules/@diplodoc/transform": { - "version": "4.32.3", - "resolved": "https://registry.npmjs.org/@diplodoc/transform/-/transform-4.32.3.tgz", - "integrity": "sha512-QJqRAUKRYjH4k//gFQNHDJVX4xsIbUuRzQKjsR9o3MzHzRJWDVe0xNm3JixEDUB7PyupYdMtDHn5MMg7pXGpFw==", + "version": "0.0.0-rc-add-file-meta-202410220915", + "resolved": "https://registry.npmjs.org/@diplodoc/transform/-/transform-0.0.0-rc-add-file-meta-202410220915.tgz", + "integrity": "sha512-52ST27vH1QbT3xNhi+5De77Wod3ab1Zp+k6FC1mE0to23pRwrfO/Ibo4/qqK/iI8+aUkIeHKcdTkNA5/Shs9Mw==", + "dev": true, "dependencies": { "@diplodoc/cut-extension": "^0.3.0", "@diplodoc/tabs-extension": "^3.5.0", @@ -2438,7 +2641,7 @@ "version": "1.6.5", "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.5.tgz", "integrity": "sha512-8mo9bslnBO3tr5PEVFzMPIWwWnipGS0xVbYf65zxDqfNwmzYn1LpiKNrR6DlClusuvo+hDHd1zKpmfAe83NQSQ==", - "license": "MIT", + "dev": true, "engines": { "node": ">=8.0.0" } @@ -2478,12 +2681,60 @@ "markdown-it": "^13.0.2" } }, + "node_modules/@diplodoc/translation/node_modules/@diplodoc/transform": { + "version": "4.32.4", + "resolved": "https://registry.npmjs.org/@diplodoc/transform/-/transform-4.32.4.tgz", + "integrity": "sha512-hhR7KsvXoxhH2TtN4J1vdLf57vs74ea17WPMdqmDRKLTsWP9ooISXieJSSESt51mHde7BX3CsHTmzkE9GMtPWw==", + "dependencies": { + "@diplodoc/cut-extension": "^0.3.0", + "@diplodoc/tabs-extension": "^3.5.0", + "chalk": "^4.1.2", + "cheerio": "^1.0.0", + "css": "^3.0.0", + "cssfilter": "0.0.10", + "get-root-node-polyfill": "1.0.0", + "github-slugger": "^1.5.0", + "js-yaml": "^4.1.0", + "lodash": "4.17.21", + "markdown-it": "^13.0.2", + "markdown-it-attrs": "^4.2.0", + "markdown-it-deflist": "2.1.0", + "markdown-it-meta": "0.0.1", + "markdown-it-sup": "1.0.0", + "markdownlint": "^0.32.1", + "markdownlint-rule-helpers": "0.17.2", + "sanitize-html": "^2.11.0", + "slugify": "1.6.5", + "svgo": "^3.2.0" + }, + "peerDependencies": { + "highlight.js": "^10.0.3 || ^11" + }, + "peerDependenciesMeta": { + "highlight.js": { + "optional": true + } + } + }, + "node_modules/@diplodoc/translation/node_modules/@diplodoc/transform/node_modules/markdown-it-deflist": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/markdown-it-deflist/-/markdown-it-deflist-2.1.0.tgz", + "integrity": "sha512-3OuqoRUlSxJiuQYu0cWTLHNhhq2xtoSFqsZK8plANg91+RJQU1ziQ6lA2LzmFAEes18uPBsHZpcX6We5l76Nzg==" + }, "node_modules/@diplodoc/translation/node_modules/markdown-it-deflist": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/markdown-it-deflist/-/markdown-it-deflist-3.0.0.tgz", "integrity": "sha512-OxPmQ/keJZwbubjiQWOvKLHwpV2wZ5I3Smc81OjhwbfJsjdRrvD5aLTQxmZzzePeO0kbGzAo3Krk4QLgA8PWLg==", "license": "MIT" }, + "node_modules/@diplodoc/translation/node_modules/slugify": { + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.5.tgz", + "integrity": "sha512-8mo9bslnBO3tr5PEVFzMPIWwWnipGS0xVbYf65zxDqfNwmzYn1LpiKNrR6DlClusuvo+hDHd1zKpmfAe83NQSQ==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@diplodoc/tsconfig": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@diplodoc/tsconfig/-/tsconfig-1.0.2.tgz", @@ -2679,7 +2930,6 @@ "version": "2.10.2", "resolved": "https://registry.npmjs.org/@gravity-ui/date-components/-/date-components-2.10.2.tgz", "integrity": "sha512-QMjFdoB3b7GwerytmqxA/pXCokujNTD6/yD+h4ER+jN/aUSTmVDMkYyiPQJ507SJ3U0qW9erVu4kpBZ0QOt/gg==", - "license": "MIT", "peer": true, "dependencies": { "@bem-react/classname": "^1.6.0", @@ -2708,7 +2958,6 @@ "version": "4.14.0", "resolved": "https://registry.npmjs.org/@gravity-ui/dynamic-forms/-/dynamic-forms-4.14.0.tgz", "integrity": "sha512-bv83s62hJbMAgKyO2g7lHN7ceCF8VlTppj0I8y/14OsUQm718XHvNIkc0GQv6kSZ9iHqNhOfhCH8zp0Qgs1reA==", - "license": "MIT", "peer": true, "dependencies": { "@bem-react/classname": "^1.6.0", @@ -2779,79 +3028,6 @@ } } }, - "node_modules/@gravity-ui/page-constructor": { - "version": "5.28.1", - "resolved": "https://registry.npmjs.org/@gravity-ui/page-constructor/-/page-constructor-5.28.1.tgz", - "integrity": "sha512-MNnkJYIcrPI0AmPVTA2rlcMowjLERnBqGTEkrSOvZHSaiixyPX5lgGD/gCGXAvkwXqJChg4Y6WoEOcV22Fsqvg==", - "license": "MIT", - "peer": true, - "dependencies": { - "@bem-react/classname": "^1.6.0", - "@gravity-ui/components": "^3.8.0", - "@gravity-ui/dynamic-forms": "^4.11.0", - "@gravity-ui/i18n": "^1.3.0", - "@react-spring/web": "^9.7.3", - "ajv": "^8.12.0", - "ajv-keywords": "^5.1.0", - "final-form": "^4.20.9", - "github-buttons": "2.23.0", - "js-yaml-source-map": "^0.2.2", - "lodash": "^4.17.21", - "monaco-editor": "^0.38.0", - "react-final-form": "^6.5.9", - "react-monaco-editor": "^0.53.0", - "react-player": "^2.9.0", - "react-slick": "^0.29.0", - "react-transition-group": "^4.4.2", - "react-waypoint": "^10.1.0", - "sanitize-html": "2.12.1", - "snakecase-keys": "^5.1.0", - "swiper": "^6.8.4", - "typograf": "^6.14.0", - "utility-types": "^3.10.0", - "uuid": "^9.0.0" - }, - "peerDependencies": { - "@diplodoc/transform": "^4.10.4", - "@gravity-ui/uikit": "^6.0.0", - "react": "^16.0.0 || ^17.0.0 || ^18.0.0" - } - }, - "node_modules/@gravity-ui/page-constructor/node_modules/htmlparser2": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", - "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "peer": true, - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.0.1", - "entities": "^4.4.0" - } - }, - "node_modules/@gravity-ui/page-constructor/node_modules/sanitize-html": { - "version": "2.12.1", - "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.12.1.tgz", - "integrity": "sha512-Plh+JAn0UVDpBRP/xEjsk+xDCoOvMBwQUf/K+/cBAVuTbtX8bj2VB7S1sL1dssVpykqp0/KPSesHrqXtokVBpA==", - "license": "MIT", - "peer": true, - "dependencies": { - "deepmerge": "^4.2.2", - "escape-string-regexp": "^4.0.0", - "htmlparser2": "^8.0.0", - "is-plain-object": "^5.0.0", - "parse-srcset": "^1.0.2", - "postcss": "^8.3.11" - } - }, "node_modules/@gravity-ui/prettier-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@gravity-ui/prettier-config/-/prettier-config-1.1.0.tgz", @@ -3253,7 +3429,6 @@ "version": "9.7.5", "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.5.tgz", "integrity": "sha512-Tqrwz7pIlsSDITzxoLS3n/v/YCUHQdOIKtOJf4yL6kYVSDTSmVK1LI1Q3M/uu2Sx4X3pIWF3xLUhlsA6SPNTNg==", - "license": "MIT", "peer": true, "dependencies": { "@react-spring/shared": "~9.7.5", @@ -3267,7 +3442,6 @@ "version": "9.7.5", "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.5.tgz", "integrity": "sha512-rmEqcxRcu7dWh7MnCcMXLvrf6/SDlSokLaLTxiPlAYi11nN3B5oiCUAblO72o+9z/87j2uzxa2Inm8UbLjXA+w==", - "license": "MIT", "peer": true, "dependencies": { "@react-spring/animated": "~9.7.5", @@ -3286,14 +3460,12 @@ "version": "9.7.5", "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.7.5.tgz", "integrity": "sha512-5ZenDQMC48wjUzPAm1EtwQ5Ot3bLIAwwqP2w2owG5KoNdNHpEJV263nGhCeKKmuA3vG2zLLOdu3or6kuDjA6Aw==", - "license": "MIT", "peer": true }, "node_modules/@react-spring/shared": { "version": "9.7.5", "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.5.tgz", "integrity": "sha512-wdtoJrhUeeyD/PP/zo+np2s1Z820Ohr/BbuVYv+3dVLW7WctoiN7std8rISoYoHpUXtbkpesSKuPIw/6U1w1Pw==", - "license": "MIT", "peer": true, "dependencies": { "@react-spring/rafz": "~9.7.5", @@ -3307,14 +3479,12 @@ "version": "9.7.5", "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.5.tgz", "integrity": "sha512-HVj7LrZ4ReHWBimBvu2SKND3cDVUPWKLqRTmWe/fNY6o1owGOX0cAHbdPDTMelgBlVbrTKrre6lFkhqGZErK/g==", - "license": "MIT", "peer": true }, "node_modules/@react-spring/web": { "version": "9.7.5", "resolved": "https://registry.npmjs.org/@react-spring/web/-/web-9.7.5.tgz", "integrity": "sha512-lmvqGwpe+CSttsWNZVr+Dg62adtKhauGwLyGE/RRyZ8AAMLgb9x3NDMA5RMElXo+IMyTkPp7nxTB8ZQlmhb6JQ==", - "license": "MIT", "peer": true, "dependencies": { "@react-spring/animated": "~9.7.5", @@ -4864,7 +5034,6 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", - "license": "MIT", "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3" @@ -5681,7 +5850,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/consolidated-events/-/consolidated-events-2.0.2.tgz", "integrity": "sha512-2/uRVMdRypf5z/TW/ncD/66l75P5hH2vM/GR8Jf8HLc2xnfJtmina6F6du8+v4Z2vTrMo7jC+W1tmEEuuELgkQ==", - "license": "MIT", "peer": true }, "node_modules/convert-source-map": { @@ -6724,7 +6892,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/dom7/-/dom7-3.0.0.tgz", "integrity": "sha512-oNlcUdHsC4zb7Msx7JN3K0Nro1dzJ48knvBOnDPKJ2GV9wl1i5vydJZUSyOfrkKFDZEud/jBsTk92S/VGSAe/g==", - "license": "MIT", "peer": true, "dependencies": { "ssr-window": "^3.0.0-alpha.1" @@ -6781,7 +6948,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz", "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==", - "license": "MIT", "peer": true, "dependencies": { "no-case": "^3.0.4", @@ -6843,7 +7009,6 @@ "version": "2.1.6", "resolved": "https://registry.npmjs.org/enquire.js/-/enquire.js-2.1.6.tgz", "integrity": "sha512-/KujNpO+PT63F7Hlpu4h3pE3TokKRHN26JYmQpPyjkRD/N57R7bPDNojMXdi7uveAKjYB7yQnartCxZnFWr0Xw==", - "license": "MIT", "peer": true }, "node_modules/ensure-posix-path": { @@ -7931,7 +8096,6 @@ "version": "4.20.10", "resolved": "https://registry.npmjs.org/final-form/-/final-form-4.20.10.tgz", "integrity": "sha512-TL48Pi1oNHeMOHrKv1bCJUrWZDcD3DIG6AGYVNOnyZPr7Bd/pStN0pL+lfzF5BNoj/FclaoiaLenk4XUIFVYng==", - "license": "MIT", "peer": true, "dependencies": { "@babel/runtime": "^7.10.0" @@ -8188,7 +8352,6 @@ "version": "2.23.0", "resolved": "https://registry.npmjs.org/github-buttons/-/github-buttons-2.23.0.tgz", "integrity": "sha512-2REUOV3ue6NmT0QThhfzfYmeSoYpCG73+tL7Ir2C7P+gshRerI05WuIQuhDkE2Zlg5Wc39hc2DHj+pE23mGJvw==", - "license": "BSD-2-Clause", "peer": true }, "node_modules/github-slugger": { @@ -9285,7 +9448,6 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/js-yaml-source-map/-/js-yaml-source-map-0.2.2.tgz", "integrity": "sha512-z45Aww8oXJh9GuWUnwmvHsAkB7I/oWrkoHU554UQ8Ik4dyhVrk/nwClTI435feU7QIy7E0XaW8jHvZ4QxaAjog==", - "license": "MIT", "peer": true, "peerDependencies": { "js-yaml": "^4.0.0" @@ -9344,7 +9506,6 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/json2mq/-/json2mq-0.2.0.tgz", "integrity": "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==", - "license": "MIT", "peer": true, "dependencies": { "string-convert": "^0.2.0" @@ -9660,7 +9821,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/load-script/-/load-script-1.0.0.tgz", "integrity": "sha512-kPEjMFtZvwL9TaZo0uZ2ml+Ye9HUMmPwbYRJ324qF9tqMejwykJ5ggTyvzmrbBeapCAbk98BSbTeovHEEP1uCA==", - "license": "MIT", "peer": true }, "node_modules/local-pkg": { @@ -9711,7 +9871,6 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", - "license": "MIT", "peer": true }, "node_modules/lodash.merge": { @@ -9836,7 +9995,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==", - "license": "MIT", "peer": true, "dependencies": { "tslib": "^2.0.3" @@ -10807,7 +10965,6 @@ "version": "0.38.0", "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.38.0.tgz", "integrity": "sha512-11Fkh6yzEmwx7O0YoLxeae0qEGFwmyPRlVxpg7oF9czOOCB/iCjdJrG5I67da5WiXK3YJCxoz9TJFE8Tfq/v9A==", - "license": "MIT", "peer": true }, "node_modules/mri": { @@ -10853,7 +11010,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==", - "license": "MIT", "peer": true, "dependencies": { "lower-case": "^2.0.2", @@ -11760,7 +11916,6 @@ "version": "6.5.9", "resolved": "https://registry.npmjs.org/react-final-form/-/react-final-form-6.5.9.tgz", "integrity": "sha512-x3XYvozolECp3nIjly+4QqxdjSSWfcnpGEL5K8OBT6xmGrq5kBqbA6+/tOqoom9NwqIPPbxPNsOViFlbKgowbA==", - "license": "MIT", "peer": true, "dependencies": { "@babel/runtime": "^7.15.4" @@ -11836,7 +11991,6 @@ "version": "0.53.0", "resolved": "https://registry.npmjs.org/react-monaco-editor/-/react-monaco-editor-0.53.0.tgz", "integrity": "sha512-ZITzsauH4CsicCGddtpgjRACaKNTVEL2hnjYFx8QMuc8zmsgfgq7D2GcF8OQsa2URIdGm/Zl7YwY2fmWpvqs/g==", - "license": "MIT", "peer": true, "dependencies": { "prop-types": "^15.8.1" @@ -11851,7 +12005,6 @@ "version": "2.16.0", "resolved": "https://registry.npmjs.org/react-player/-/react-player-2.16.0.tgz", "integrity": "sha512-mAIPHfioD7yxO0GNYVFD1303QFtI3lyyQZLY229UEAp/a10cSW+hPcakg0Keq8uWJxT2OiT/4Gt+Lc9bD6bJmQ==", - "license": "MIT", "peer": true, "dependencies": { "deepmerge": "^4.0.0", @@ -11917,7 +12070,6 @@ "version": "0.29.0", "resolved": "https://registry.npmjs.org/react-slick/-/react-slick-0.29.0.tgz", "integrity": "sha512-TGdOKE+ZkJHHeC4aaoH85m8RnFyWqdqRfAGkhd6dirmATXMZWAxOpTLmw2Ll/jPTQ3eEG7ercFr/sbzdeYCJXA==", - "license": "MIT", "peer": true, "dependencies": { "classnames": "^2.2.5", @@ -11963,7 +12115,6 @@ "version": "10.3.0", "resolved": "https://registry.npmjs.org/react-waypoint/-/react-waypoint-10.3.0.tgz", "integrity": "sha512-iF1y2c1BsoXuEGz08NoahaLFIGI9gTUAAOKip96HUmylRT6DUtpgoBPjk/Y8dfcFVmfVDvUzWjNXpZyKTOV0SQ==", - "license": "MIT", "peer": true, "dependencies": { "@babel/runtime": "^7.12.5", @@ -12722,7 +12873,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-3.0.4.tgz", "integrity": "sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==", - "license": "MIT", "peer": true, "dependencies": { "dot-case": "^3.0.4", @@ -12733,7 +12883,6 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/snakecase-keys/-/snakecase-keys-5.5.0.tgz", "integrity": "sha512-r3kRtnoPu3FxGJ3fny6PKNnU3pteb29o6qAa0ugzhSseKNWRkw1dw8nIjXMyyKaU9vQxxVIE62Mb3bKbdrgpiw==", - "license": "MIT", "peer": true, "dependencies": { "map-obj": "^4.1.0", @@ -12748,7 +12897,6 @@ "version": "3.13.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz", "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", - "license": "(MIT OR CC0-1.0)", "peer": true, "engines": { "node": ">=14.16" @@ -12828,7 +12976,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/ssr-window/-/ssr-window-3.0.0.tgz", "integrity": "sha512-q+8UfWDg9Itrg0yWK7oe5p/XRCJpJF9OBtXfOPgSJl+u3Xd5KI328RUEvUqSMVM9CiQUEf1QdBzJMkYGErj9QA==", - "license": "MIT", "peer": true }, "node_modules/stackback": { @@ -12885,7 +13032,6 @@ "version": "0.2.1", "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", "integrity": "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==", - "license": "MIT", "peer": true }, "node_modules/string-width": { @@ -13383,7 +13529,6 @@ } ], "hasInstallScript": true, - "license": "MIT", "peer": true, "dependencies": { "dom7": "^3.0.0", @@ -13895,7 +14040,6 @@ "version": "6.15.1", "resolved": "https://registry.npmjs.org/typograf/-/typograf-6.15.1.tgz", "integrity": "sha512-G1IIJlh4ycW+rvTspdgZKQoj/fKpmwif6MUhrmn3LY50W8njRnBqdFIJcfDqbmDg+RmmP51R9jdjjMwBYA5P0Q==", - "license": "MIT", "peer": true, "engines": { "node": ">= 4" @@ -14054,7 +14198,6 @@ "version": "3.11.0", "resolved": "https://registry.npmjs.org/utility-types/-/utility-types-3.11.0.tgz", "integrity": "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw==", - "license": "MIT", "peer": true, "engines": { "node": ">= 4" diff --git a/package.json b/package.json index a81edfae..81109431 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "@diplodoc/mermaid-extension": "^1.3.1", "@diplodoc/openapi-extension": "^2.4.1", "@diplodoc/prettier-config": "^2.0.0", - "@diplodoc/transform": "^4.32.3", + "@diplodoc/transform": "0.0.0-rc-add-file-meta-202410220915", "@diplodoc/tsconfig": "^1.0.2", "@octokit/core": "4.2.4", "@types/async": "^3.2.15", diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts index 658457ec..a62ae76a 100644 --- a/src/cmd/build/index.ts +++ b/src/cmd/build/index.ts @@ -1,26 +1,29 @@ -import glob from 'glob'; import {Arguments, Argv} from 'yargs'; import {join, resolve} from 'path'; import shell from 'shelljs'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; -import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants'; -import {argvValidator} from '../../validator'; -import {ArgvService, Includers, SearchService} from '../../services'; +import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '~/constants'; +import {argvValidator} from '~/validator'; +import {ArgvService, Includers, SearchService, TocService} from '~/services'; import { - initLinterWorkers, + finishProcessPages, + getLintFn, + getProcessPageFn, processAssets, processChangelogs, processExcludedFiles, - processLinter, processLogs, - processPages, processServiceFiles, -} from '../../steps'; -import {prepareMapFile} from '../../steps/processMapFile'; -import {copyFiles, logger} from '../../utils'; -import {upload as publishFilesToS3} from '../../commands/publish/upload'; +} from '~/steps'; +import {prepareMapFile} from '~/steps/processMapFile'; +import {copyFiles, logger} from '~/utils'; +import {upload as publishFilesToS3} from '~/commands/publish/upload'; +import {RevisionContext, makeRevisionContext, setRevisionContext} from '~/context/context'; +import {FsContextCli} from '~/context/fs'; +import {DependencyContextCli} from '~/context/dependency'; +import {FileQueueProcessor} from '~/context/processor'; export const build = { command: ['build', '$0'], @@ -37,12 +40,30 @@ function builder(argv: Argv) { type: 'string', group: 'Build options:', }) + .option('debug', { + alias: 'd', + describe: 'Debug mode for development', + type: 'string', + group: 'Build options:', + }) .option('output', { alias: 'o', describe: 'Path to output folder', type: 'string', group: 'Build options:', }) + .option('plugins', { + alias: 'p', + describe: 'Path to plugins js file', + type: 'string', + group: 'Build options:', + }) + .option('cached', { + default: false, + describe: 'Use cache from revision meta file', + type: 'boolean', + group: 'Build options:', + }) .option('varsPreset', { default: 'default', describe: 'Target vars preset of documentation ', @@ -176,6 +197,8 @@ function builder(argv: Argv) { } async function handler(args: Arguments) { + let hasError = false; + const userOutputFolder = resolve(args.output); const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER); const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER); @@ -203,37 +226,61 @@ async function handler(args: Arguments) { addMapFile, } = ArgvService.getConfig(); - preparingTemporaryFolders(userOutputFolder); + const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); + + const context = await makeRevisionContext( + userOutputFolder, + tmpInputFolder, + tmpOutputFolder, + outputBundlePath, + ); + + const fs = new FsContextCli(context); + const deps = new DependencyContextCli(context); + const pageProcessor = new FileQueueProcessor(context, deps); + const pageLintProcessor = new FileQueueProcessor(context, deps); - await processServiceFiles(); - processExcludedFiles(); + await preparingTemporaryFolders(context); + await processServiceFiles(context, fs); + await processExcludedFiles(); if (addMapFile) { prepareMapFile(); } - const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); + const navigationPaths = TocService.getNavigationPaths(); + + pageProcessor.setNavigationPaths(navigationPaths); + pageLintProcessor.setNavigationPaths(navigationPaths); if (!lintDisabled) { - /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */ - await initLinterWorkers(); + const filesToProcess = pageLintProcessor.getFilesToProcess(); + + const processLintPageFn = await getLintFn(context); + + await pageLintProcessor.processQueue(processLintPageFn, filesToProcess); } - const processes = [ - !lintDisabled && processLinter(), - !buildDisabled && processPages(outputBundlePath), - ].filter(Boolean) as Promise[]; + if (!buildDisabled) { + const filesToProcess = pageProcessor.getFilesToProcess(); + + const processPageFn = await getProcessPageFn(fs, deps, context, outputBundlePath); + + await pageProcessor.processQueue(processPageFn, filesToProcess); - await Promise.all(processes); + await finishProcessPages(fs); + } if (!buildDisabled) { // process additional files - processAssets({ + await processAssets({ args, outputFormat, outputBundlePath, tmpOutputFolder, userOutputFolder, + context, + fs, }); await processChangelogs(); @@ -271,34 +318,37 @@ async function handler(args: Arguments) { }); } } + + await setRevisionContext(context); } catch (err) { + if (args.debug) { + console.error(err); + } + logger.error('', err.message); + + hasError = true; } finally { processLogs(tmpInputFolder); shell.rm('-rf', tmpInputFolder, tmpOutputFolder); } + + if (hasError) { + process.exit(1); + } } -function preparingTemporaryFolders(userOutputFolder: string) { +async function preparingTemporaryFolders(revisionContext: RevisionContext) { const args = ArgvService.getConfig(); - shell.mkdir('-p', userOutputFolder); + shell.mkdir('-p', revisionContext.userOutputFolder); // Create temporary input/output folders shell.rm('-rf', args.input, args.output); shell.mkdir(args.input, args.output); - copyFiles( - args.rootInput, - args.input, - glob.sync('**', { - cwd: args.rootInput, - nodir: true, - follow: true, - ignore: ['node_modules/**', '*/node_modules/**'], - }), - ); + await copyFiles(args.rootInput, args.input, revisionContext.files, revisionContext.meta); shell.chmod('-R', 'u+w', args.input); } diff --git a/src/commands/publish/upload.ts b/src/commands/publish/upload.ts index 87227912..7c74468b 100644 --- a/src/commands/publish/upload.ts +++ b/src/commands/publish/upload.ts @@ -1,14 +1,15 @@ import type {Run} from './run'; import {join} from 'path'; import {asyncify, mapLimit} from 'async'; -import walkSync from 'walk-sync'; import mime from 'mime-types'; import {LogLevel} from '~/logger'; +import {walk} from '~/utils'; export async function upload(run: Run): Promise { const {input, endpoint, bucket, prefix, hidden = []} = run.config; const logUpload = run.logger.topic(LogLevel.INFO, 'UPLOAD'); - const filesToPublish: string[] = walkSync(run.root, { + const filesToPublish: string[] = walk({ + folder: run.root, directories: false, includeBasePath: false, ignore: hidden, diff --git a/src/context/context.ts b/src/context/context.ts new file mode 100644 index 00000000..ecbebaba --- /dev/null +++ b/src/context/context.ts @@ -0,0 +1,59 @@ +import { + RevisionContext as RevisionContextTransfrom, + RevisionMeta, +} from '@diplodoc/transform/lib/typings'; +import glob from 'glob'; +import {ArgvService} from '~/services'; +import {getMetaFile, makeMetaFile, updateChangedMetaFile, updateMetaFile} from '~/utils/meta'; + +export interface RevisionContext extends RevisionContextTransfrom { + userInputFolder: string; + userOutputFolder: string; + tmpInputFolder: string; + tmpOutputFolder: string; + outputBundlePath: string; +} + +export async function makeRevisionContext( + userOutputFolder: string, + tmpInputFolder: string, + tmpOutputFolder: string, + outputBundlePath: string, +): Promise { + const args = ArgvService.getConfig(); + + const files = glob.sync('**', { + cwd: args.rootInput, + nodir: true, + follow: true, + ignore: ['node_modules/**', '*/node_modules/**'], + }); + + const meta = normalizeMeta(await getMetaFile(userOutputFolder)); + + await updateMetaFile(args.cached, args.rootInput, meta.files, files); + + await updateChangedMetaFile(args.cached, args.rootInput, meta.files); + + return { + userInputFolder: args.rootInput, + userOutputFolder, + tmpInputFolder, + tmpOutputFolder, + outputBundlePath, + files, + meta, + }; +} + +function normalizeMeta(meta?: RevisionMeta | undefined | null) { + const metaSafe: RevisionMeta = meta ?? { + files: {}, + }; + metaSafe.files = metaSafe.files ?? {}; + return metaSafe; +} + +export async function setRevisionContext(context: RevisionContext): Promise { + await makeMetaFile(context.userOutputFolder, context.files, context.meta); +} diff --git a/src/context/dependency.ts b/src/context/dependency.ts new file mode 100644 index 00000000..d28d0a23 --- /dev/null +++ b/src/context/dependency.ts @@ -0,0 +1,62 @@ +import {resolve} from 'path'; +import {DependencyContext} from '@diplodoc/transform/lib/typings'; +import {RevisionContext} from './context'; + +export class DependencyContextCli implements DependencyContext { + private context: RevisionContext; + + constructor(context: RevisionContext) { + this.context = context; + } + + getAssetPath(path: string) { + const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/'); + if (isFromTmpInputFolder) { + const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', ''); + return assetPath; + } + + const isFromInputFolder = path.startsWith(resolve(this.context.userInputFolder) + '/'); + if (isFromInputFolder) { + const assetPath = path.replace(resolve(this.context.userInputFolder) + '/', ''); + return assetPath; + } + + return path; + } + + markDep(path: string, dependencyPath: string, type?: string): void { + type = type ?? 'include'; + + const assetPath = this.getAssetPath(path); + const depAssetPath = this.getAssetPath(dependencyPath); + + if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) { + const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? []; + const array = [...dependencies, depAssetPath]; + this.context.meta.files[assetPath].dependencies[type] = [...new Set(array)]; + } + } + + unmarkDep(path: string, dependencyPath: string, type?: string): void { + type = type ?? 'include'; + + const assetPath = this.getAssetPath(path); + const depAssetPath = this.getAssetPath(dependencyPath); + + if (assetPath && depAssetPath && this.context?.meta?.files?.[assetPath]) { + const dependencies = this.context.meta.files[assetPath].dependencies[type] ?? []; + this.context.meta.files[assetPath].dependencies[type] = dependencies.filter( + (file) => file !== depAssetPath, + ); + } + } + + resetDeps(path: string): void { + const assetPath = this.getAssetPath(path); + + if (assetPath && this.context?.meta?.files?.[assetPath]) { + this.context.meta.files[assetPath].dependencies = {}; + } + } +} diff --git a/src/context/fs.ts b/src/context/fs.ts new file mode 100644 index 00000000..24065e3c --- /dev/null +++ b/src/context/fs.ts @@ -0,0 +1,107 @@ +import {readFileSync, statSync, writeFileSync} from 'fs'; +import {readFile, stat, writeFile} from 'fs/promises'; +import {resolve} from 'path'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {RevisionContext} from './context'; + +export function isFileExists(file: string) { + try { + const stats = statSync(file); + + return stats.isFile(); + } catch (e) { + return false; + } +} + +export async function isFileExistsAsync(file: string) { + try { + const stats = await stat(file); + + return stats.isFile(); + } catch (e) { + return false; + } +} + +export class FsContextCli implements FsContext { + private context: RevisionContext; + + constructor(context: RevisionContext) { + this.context = context; + } + + getPaths(path: string) { + const arr = [path]; + + const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/'); + if (isFromTmpInputFolder) { + const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', ''); + const originPath = resolve(this.context.userInputFolder, assetPath); + + arr.unshift(originPath); + } + + return arr; + } + + exist(path: string): boolean { + const paths = this.getPaths(path); + + for (const path of paths) { + if (isFileExists(path)) { + return true; + } + } + + return false; + } + + read(path: string): string { + const paths = this.getPaths(path); + + for (const path of paths) { + if (isFileExists(path)) { + return readFileSync(path, 'utf8'); + } + } + + throw Error(`File has not been found at: ${path}`); + } + + write(path: string, content: string): void { + writeFileSync(path, content, { + encoding: 'utf8', + }); + } + + async existAsync(path: string): Promise { + const paths = this.getPaths(path); + + for (const path of paths) { + if (await isFileExistsAsync(path)) { + return true; + } + } + + return false; + } + + async readAsync(path: string): Promise { + const paths = this.getPaths(path); + + for (const path of paths) { + if (await isFileExistsAsync(path)) { + return await readFile(path, 'utf8'); + } + } + + throw Error(`File has not been found at: ${path}`); + } + + async writeAsync(path: string, content: string): Promise { + await writeFile(path, content, { + encoding: 'utf8', + }); + } +} diff --git a/src/context/processor.ts b/src/context/processor.ts new file mode 100644 index 00000000..a73eb63e --- /dev/null +++ b/src/context/processor.ts @@ -0,0 +1,89 @@ +import {DependencyContext} from '@diplodoc/transform/lib/typings'; +import {logger} from '~/utils/logger'; +import {Queue} from '~/utils/queue'; +import {RevisionContext} from './context'; + +const PAGES_ACTIVE_QUEUE_LENGTH = 200; + +type FileQueueProcessorFn = (path: string) => Promise | void; + +export class FileQueueProcessor { + private context: RevisionContext; + private deps: DependencyContext; + + private processed = new Set(); + private navigationPaths = new Set(); + + constructor(context: RevisionContext, deps: DependencyContext) { + this.context = context; + this.deps = deps; + } + + setNavigationPaths(navigationPaths: string[]) { + this.navigationPaths = new Set(navigationPaths); + } + + getFilesToProcess() { + const files = new Set( + Object.keys(this.context.meta?.files || {}).filter((path) => this.isChanged(path)), + ); + + for (const path of this.navigationPaths) { + if (this.isChanged(path)) { + files.add(path); + } + } + + return [...files]; + } + + addDepsToQueue(path: string, add: (path: string) => void) { + const dependencies = Object.keys(this.context.meta?.files || {}).filter((file) => { + const dependencies = this.context.meta?.files?.[file]?.dependencies; + return ( + dependencies?.['include']?.includes(path) || + dependencies?.['toc']?.includes(path) || + dependencies?.['presets']?.includes(path) + ); + }); + + for (const file of dependencies) { + if (!this.processed.has(file)) { + if (this.context.meta?.files?.[file]) { + this.context.meta.files[file].changed = true; + } + add(file); + } + } + } + + isChanged(path: string) { + return this.context.meta?.files?.[path]?.changed !== false; + } + + isProcessable(pattern: string) { + return this.navigationPaths.has(pattern); + } + + async processQueue(fn: FileQueueProcessorFn, files: string[] = []) { + if (files.length > 0) { + const queue = new Queue( + async (file: string) => { + if (!this.processed.has(file)) { + this.processed.add(file); + this.deps.resetDeps?.(file); + if (this.isProcessable(file)) { + await fn(file); + } + this.addDepsToQueue(file, queue.add); + } + }, + PAGES_ACTIVE_QUEUE_LENGTH, + (error, file) => logger.error(file, error.message), + ); + + files.forEach(queue.add); + await queue.loop(); + } + } +} diff --git a/src/models.ts b/src/models.ts index 7b447965..0620543f 100644 --- a/src/models.ts +++ b/src/models.ts @@ -6,6 +6,8 @@ import {LintConfig} from '@diplodoc/transform/lib/yfmlint'; import {IncludeMode, Lang, ResourceType, Stage} from './constants'; import {FileContributors, VCSConnector, VCSConnectorConfig} from './vcs-connector/connector-models'; +import {RevisionContext} from './context/context'; +import {DependencyContext, FsContext} from '@diplodoc/transform/lib/typings'; export type VarsPreset = 'internal' | 'external'; @@ -26,7 +28,10 @@ export type NestedContributorsForPathFunction = ( nestedContributors: Contributors, ) => void; export type UserByLoginFunction = (login: string) => Promise; -export type CollectionOfPluginsFunction = (output: string, options: PluginOptions) => string; +export type CollectionOfPluginsFunction = ( + output: string, + options: PluginOptions, +) => Promise; export type GetModifiedTimeByPathFunction = (filepath: string) => number | undefined; /** @@ -58,6 +63,9 @@ interface YfmConfig { varsPreset: VarsPreset; ignore: string[]; outputFormat: string; + debug: boolean; + cached: boolean; + plugins: string; allowHTML: boolean; vars: Record; applyPresets: boolean; @@ -96,6 +104,7 @@ interface YfmConfig { export interface YfmArgv extends YfmConfig { rootInput: string; input: string; + config: string; output: string; quiet: string; publish: boolean; @@ -257,16 +266,22 @@ export interface PluginOptions { changelogs?: ChangelogItem[]; extractChangelogs?: boolean; included?: boolean; + context: RevisionContext; + fs?: FsContext; + deps?: DependencyContext; } export interface Plugin { - collect: (input: string, options: PluginOptions) => string | void; + collect: (input: string, options: PluginOptions) => Promise; } export interface ResolveMd2MdOptions { inputPath: string; outputPath: string; metadata: MetaDataOptions; + context: RevisionContext; + fs: FsContext; + deps?: DependencyContext; } export interface ResolverOptions { @@ -278,6 +293,9 @@ export interface ResolverOptions { outputPath: string; outputBundlePath: string; metadata?: MetaDataOptions; + context: RevisionContext; + fs: FsContext; + deps?: DependencyContext; } export interface PathData { diff --git a/src/pages/document.ts b/src/pages/document.ts index aad0c8ad..392bbad5 100644 --- a/src/pages/document.ts +++ b/src/pages/document.ts @@ -1,14 +1,15 @@ import {join} from 'path'; +import {cloneDeepWith, flatMapDeep, isArray, isObject, isString} from 'lodash'; +import {escape} from 'html-escaper'; import {BUNDLE_FOLDER, CARRIAGE_RETURN, CUSTOM_STYLE, RTL_LANGS} from '../constants'; -import {LeadingPage, Resources, TextItems, VarsMetadata} from '../models'; +import {LeadingPage, Resources, SinglePageResult, TextItems, VarsMetadata} from '../models'; import {ArgvService, PluginService} from '../services'; +import {preprocessPageHtmlForSinglePage} from '../utils'; import {DocInnerProps, DocPageData, render} from '@diplodoc/client/ssr'; import manifest from '@diplodoc/client/manifest'; -import {escape} from 'html-escaper'; - export interface TitleMeta { title?: string; } @@ -149,3 +150,65 @@ function getResources({style, script}: Resources) { return resourcesTags.join('\n'); } + +export function joinSinglePageResults( + singlePageResults: SinglePageResult[], + root: string, + tocDir: string, +): string { + const delimeter = `
`; + return singlePageResults + .filter(({content}) => content) + .map(({content, path, title}) => + preprocessPageHtmlForSinglePage(content, {root, path, tocDir, title}), + ) + .join(delimeter); +} + +export function replaceDoubleToSingleQuotes(str: string): string { + return str.replace(/"/g, "'"); +} + +export function findAllValuesByKeys(obj, keysToFind: string[]) { + return flatMapDeep(obj, (value: string | string[], key: string) => { + if ( + keysToFind?.includes(key) && + (isString(value) || (isArray(value) && value.every(isString))) + ) { + return [value]; + } + + if (isObject(value)) { + return findAllValuesByKeys(value, keysToFind); + } + + return []; + }); +} + +export function modifyValuesByKeys( + originalObj, + keysToFind: string[], + modifyFn: (value: string) => string, +) { + function customizer(value, key) { + if (keysToFind?.includes(key) && isString(value)) { + return modifyFn(value); + } + } + + // Clone the object deeply with a customizer function that modifies matching keys + return cloneDeepWith(originalObj, customizer); +} + +export function getLinksWithContentExtersion(link: string) { + return new RegExp(/^\S.*\.(md|ya?ml|html)$/gm).test(link); +} + +export function getLinksWithExtension(link: string) { + const oneLineWithExtension = new RegExp( + /^\S.*\.(md|html|yaml|svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, + ); + + return oneLineWithExtension.test(link); +} diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts index a98ed0b7..ead38a69 100644 --- a/src/resolvers/lintPage.ts +++ b/src/resolvers/lintPage.ts @@ -10,23 +10,27 @@ import {isLocalUrl} from '@diplodoc/transform/lib/utils'; import {getLogLevel} from '@diplodoc/transform/lib/yfmlint/utils'; import {LINK_KEYS} from '@diplodoc/client/ssr'; -import {readFileSync} from 'fs'; import {bold} from 'chalk'; -import {ArgvService, PluginService} from '../services'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {ArgvService, PluginService} from '~/services'; +import {RevisionContext} from '~/context/context'; +import {FsContextCli} from '~/context/fs'; import { checkPathExists, findAllValuesByKeys, getLinksWithExtension, getVarsPerFile, getVarsPerRelativeFile, -} from '../utils'; +} from '~/utils'; import {liquidMd2Html} from './md2html'; import {liquidMd2Md} from './md2md'; interface FileTransformOptions { path: string; root?: string; + context: RevisionContext; + fs: FsContext; } const FileLinter: Record = { @@ -38,22 +42,24 @@ export interface ResolverLintOptions { inputPath: string; fileExtension: string; onFinish?: () => void; + context: RevisionContext; } -export function lintPage(options: ResolverLintOptions) { - const {inputPath, fileExtension, onFinish} = options; +export async function lintPage(options: ResolverLintOptions) { + const {inputPath, fileExtension, onFinish, context} = options; const {input} = ArgvService.getConfig(); const resolvedPath: string = resolve(input, inputPath); + const fs = new FsContextCli(context); try { - const content: string = readFileSync(resolvedPath, 'utf8'); + const content: string = await fs.readAsync(resolvedPath); const lintFn: Function = FileLinter[fileExtension]; if (!lintFn) { return; } - lintFn(content, {path: inputPath}); + await lintFn(content, {path: inputPath, fs, context}); } catch (e) { const message = `No such file or has no access to ${bold(resolvedPath)}`; console.error(message, e); @@ -65,7 +71,7 @@ export function lintPage(options: ResolverLintOptions) { } } -function YamlFileLinter(content: string, lintOptions: FileTransformOptions): void { +async function YamlFileLinter(content: string, lintOptions: FileTransformOptions): Promise { const {input, lintConfig} = ArgvService.getConfig(); const {path: filePath} = lintOptions; const currentFilePath: string = resolve(input, filePath); @@ -76,21 +82,24 @@ function YamlFileLinter(content: string, lintOptions: FileTransformOptions): voi defaultLevel: log.LogLevels.ERROR, }); - const contentLinks = findAllValuesByKeys(load(content), LINK_KEYS); + const data = load(content) as object; + const contentLinks: string[] = findAllValuesByKeys(data, LINK_KEYS); const localLinks = contentLinks.filter( (link) => getLinksWithExtension(link) && isLocalUrl(link), ); - return localLinks.forEach( - (link) => - checkPathExists(link, currentFilePath) || - log[logLevel](`Link is unreachable: ${bold(link)} in ${bold(currentFilePath)}`), + await Promise.all( + localLinks.map( + async (link) => + (await checkPathExists(lintOptions.fs, link, currentFilePath)) || + log[logLevel](`Link is unreachable: ${bold(link)} in ${bold(currentFilePath)}`), + ), ); } -function MdFileLinter(content: string, lintOptions: FileTransformOptions): void { +async function MdFileLinter(content: string, lintOptions: FileTransformOptions): Promise { const {input, lintConfig, disableLiquid, outputFormat, ...options} = ArgvService.getConfig(); - const {path: filePath} = lintOptions; + const {path: filePath, fs} = lintOptions; const plugins = outputFormat === 'md' ? [] : PluginService.getPlugins(); const vars = getVarsPerFile(filePath); @@ -101,7 +110,7 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void /* Relative path from folder of .md file to root of user' output folder */ const assetsPublicPath = relative(dirname(path), root); - const lintMarkdown = function lintMarkdown(opts: LintMarkdownFunctionOptions) { + const lintMarkdown = async function lintMarkdown(opts: LintMarkdownFunctionOptions) { const {input: localInput, path: localPath, sourceMap} = opts; const pluginOptions: PluginOptions = { @@ -114,9 +123,10 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void disableLiquid, log, getVarsPerFile: getVarsPerRelativeFile, + fs, }; - yfmlint({ + await yfmlint({ input: localInput, lintConfig, pluginOptions, @@ -140,7 +150,7 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void sourceMap = liquidResult.sourceMap; } - lintMarkdown({ + await lintMarkdown({ input: preparedContent, path, sourceMap, diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts index 4df66f56..6aa47506 100644 --- a/src/resolvers/md2html.ts +++ b/src/resolvers/md2html.ts @@ -1,21 +1,19 @@ -import type {DocInnerProps} from '@diplodoc/client'; - -import {readFileSync, writeFileSync} from 'fs'; import {basename, dirname, join, resolve, sep} from 'path'; -import {LINK_KEYS, preprocess} from '@diplodoc/client/ssr'; import {isString} from 'lodash'; +import yaml from 'js-yaml'; +import type {DocInnerProps} from '@diplodoc/client'; +import {LINK_KEYS, preprocess} from '@diplodoc/client/ssr'; import transform, {Output} from '@diplodoc/transform'; import liquid from '@diplodoc/transform/lib/liquid'; import log from '@diplodoc/transform/lib/log'; import {MarkdownItPluginCb} from '@diplodoc/transform/lib/plugins/typings'; import {getPublicPath, isFileExists} from '@diplodoc/transform/lib/utilsFS'; -import yaml from 'js-yaml'; -import {Lang, PROCESSING_FINISHED} from '../constants'; -import {LeadingPage, ResolverOptions, YfmToc} from '../models'; -import {ArgvService, PluginService, SearchService, TocService} from '../services'; -import {getAssetsPublicPath, getAssetsRootPath, getVCSMetadata} from '../services/metadata'; +import {Lang, PROCESSING_FINISHED} from '~/constants'; +import {LeadingPage, ResolverOptions, YfmToc} from '~/models'; +import {ArgvService, PluginService, SearchService, TocService} from '~/services'; +import {getAssetsPublicPath, getAssetsRootPath, getVCSMetadata} from '~/services/metadata'; import { getLinksWithContentExtersion, getVarsPerFile, @@ -24,11 +22,17 @@ import { modifyValuesByKeys, transformToc, } from '../utils'; -import {generateStaticMarkup} from '../pages'; +import {RevisionContext} from '~/context/context'; +import {DependencyContext, FsContext} from '@diplodoc/transform/lib/typings'; +import {generateStaticMarkup} from '~/pages'; export interface FileTransformOptions { + lang: string; path: string; root?: string; + fs: FsContext; + context: RevisionContext; + deps: DependencyContext; } const FileTransformer: Record = { @@ -40,14 +44,22 @@ const fixRelativePath = (relativeTo: string) => (path: string) => { return join(getAssetsPublicPath(relativeTo), path); }; -const getFileMeta = async ({fileExtension, metadata, inputPath}: ResolverOptions) => { +const getFileMeta = async ({ + fileExtension, + metadata, + inputPath, + context, + fs, + deps, +}: ResolverOptions) => { const {input, allowCustomResources} = ArgvService.getConfig(); const resolvedPath: string = resolve(input, inputPath); - const content: string = readFileSync(resolvedPath, 'utf8'); + const content: string = fs.read(resolvedPath); const transformFn: Function = FileTransformer[fileExtension]; - const {result} = transformFn(content, {path: inputPath}); + + const {result} = transformFn(content, {path: inputPath, context, fs, deps}); const vars = getVarsPerFile(inputPath); const updatedMetadata = metadata?.isContributorsEnabled @@ -120,11 +132,11 @@ const getFileProps = async (options: ResolverOptions) => { }; export async function resolveMd2HTML(options: ResolverOptions): Promise { - const {outputPath, inputPath, deep, deepBase} = options; + const {outputPath, inputPath, deep, deepBase, fs} = options; const props = await getFileProps(options); const outputFileContent = generateStaticMarkup(props, deepBase, deep); - writeFileSync(outputPath, outputFileContent); + fs.write(outputPath, outputFileContent); logger.info(inputPath, PROCESSING_FINISHED); return props; @@ -222,7 +234,7 @@ export function liquidMd2Html(input: string, vars: Record, path function MdFileTransformer(content: string, transformOptions: FileTransformOptions): Output { const {input, ...options} = ArgvService.getConfig(); - const {path: filePath} = transformOptions; + const {path: filePath, context, fs, deps} = transformOptions; const plugins = PluginService.getPlugins(); const vars = getVarsPerFile(filePath); @@ -231,7 +243,7 @@ function MdFileTransformer(content: string, transformOptions: FileTransformOptio return transform(content, { ...options, - plugins: plugins as MarkdownItPluginCb[], + plugins: plugins as MarkdownItPluginCb[], vars, root, path, @@ -240,5 +252,8 @@ function MdFileTransformer(content: string, transformOptions: FileTransformOptio getVarsPerFile: getVarsPerRelativeFile, getPublicPath, extractTitle: true, + context, + fs, + deps, }); } diff --git a/src/resolvers/md2md.ts b/src/resolvers/md2md.ts index ce36f57f..04873c2b 100644 --- a/src/resolvers/md2md.ts +++ b/src/resolvers/md2md.ts @@ -1,25 +1,24 @@ -import {readFileSync, writeFileSync} from 'fs'; import {basename, dirname, extname, join, resolve} from 'path'; import shell from 'shelljs'; import log from '@diplodoc/transform/lib/log'; import liquid from '@diplodoc/transform/lib/liquid'; +import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types'; import {ArgvService, PluginService} from '../services'; import {getVarsPerFile, logger} from '../utils'; import {PluginOptions, ResolveMd2MdOptions} from '../models'; import {PROCESSING_FINISHED} from '../constants'; -import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types'; import {enrichWithFrontMatter} from '../services/metadata'; export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise { - const {inputPath, outputPath, metadata: metadataOptions} = options; + const {inputPath, outputPath, metadata: metadataOptions, fs} = options; const {input, output, changelogs: changelogsSetting, included} = ArgvService.getConfig(); const resolvedInputPath = resolve(input, inputPath); const vars = getVarsPerFile(inputPath); const content = await enrichWithFrontMatter({ - fileContent: readFileSync(resolvedInputPath, 'utf8'), + fileContent: await fs.readAsync(resolvedInputPath), metadataOptions, resolvedFrontMatterVars: { systemVars: vars.__system as unknown, @@ -27,7 +26,21 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise }, }); - const {result, changelogs} = transformMd2Md(content, { + async function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) { + shell.mkdir('-p', dirname(targetDestPath)); + + if (options) { + const sourceIncludeContent = fs.read(targetPath); + const {result} = await transformMd2Md(sourceIncludeContent, options); + + await fs.writeAsync(targetDestPath, result); + } else { + shell.cp(targetPath, targetDestPath); + } + } + + const {result, changelogs} = await transformMd2Md(content, { + ...options, path: resolvedInputPath, destPath: outputPath, root: resolve(input), @@ -35,16 +48,21 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise collectOfPlugins: PluginService.getCollectOfPlugins(), vars: vars, log, - copyFile, included, + copyFile, + context: options.context, + fs: options.fs, + deps: options.deps, }); - writeFileSync(outputPath, result); + await fs.writeAsync(outputPath, result); if (changelogsSetting && changelogs?.length) { const mdFilename = basename(outputPath, extname(outputPath)); const outputDir = dirname(outputPath); - changelogs.forEach((changes, index) => { + + let index = 0; + for (const changes of changelogs) { let changesName; const changesDate = changes.date as string | undefined; const changesIdx = changes.index as number | undefined; @@ -63,14 +81,16 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise const changesPath = join(outputDir, `__changes-${changesName}.json`); - writeFileSync( + await fs.writeAsync( changesPath, JSON.stringify({ ...changes, source: mdFilename, }), ); - }); + + index++; + } } logger.info(inputPath, PROCESSING_FINISHED); @@ -78,23 +98,11 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise return undefined; } -function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) { - shell.mkdir('-p', dirname(targetDestPath)); - - if (options) { - const sourceIncludeContent = readFileSync(targetPath, 'utf8'); - const {result} = transformMd2Md(sourceIncludeContent, options); - writeFileSync(targetDestPath, result); - } else { - shell.cp(targetPath, targetDestPath); - } -} - -export function liquidMd2Md(input: string, vars: Record, path: string) { +export async function liquidMd2Md(input: string, vars: Record, path: string) { const {applyPresets, resolveConditions, conditionsInCode, useLegacyConditions} = ArgvService.getConfig(); - return liquid(input, vars, path, { + return await liquid(input, vars, path, { conditions: resolveConditions, substitutions: applyPresets, conditionsInCode, @@ -104,7 +112,7 @@ export function liquidMd2Md(input: string, vars: Record, path: }); } -function transformMd2Md(input: string, options: PluginOptions) { +async function transformMd2Md(input: string, options: PluginOptions) { const {disableLiquid, changelogs: changelogsSetting} = ArgvService.getConfig(); const { vars = {}, @@ -122,13 +130,14 @@ function transformMd2Md(input: string, options: PluginOptions) { const changelogs: ChangelogItem[] = []; if (!disableLiquid) { - const liquidResult = liquidMd2Md(input, vars, path); + const liquidResult = await liquidMd2Md(input, vars, path); output = liquidResult.output; } if (collectOfPlugins) { - output = collectOfPlugins(output, { + output = await collectOfPlugins(output, { + ...options, vars, path, root, @@ -140,6 +149,9 @@ function transformMd2Md(input: string, options: PluginOptions) { changelogs, extractChangelogs: Boolean(changelogsSetting), included, + context: options.context, + fs: options.fs, + deps: options.deps, }); } diff --git a/src/services/plugins.ts b/src/services/plugins.ts index b941a0ab..b623b7c0 100644 --- a/src/services/plugins.ts +++ b/src/services/plugins.ts @@ -1,7 +1,10 @@ import {LintConfig, LintRule} from '@diplodoc/transform/lib/yfmlint'; +import {existsSync} from 'fs'; +import {resolve} from 'path'; import {CollectionOfPluginsFunction, Plugin, PluginOptions} from '../models'; import {YFM_PLUGINS} from '../constants'; +import {ArgvService} from '.'; let plugins: Function[] | Plugin[]; let collectionOfPlugins: CollectionOfPluginsFunction; @@ -24,22 +27,22 @@ function makeCollectOfPlugins(): CollectionOfPluginsFunction { return typeof plugin.collect === 'function'; }); - return (output: string, options: PluginOptions) => { + return async (output: string, options: PluginOptions) => { let collectsOutput = output; - pluginsWithCollect.forEach((plugin: Plugin) => { - const collectOutput = plugin.collect(collectsOutput, options); - + for (const plugin of pluginsWithCollect) { + const collectOutput = await plugin.collect(collectsOutput, options); collectsOutput = typeof collectOutput === 'string' ? collectOutput : collectsOutput; - }); + } return collectsOutput; }; } function getAllPlugins(): Function[] { + const argsPlugins = getArgsPlugins(); const customPlugins = getCustomPlugins(); - return [...YFM_PLUGINS, ...customPlugins]; + return [...YFM_PLUGINS, ...argsPlugins, ...customPlugins]; } function getCustomPlugins(): Function[] { @@ -51,6 +54,14 @@ function getCustomPlugins(): Function[] { } } +function getArgsPlugins(): Function[] { + const {plugins: pluginsFile} = ArgvService.getConfig(); + if (pluginsFile && existsSync(resolve(pluginsFile))) { + return require(resolve(pluginsFile)); + } + return []; +} + export function getHeadContent(): string { try { return require(require.resolve('./head-content.js')); diff --git a/src/services/tocs.ts b/src/services/tocs.ts index d4bbcd98..300f40cd 100644 --- a/src/services/tocs.ts +++ b/src/services/tocs.ts @@ -1,16 +1,15 @@ import {dirname, extname, join, normalize, parse, relative, resolve, sep} from 'path'; -import {existsSync, readFileSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import shell from 'shelljs'; -import walkSync from 'walk-sync'; import liquid from '@diplodoc/transform/lib/liquid'; import log from '@diplodoc/transform/lib/log'; +import {FsContext} from '@diplodoc/transform/lib/typings'; import {bold} from 'chalk'; import {ArgvService, PresetService} from './index'; import {YfmToc} from '../models'; import {IncludeMode, Stage} from '../constants'; -import {isExternalHref, logger} from '../utils'; +import {isExternalHref, logger, walk} from '../utils'; import {filterFiles, firstFilterItem, firstFilterTextItems, liquidField} from './utils'; import {IncludersError, applyIncluders} from './includers'; import {addSourcePath} from './metadata'; @@ -22,13 +21,16 @@ export interface TocServiceData { includedTocPaths: Set; } +let fsContext: FsContext; const storage: TocServiceData['storage'] = new Map(); const tocs: TocServiceData['tocs'] = new Map(); let navigationPaths: TocServiceData['navigationPaths'] = []; const includedTocPaths: TocServiceData['includedTocPaths'] = new Set(); const tocFileCopyMap = new Map(); -async function init(tocFilePaths: string[]) { +async function init(fs: FsContext, tocFilePaths: string[]) { + fsContext = fs; + for (const path of tocFilePaths) { logger.proc(path); @@ -58,7 +60,7 @@ async function add(path: string) { } = ArgvService.getConfig(); const pathToDir = dirname(path); - const content = readFileSync(resolve(inputFolderPath, path), 'utf8'); + const content = fsContext.read(resolve(inputFolderPath, path)); const parsedToc = load(content) as YfmToc; // Should ignore toc with specified stage. @@ -114,7 +116,7 @@ async function add(path: string) { const outputPath = resolve(outputFolderPath, path); const outputToc = dump(parsedToc); shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputToc); + fsContext.write(outputPath, outputToc); } } @@ -262,7 +264,8 @@ function _copyTocDir(tocPath: string, destDir: string) { const {input: inputFolderPath} = ArgvService.getConfig(); const {dir: tocDir} = parse(tocPath); - const files: string[] = walkSync(tocDir, { + const files: string[] = walk({ + folder: tocDir, globs: ['**/*.*'], ignore: ['**/toc.yaml'], directories: false, @@ -277,11 +280,11 @@ function _copyTocDir(tocPath: string, destDir: string) { shell.mkdir('-p', parse(to).dir); if (isMdFile) { - const fileContent = readFileSync(from, 'utf8'); + const fileContent = fsContext.read(from); const sourcePath = relative(inputFolderPath, from); const updatedFileContent = addSourcePath(fileContent, sourcePath); - writeFileSync(to, updatedFileContent); + fsContext.write(to, updatedFileContent); } else { shell.cp(from, to); } @@ -392,7 +395,7 @@ async function _replaceIncludes( const includeTocDir = dirname(includeTocPath); try { - const includeToc = load(readFileSync(includeTocPath, 'utf8')) as YfmToc; + const includeToc = load(fsContext.read(includeTocPath)) as YfmToc; // Should ignore included toc with tech-preview stage. if (includeToc.stage === Stage.TECH_PREVIEW) { @@ -457,21 +460,23 @@ async function _replaceIncludes( return result; } -function getTocDir(pagePath: string): string { +function getTocDir(pagePath: string, pageBasePath?: string): string { + pageBasePath = pageBasePath ?? pagePath; + const {input: inputFolderPath} = ArgvService.getConfig(); const tocDir = dirname(pagePath); const tocPath = resolve(tocDir, 'toc.yaml'); if (!tocDir.includes(inputFolderPath)) { - throw new Error('Error while finding toc dir'); + throw new Error(`Error while finding toc dir for "${pageBasePath}"`); } - if (existsSync(tocPath)) { + if (fsContext.exist(tocPath)) { return tocDir; } - return getTocDir(tocDir); + return getTocDir(tocDir, pageBasePath); } function setNavigationPaths(paths: TocServiceData['navigationPaths']) { diff --git a/src/steps/processAssets.ts b/src/steps/processAssets.ts index f7337228..6da0c121 100644 --- a/src/steps/processAssets.ts +++ b/src/steps/processAssets.ts @@ -1,23 +1,23 @@ -import walkSync from 'walk-sync'; import {load} from 'js-yaml'; -import {readFileSync} from 'fs'; import shell from 'shelljs'; import {join, resolve, sep} from 'path'; -import {ArgvService, TocService} from '../services'; -import {checkPathExists, copyFiles, findAllValuesByKeys} from '../utils'; - import {LINK_KEYS} from '@diplodoc/client/ssr'; import {isLocalUrl} from '@diplodoc/transform/lib/utils'; +import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; import { ASSETS_FOLDER, LINT_CONFIG_FILENAME, REDIRECTS_FILENAME, + RTL_LANGS, YFM_CONFIG_FILENAME, -} from '../constants'; -import {Resources} from '../models'; -import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +} from '~/constants'; +import {ArgvService, TocService} from '~/services'; +import {checkPathExists, copyFiles, findAllValuesByKeys, walk} from '~/utils'; +import {Resources, YfmArgv} from '~/models'; +import {RevisionContext} from '~/context/context'; +import {FsContext} from '@diplodoc/transform/lib/typings'; /** * @param {Array} args @@ -28,45 +28,53 @@ import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; */ type Props = { - args: string[]; + args: YfmArgv; outputBundlePath: string; outputFormat: string; tmpOutputFolder: string; + userOutputFolder: string; + context: RevisionContext; + fs: FsContext; }; + /* * Processes assets files (everything except .md files) */ -export function processAssets({args, outputFormat, outputBundlePath, tmpOutputFolder}: Props) { - switch (outputFormat) { +export async function processAssets(props: Props) { + switch (props.outputFormat) { case 'html': - processAssetsHtmlRun({outputBundlePath}); + await processAssetsHtmlRun(props); break; case 'md': - processAssetsMdRun({args, tmpOutputFolder}); + await processAssetsMdRun(props); break; } } -function processAssetsHtmlRun({outputBundlePath}) { - const {input: inputFolderPath, output: outputFolderPath} = ArgvService.getConfig(); +async function processAssetsHtmlRun({outputBundlePath, context}: Props) { + const {input: inputFolderPath, output: outputFolderPath, langs} = ArgvService.getConfig(); - const documentationAssetFilePath: string[] = walkSync(inputFolderPath, { + const documentationAssetFilePath: string[] = walk({ + folder: inputFolderPath, directories: false, includeBasePath: false, ignore: ['**/*.yaml', '**/*.md'], }); - copyFiles(inputFolderPath, outputFolderPath, documentationAssetFilePath); + await copyFiles(inputFolderPath, outputFolderPath, documentationAssetFilePath, context.meta); - const bundleAssetFilePath: string[] = walkSync(ASSETS_FOLDER, { + const hasRTLlang = hasIntersection(langs, RTL_LANGS); + const bundleAssetFilePath: string[] = walk({ + folder: ASSETS_FOLDER, directories: false, includeBasePath: false, + ignore: hasRTLlang ? undefined : ['**/*.rtl.css'], }); - copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath); + await copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath, context.meta); } -function processAssetsMdRun({args, tmpOutputFolder}) { +async function processAssetsMdRun({args, tmpOutputFolder, context, fs}: Props) { const {input: inputFolderPath, allowCustomResources, resources} = ArgvService.getConfig(); const pathToConfig = args.config || join(args.input, YFM_CONFIG_FILENAME); @@ -85,11 +93,11 @@ function processAssetsMdRun({args, tmpOutputFolder}) { resources[type as keyof Resources]?.forEach((path: string) => resourcePaths.push(path)), ); - //copy resources - copyFiles(args.input, tmpOutputFolder, resourcePaths); + // copy resources + await copyFiles(args.input, tmpOutputFolder, resourcePaths, context.meta); } - const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { + const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { if (file.endsWith('.yaml')) { const resolvedPathToFile = resolve(inputFolderPath, file); @@ -98,32 +106,38 @@ function processAssetsMdRun({args, tmpOutputFolder}) { return acc; }, []); - tocYamlFiles.forEach((yamlFile) => { - const content = load(readFileSync(yamlFile, 'utf8')); + for (const yamlFile of tocYamlFiles) { + const content = load(fs.read(yamlFile)) as object; if (!Object.prototype.hasOwnProperty.call(content, 'blocks')) { return; } const contentLinks = findAllValuesByKeys(content, LINK_KEYS); - const localMediaLinks = contentLinks.reduce( - (acc, link) => { - const linkHasMediaExt = new RegExp( - /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, - ).test(link); - - if (linkHasMediaExt && isLocalUrl(link) && checkPathExists(link, yamlFile)) { - const linkAbsolutePath = resolveRelativePath(yamlFile, link); - const linkRootPath = linkAbsolutePath.replace(`${inputFolderPath}${sep}`, ''); - - acc.push(linkRootPath); - } - return acc; - }, - - [], - ); + const localMediaLinks = []; + + for (const link of contentLinks) { + const linkHasMediaExt = new RegExp( + /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, + ).test(link); + + if ( + linkHasMediaExt && + isLocalUrl(link) && + (await checkPathExists(fs, link, yamlFile)) + ) { + const linkAbsolutePath = resolveRelativePath(yamlFile, link); + const linkRootPath = linkAbsolutePath.replace(`${inputFolderPath}${sep}`, ''); + + localMediaLinks.push(linkRootPath); + } + } - copyFiles(args.input, tmpOutputFolder, localMediaLinks); - }); + await copyFiles(args.input, tmpOutputFolder, localMediaLinks, context.meta); + } +} + +function hasIntersection(array1: string[] | undefined | null, array2: string[] | undefined | null) { + const set1 = new Set(array1); + return array2?.some((element) => set1.has(element)); } diff --git a/src/steps/processExcludedFiles.ts b/src/steps/processExcludedFiles.ts index 7dfbc683..dc1766bc 100644 --- a/src/steps/processExcludedFiles.ts +++ b/src/steps/processExcludedFiles.ts @@ -1,9 +1,8 @@ import {relative, resolve} from 'path'; -import walkSync from 'walk-sync'; import shell from 'shelljs'; import {ArgvService, TocService} from '../services'; -import {convertBackSlashToSlash} from '../utils'; +import {convertBackSlashToSlash, walk} from '../utils'; /** * Removes all content files that unspecified in toc files or ignored. @@ -12,7 +11,8 @@ import {convertBackSlashToSlash} from '../utils'; export function processExcludedFiles() { const {input: inputFolderPath, output: outputFolderPath, ignore} = ArgvService.getConfig(); - const allContentFiles: string[] = walkSync(inputFolderPath, { + const allContentFiles: string[] = walk({ + folder: inputFolderPath, directories: false, includeBasePath: true, globs: ['**/*.md', '**/index.yaml', ...ignore], @@ -25,7 +25,9 @@ export function processExcludedFiles() { const tocSpecifiedFiles = new Set(navigationPaths); const excludedFiles = allContentFiles.filter((filePath) => !tocSpecifiedFiles.has(filePath)); - shell.rm('-f', excludedFiles); + if (excludedFiles?.length) { + shell.rm('-f', excludedFiles); + } const includedTocPaths = TocService.getIncludedTocPaths().map((filePath) => { const relativeTocPath = relative(inputFolderPath, filePath); @@ -34,5 +36,7 @@ export function processExcludedFiles() { return convertBackSlashToSlash(destTocPath); }); - shell.rm('-rf', includedTocPaths); + if (includedTocPaths?.length) { + shell.rm('-rf', includedTocPaths); + } } diff --git a/src/steps/processLinter.ts b/src/steps/processLinter.ts index 13c3c050..384704bc 100644 --- a/src/steps/processLinter.ts +++ b/src/steps/processLinter.ts @@ -2,23 +2,31 @@ import log from '@diplodoc/transform/lib/log'; import {Thread, Worker, spawn} from 'threads'; import {extname} from 'path'; -import {ArgvService, PluginService, PresetService, TocService} from '../services'; -import {ProcessLinterWorker} from '../workers/linter'; -import {logger} from '../utils'; -import {LINTING_FINISHED, MIN_CHUNK_SIZE, WORKERS_COUNT} from '../constants'; -import {lintPage} from '../resolvers'; -import {splitOnChunks} from '../utils/worker'; +import {LINTING_FINISHED, MIN_CHUNK_SIZE, WORKERS_COUNT} from '~/constants'; +import {ArgvService, PluginService, PresetService} from '~/services'; +import {ProcessLinterWorker} from '~/workers/linter'; +import {logger} from '~/utils'; +import {lintPage} from '~/resolvers'; +import {splitOnChunks} from '~/utils/worker'; +import {RevisionContext} from '~/context/context'; let processLinterWorkers: (ProcessLinterWorker & Thread)[]; -let navigationPathsChunks: string[][]; +let filesToProcessChunks: string[][]; -export async function processLinter(): Promise { +export async function processLinter( + context: RevisionContext, + filesToProcess: string[], +): Promise { const argvConfig = ArgvService.getConfig(); - const navigationPaths = TocService.getNavigationPaths(); - if (!processLinterWorkers) { - lintPagesFallback(navigationPaths); + lintPagesFallback(filesToProcess, context); + + const {error} = log.get(); + + if (error.length > 0) { + throw Error('Linting the project has failed'); + } return; } @@ -35,20 +43,27 @@ export async function processLinter(): Promise { /* Run processing the linter */ await Promise.all( processLinterWorkers.map((worker, i) => { - const navigationPathsChunk = navigationPathsChunks[i]; + const navigationPathsChunk = filesToProcessChunks[i]; return worker.run({ argvConfig, presetStorage, navigationPaths: navigationPathsChunk, + context, }); }), ); + let isSuccess = true; + /* Unsubscribe from workers */ await Promise.all( processLinterWorkers.map((worker) => { return worker.finish().then((logs) => { + if (logs.error?.length > 0) { + isSuccess = false; + } + log.add(logs); }); }), @@ -60,19 +75,22 @@ export async function processLinter(): Promise { return Thread.terminate(worker); }), ); + + if (!isSuccess) { + throw Error('Linting the project has failed'); + } } -export async function initLinterWorkers() { - const navigationPaths = TocService.getNavigationPaths(); - const chunkSize = getChunkSize(navigationPaths); +export async function initLinterWorkers(filesToProcess: string[]) { + const chunkSize = getChunkSize(filesToProcess); if (process.env.DISABLE_PARALLEL_BUILD || chunkSize < MIN_CHUNK_SIZE || WORKERS_COUNT <= 0) { return; } - navigationPathsChunks = splitOnChunks(navigationPaths, chunkSize).filter((arr) => arr.length); + filesToProcessChunks = splitOnChunks(filesToProcess, chunkSize).filter((arr) => arr.length); - const workersCount = navigationPathsChunks.length; + const workersCount = filesToProcessChunks.length; processLinterWorkers = await Promise.all( new Array(workersCount).fill(null).map(() => { @@ -86,16 +104,32 @@ function getChunkSize(arr: string[]) { return Math.ceil(arr.length / WORKERS_COUNT); } -function lintPagesFallback(navigationPaths: string[]) { +function lintPagesFallback(filesToProcess: string[], context: RevisionContext) { PluginService.setPlugins(); - navigationPaths.forEach((pathToFile) => { + filesToProcess.forEach((pathToFile) => { lintPage({ inputPath: pathToFile, fileExtension: extname(pathToFile), onFinish: () => { logger.info(pathToFile, LINTING_FINISHED); }, + context, }); }); } + +export async function getLintFn(context: RevisionContext) { + PluginService.setPlugins(); + + return async (pathToFile: string) => { + await lintPage({ + inputPath: pathToFile, + fileExtension: extname(pathToFile), + onFinish: () => { + logger.info(pathToFile, LINTING_FINISHED); + }, + context, + }); + }; +} diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index 965c3e1f..05560097 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -1,6 +1,5 @@ import type {DocInnerProps} from '@diplodoc/client'; -import {basename, dirname, extname, join, relative, resolve} from 'path'; -import {existsSync, readFileSync, writeFileSync} from 'fs'; +import {basename, dirname, extname, join, relative, resolve, sep} from 'path'; import log from '@diplodoc/transform/lib/log'; import {asyncify, mapLimit} from 'async'; import {bold} from 'chalk'; @@ -13,7 +12,7 @@ import { ResourceType, SINGLE_PAGE_DATA_FILENAME, SINGLE_PAGE_FILENAME, -} from '../constants'; +} from '~/constants'; import { LeadingPage, MetaDataOptions, @@ -21,20 +20,26 @@ import { Resources, SinglePageResult, YfmToc, -} from '../models'; -import {resolveMd2HTML, resolveMd2Md} from '../resolvers'; -import {ArgvService, LeadingService, PluginService, SearchService, TocService} from '../services'; -import {generateStaticMarkup} from '~/pages/document'; -import {generateStaticRedirect} from '~/pages/redirect'; -import {joinSinglePageResults, logger, transformTocForSinglePage} from '../utils'; -import {getVCSConnector} from '../vcs-connector'; -import {VCSConnector} from '../vcs-connector/connector-models'; +} from '~/models'; +import {resolveMd2HTML, resolveMd2Md} from '~/resolvers'; +import {ArgvService, LeadingService, PluginService, SearchService, TocService} from '~/services'; +import {joinSinglePageResults, logger, transformTocForSinglePage} from '~/utils'; +import {generateStaticMarkup, generateStaticRedirect} from '~/pages'; +import {getVCSConnector} from '~/vcs-connector'; +import {VCSConnector} from '~/vcs-connector/connector-models'; +import {RevisionContext} from '~/context/context'; +import {DependencyContext, FsContext} from '@diplodoc/transform/lib/typings'; const singlePageResults: Record = {}; const singlePagePaths: Record> = {}; // Processes files of documentation (like index.yaml, *.md) -export async function processPages(outputBundlePath: string): Promise { +export async function processPages( + fs: FsContext, + deps: DependencyContext, + outputBundlePath: string, + context: RevisionContext, +): Promise { const { input: inputFolderPath, output: outputFolderPath, @@ -66,23 +71,81 @@ export async function processPages(outputBundlePath: string): Promise { const metaDataOptions = getMetaDataOptions(pathData, vcsConnector); await preparingPagesByOutputFormat( + fs, + deps, pathData, metaDataOptions, resolveConditions, singlePage, + context, ); }), ); if (singlePage) { - await saveSinglePages(); + await saveSinglePages(fs); } if (outputFormat === 'html') { - saveRedirectPage(outputFolderPath); + await saveRedirectPage(fs, outputFolderPath); } } +export const getProcessPageFn = async ( + fs: FsContext, + deps: DependencyContext, + context: RevisionContext, + outputBundlePath: string, +) => { + const { + input: inputFolderPath, + output: outputFolderPath, + outputFormat, + singlePage, + resolveConditions, + } = ArgvService.getConfig(); + + const vcsConnector = await getVCSConnector(); + + PluginService.setPlugins(); + + return async (pathToFile: string) => { + const pathData = getPathData( + pathToFile, + inputFolderPath, + outputFolderPath, + outputFormat, + outputBundlePath, + ); + + logger.proc(pathToFile); + + const metaDataOptions = getMetaDataOptions(pathData, vcsConnector); + + await preparingPagesByOutputFormat( + fs, + deps, + pathData, + metaDataOptions, + resolveConditions, + singlePage, + context, + ); + }; +}; + +export const finishProcessPages = async (fs: FsContext) => { + const {output: outputFolderPath, outputFormat, singlePage} = ArgvService.getConfig(); + + if (singlePage) { + await saveSinglePages(fs); + } + + if (outputFormat === 'html') { + saveRedirectPage(fs, outputFolderPath); + } +}; + function getPathData( pathToFile: string, inputFolderPath: string, @@ -118,7 +181,7 @@ function getPathData( return pathData; } -async function saveSinglePages() { +async function saveSinglePages(fs: FsContext) { const { input: inputFolderPath, lang: configLang, @@ -171,8 +234,8 @@ async function saveSinglePages() { toc?.root?.deepBase || toc?.deepBase || 0, ); - writeFileSync(singlePageFn, singlePageContent); - writeFileSync(singlePageDataFn, JSON.stringify(pageData)); + fs.write(singlePageFn, singlePageContent); + fs.write(singlePageDataFn, JSON.stringify(pageData)); }), ); } catch (error) { @@ -180,7 +243,7 @@ async function saveSinglePages() { } } -function saveRedirectPage(outputDir: string): void { +async function saveRedirectPage(fs: FsContext, outputDir: string): Promise { const {lang, langs} = ArgvService.getConfig(); const redirectLang = lang || langs?.[0] || Lang.RU; @@ -189,9 +252,9 @@ function saveRedirectPage(outputDir: string): void { const redirectPagePath = join(outputDir, 'index.html'); const redirectLangPath = join(outputDir, redirectLangRelativePath); - if (!existsSync(redirectPagePath) && existsSync(redirectLangPath)) { + if (!(await fs.existAsync(redirectPagePath)) && (await fs.existAsync(redirectLangPath))) { const content = generateStaticRedirect(redirectLang, redirectLangRelativePath); - writeFileSync(redirectPagePath, content); + fs.write(redirectPagePath, content); } } @@ -245,10 +308,13 @@ function getMetaDataOptions(pathData: PathData, vcsConnector?: VCSConnector): Me } async function preparingPagesByOutputFormat( + fs: FsContext, + deps: DependencyContext, path: PathData, metaDataOptions: MetaDataOptions, resolveConditions: boolean, singlePage: boolean, + context: RevisionContext, ): Promise { const { filename, @@ -271,7 +337,7 @@ async function preparingPagesByOutputFormat( } if (outputFormat === 'md' && isYamlFileExtension && allowCustomResources) { - processingYamlFile(path, metaDataOptions); + processingYamlFile(fs, path, metaDataOptions); return; } @@ -283,12 +349,20 @@ async function preparingPagesByOutputFormat( return; } + await addTocPresetsDeps(path, fs, deps); + switch (outputFormat) { case 'md': - await processingFileToMd(path, metaDataOptions); + await processingFileToMd(path, metaDataOptions, context, fs, deps); return; case 'html': { - const resolvedFileProps = await processingFileToHtml(path, metaDataOptions); + const resolvedFileProps = await processingFileToHtml( + path, + metaDataOptions, + context, + fs, + deps, + ); SearchService.add(resolvedFileProps); @@ -305,19 +379,39 @@ async function preparingPagesByOutputFormat( log.error(message); } } -//@ts-ignore -function processingYamlFile(path: PathData, metaDataOptions: MetaDataOptions) { + +async function addTocPresetsDeps(path: PathData, fs: FsContext, deps: DependencyContext) { + const {pathToFile} = path; + + const names = pathToFile.split(sep).filter((file) => !file.includes('.')); + + for (let index = names.length; index >= 1; index--) { + const dirs = names.slice(0, index); + const tocPath = resolve(...dirs, 'toc.yaml'); + const presetsPath = resolve(...dirs, 'presets.yaml'); + + if (await fs.existAsync(tocPath)) { + deps.markDep?.(pathToFile, tocPath, 'toc'); + } + + if (await fs.existAsync(presetsPath)) { + deps.markDep?.(pathToFile, presetsPath, 'presets'); + } + } +} + +function processingYamlFile(fs: FsContext, path: PathData, metaDataOptions: MetaDataOptions) { const {pathToFile, outputFolderPath, inputFolderPath} = path; const filePath = resolve(inputFolderPath, pathToFile); - const content = readFileSync(filePath, 'utf8'); + const content = fs.read(filePath); const parsedContent = load(content) as LeadingPage; if (metaDataOptions.resources) { parsedContent.meta = {...parsedContent.meta, ...metaDataOptions.resources}; } - writeFileSync(resolve(outputFolderPath, pathToFile), dump(parsedContent)); + fs.write(resolve(outputFolderPath, pathToFile), dump(parsedContent)); } function copyFileWithoutChanges( @@ -331,19 +425,31 @@ function copyFileWithoutChanges( shell.cp(from, to); } -async function processingFileToMd(path: PathData, metaDataOptions: MetaDataOptions): Promise { +async function processingFileToMd( + path: PathData, + metaDataOptions: MetaDataOptions, + context: RevisionContext, + fs: FsContext, + deps: DependencyContext, +): Promise { const {outputPath, pathToFile} = path; await resolveMd2Md({ inputPath: pathToFile, outputPath, metadata: metaDataOptions, + context, + fs, + deps, }); } async function processingFileToHtml( path: PathData, metaDataOptions: MetaDataOptions, + context: RevisionContext, + fs: FsContext, + deps: DependencyContext, ): Promise { const {outputBundlePath, filename, fileExtension, outputPath, pathToFile} = path; const {deepBase, deep} = TocService.getDeepForPath(pathToFile); @@ -357,5 +463,8 @@ async function processingFileToHtml( metadata: metaDataOptions, deep, deepBase, + context, + fs, + deps, }); } diff --git a/src/steps/processServiceFiles.ts b/src/steps/processServiceFiles.ts index a9aa7fcc..b6714160 100644 --- a/src/steps/processServiceFiles.ts +++ b/src/steps/processServiceFiles.ts @@ -1,21 +1,22 @@ import {dirname, resolve} from 'path'; -import walkSync from 'walk-sync'; -import {readFileSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; import {ArgvService, PresetService, TocService} from '../services'; -import {logger} from '../utils'; +import {logger, walk} from '../utils'; import {DocPreset} from '../models'; import shell from 'shelljs'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {RevisionContext} from '~/context/context'; type GetFilePathsByGlobalsFunction = (globs: string[]) => string[]; -export async function processServiceFiles(): Promise { - const {input: inputFolderPath, ignore = []} = ArgvService.getConfig(); +export async function processServiceFiles(context: RevisionContext, fs: FsContext): Promise { + const {input: inputFolderPath, ignore} = ArgvService.getConfig(); const getFilePathsByGlobals = (globs: string[]): string[] => { - return walkSync(inputFolderPath, { + return walk({ + folder: [inputFolderPath, context.userInputFolder], directories: false, includeBasePath: false, globs, @@ -23,11 +24,14 @@ export async function processServiceFiles(): Promise { }); }; - preparingPresetFiles(getFilePathsByGlobals); - await preparingTocFiles(getFilePathsByGlobals); + await preparingPresetFiles(fs, getFilePathsByGlobals); + await preparingTocFiles(fs, getFilePathsByGlobals); } -function preparingPresetFiles(getFilePathsByGlobals: GetFilePathsByGlobalsFunction): void { +async function preparingPresetFiles( + fs: FsContext, + getFilePathsByGlobals: GetFilePathsByGlobalsFunction, +) { const { input: inputFolderPath, varsPreset = '', @@ -43,14 +47,14 @@ function preparingPresetFiles(getFilePathsByGlobals: GetFilePathsByGlobalsFuncti logger.proc(path); const pathToPresetFile = resolve(inputFolderPath, path); - const content = readFileSync(pathToPresetFile, 'utf8'); + const content = fs.read(pathToPresetFile); const parsedPreset = load(content) as DocPreset; PresetService.add(parsedPreset, path, varsPreset); if (outputFormat === 'md' && (!applyPresets || !resolveConditions)) { // Should save filtered presets.yaml only when --apply-presets=false or --resolve-conditions=false - saveFilteredPresets(path, parsedPreset); + saveFilteredPresets(fs, path, parsedPreset); } } } catch (error) { @@ -59,7 +63,7 @@ function preparingPresetFiles(getFilePathsByGlobals: GetFilePathsByGlobalsFuncti } } -function saveFilteredPresets(path: string, parsedPreset: DocPreset): void { +function saveFilteredPresets(fs: FsContext, path: string, parsedPreset: DocPreset): void { const {output: outputFolderPath, varsPreset = ''} = ArgvService.getConfig(); const outputPath = resolve(outputFolderPath, path); @@ -76,15 +80,16 @@ function saveFilteredPresets(path: string, parsedPreset: DocPreset): void { }); shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputPreset); + fs.write(outputPath, outputPreset); } async function preparingTocFiles( + fs: FsContext, getFilePathsByGlobals: GetFilePathsByGlobalsFunction, ): Promise { try { const tocFilePaths = getFilePathsByGlobals(['**/toc.yaml']); - await TocService.init(tocFilePaths); + await TocService.init(fs, tocFilePaths); } catch (error) { log.error(`Preparing toc.yaml files failed. Error: ${error}`); throw error; diff --git a/src/utils/common.ts b/src/utils/common.ts index c2c3782c..e634c0c4 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -1,5 +1,6 @@ import {cloneDeepWith, flatMapDeep, isArray, isObject, isString} from 'lodash'; -import {isFileExists, resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {FsContext} from '@diplodoc/transform/lib/typings'; export function findAllValuesByKeys(obj: object, keysToFind: string[]) { return flatMapDeep(obj, (value: string | string[], key: string) => { @@ -21,10 +22,10 @@ export function findAllValuesByKeys(obj: object, keysToFind: string[]) { export function modifyValuesByKeys( originalObj: object, keysToFind: string[], - modifyFn: (value: string) => string, + modifyFn: (value: string) => string | undefined, ) { // Clone the object deeply with a customizer function that modifies matching keys - return cloneDeepWith(originalObj, function (value: unknown, key) { + return cloneDeepWith(originalObj, (value: unknown, key) => { if (keysToFind.includes(key as string) && isString(value)) { return modifyFn(value); } @@ -45,8 +46,8 @@ export function getLinksWithExtension(link: string) { return oneLineWithExtension.test(link); } -export function checkPathExists(path: string, parentFilePath: string) { +export async function checkPathExists(fs: FsContext, path: string, parentFilePath: string) { const includePath = resolveRelativePath(parentFilePath, path); - return isFileExists(includePath); + return fs.existAsync(includePath); } diff --git a/src/utils/file.ts b/src/utils/file.ts index c599f84f..d4a91ca7 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -1,26 +1,77 @@ import {dirname, resolve} from 'path'; +import {copyFile} from 'node:fs/promises'; import shell from 'shelljs'; +import walkSync from 'walk-sync'; +import {RevisionMeta} from '@diplodoc/transform/lib/typings'; import {logger} from './logger'; +import {Queue} from './queue'; -export function copyFiles( +const COPY_FILES_ACTIVE_QUEUE_LENGTH = 50; + +export async function copyFiles( inputFolderPath: string, outputFolderPath: string, files: string[], -): void { + meta?: RevisionMeta | null, +) { + if (files.length === 0) { + return; + } + const dirs = new Set(); - files.forEach((pathToAsset) => { - const outputDir = resolve(outputFolderPath, dirname(pathToAsset)); - const from = resolve(inputFolderPath, pathToAsset); - const to = resolve(outputFolderPath, pathToAsset); + const queue = new Queue( + async (pathToAsset: string) => { + const from = resolve(inputFolderPath, pathToAsset); + const to = resolve(outputFolderPath, pathToAsset); + const isChanged = meta?.files?.[pathToAsset]?.changed !== false; + + if (isChanged) { + const outputDir = resolve(outputFolderPath, dirname(pathToAsset)); + + if (!dirs.has(outputDir)) { + dirs.add(outputDir); + shell.mkdir('-p', outputDir); + } + + await copyFile(from, to); + logger.copy(pathToAsset); + } + }, + COPY_FILES_ACTIVE_QUEUE_LENGTH, + (error, pathToAsset) => logger.error(pathToAsset, error.message), + ); + + files.forEach(queue.add); + await queue.loop(); +} - if (!dirs.has(outputDir)) { - dirs.add(outputDir); - shell.mkdir('-p', outputDir); - } +export function walk({ + folder, + globs, + ignore, + directories, + includeBasePath, +}: { + folder?: string | string[]; + globs?: string[]; + ignore?: string[]; + directories?: boolean; + includeBasePath?: boolean; +}) { + if (!Array.isArray(folder) && folder) { + folder = [folder]; + } - shell.cp(from, to); + const dirs = [...(folder || [])].filter(Boolean) as string[]; + const files = dirs.map((folder) => + walkSync(folder as string, { + directories, + includeBasePath, + globs, + ignore, + }), + ); - logger.copy(pathToAsset); - }); + return [...new Set(files.flat())]; } diff --git a/src/utils/meta.ts b/src/utils/meta.ts new file mode 100644 index 00000000..c4f8e7df --- /dev/null +++ b/src/utils/meta.ts @@ -0,0 +1,117 @@ +import {resolve} from 'path'; +import {readFile, stat, unlink, writeFile} from 'node:fs/promises'; +import {logger} from './logger'; +import {Queue} from './queue'; +import {RevisionMeta} from '@diplodoc/transform/lib/typings'; + +const FILE_META_NAME = '.revision.meta.json'; +const META_ACTIVE_QUEUE_LENGTH = 50; + +export async function makeMetaFile(userOutputFolder: string, files: string[], meta: RevisionMeta) { + if (meta.files) { + for (const file of Object.keys(meta.files)) { + if (!files.includes(file)) { + delete meta.files[file]; + } + } + } + + const outputFile = resolve(userOutputFolder, FILE_META_NAME); + + try { + await unlink(outputFile); + } catch (error) { + // ignore + } + + await writeFile(outputFile, JSON.stringify(meta, null, 4), {encoding: 'utf8'}); +} + +export async function getMetaFile(userOutputFolder: string): Promise { + const outputFile = resolve(userOutputFolder, FILE_META_NAME); + + try { + return JSON.parse(await readFile(outputFile, 'utf8')); + } catch (_) { + return null; + } +} + +export async function updateMetaFile( + cached: boolean, + outputFolderPath: string, + metaFiles: RevisionMeta['files'], + files: string[], +) { + if (files.length) { + const queue = new Queue( + async (pathToAsset: string) => { + const from = resolve(outputFolderPath, pathToAsset); + + try { + const changed = !cached || !metaFiles[pathToAsset]; + const modDate = Number((await stat(from)).mtime); + metaFiles[pathToAsset] = { + mod_date: changed ? modDate : (metaFiles[pathToAsset]?.mod_date ?? modDate), + dependencies: metaFiles[pathToAsset]?.dependencies || {}, + changed, + }; + } catch (error) { + // ignore + } + }, + META_ACTIVE_QUEUE_LENGTH, + (error, pathToAsset) => logger.error(pathToAsset, error.message), + ); + + files.forEach(queue.add); + + await queue.loop(); + } +} + +export async function updateChangedMetaFile( + cached: boolean, + inputFolderPath: string, + metaFiles: RevisionMeta['files'], +) { + const files = Object.keys(metaFiles); + + if (files.length) { + const queue = new Queue( + async (pathToAsset: string) => { + if (metaFiles[pathToAsset] && !metaFiles[pathToAsset].changed) { + const from = resolve(inputFolderPath, pathToAsset); + const modDateNullable = await getFileModifiedDate(from); + const modDate = modDateNullable ?? metaFiles[pathToAsset].mod_date; + + metaFiles[pathToAsset].changed = + !cached || + !modDateNullable || + isFileModified(modDate, metaFiles[pathToAsset].mod_date); + metaFiles[pathToAsset].mod_date = modDate; + } + }, + META_ACTIVE_QUEUE_LENGTH, + (error, pathToAsset) => logger.error(pathToAsset, error.message), + ); + + files.forEach(queue.add); + + await queue.loop(); + } +} + +async function getFileModifiedDate(from: string) { + try { + const data = await stat(from); + const folderLMM = Number(data.mtime); + return folderLMM; + } catch (_) { + return null; + } +} + +function isFileModified(newModDate: number, oldModDate: number) { + return Math.abs(newModDate - oldModDate) > 1000; +} diff --git a/src/utils/queue.ts b/src/utils/queue.ts new file mode 100644 index 00000000..c7aac043 --- /dev/null +++ b/src/utils/queue.ts @@ -0,0 +1,94 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +export class Queue { + stack = new Set(); + queue: unknown[][] = []; + promise: Promise | null = null; + promiseResolve: (() => void) | null = null; + promiseFinish: Promise | null = null; + promiseFinishResolve: (() => void) | null = null; + + processTask: (...args: unknown[]) => Promise; + paralleledTasks: number; + whenEmpty?: () => void; + whenError?: (e: Error, ...args: unknown[]) => void; + + constructor( + processTask: (...args: any[]) => Promise, + paralleledTasks: number, + whenError?: (error: Error, ...args: any[]) => void, + whenEmpty?: () => void, + ) { + this.processTask = processTask; + this.paralleledTasks = paralleledTasks; + this.whenEmpty = whenEmpty; + this.whenError = whenError; + } + + add = (...args: unknown[]) => { + this.queue.push(args); + }; + + loop = async () => { + for await (const _ of this.getTasks()) { + // process tasks + } + await this.promiseFinish; + this.whenEmpty?.(); + }; + + private canDryStack() { + return this.stack.size < this.paralleledTasks; + } + + private createPromiseForStack() { + if (!this.canDryStack()) { + if (!this.promise) { + this.promise = new Promise((r) => { + this.promiseResolve = r; + }); + } + } + return this.promise; + } + + private createPromiseFinish() { + if (!this.promiseFinish) { + this.promiseFinish = new Promise((r) => { + this.promiseFinishResolve = r; + }); + } + } + + private checkStack() { + if (this.promise && this.canDryStack()) { + this.promiseResolve?.(); + this.promise = null; + } + + if (this.promiseFinish && this.stack.size === 0) { + this.promiseFinishResolve?.(); + this.promiseFinish = null; + } + } + + private async *getTasks() { + this.createPromiseFinish(); + while (this.queue.length > 0) { + const task = this.queue.shift(); + if (task !== null && task !== undefined) { + this.stack.add(task); + try { + this.processTask(...task) + .catch(this.whenError) + .finally(() => { + this.stack.delete(task); + this.checkStack(); + }); + } catch (error) { + this.whenError?.(error as Error, ...task); + } + yield await this.createPromiseForStack(); + } + } + } +} diff --git a/src/workers/linter/index.ts b/src/workers/linter/index.ts index 969ee487..9d681949 100644 --- a/src/workers/linter/index.ts +++ b/src/workers/linter/index.ts @@ -3,11 +3,12 @@ import {extname} from 'path'; import {Observable, Subject} from 'threads/observable'; import {expose} from 'threads'; -import {ArgvService, PluginService, PresetService, TocService} from '../../services'; -import {TocServiceData} from '../../services/tocs'; -import {PresetStorage} from '../../services/preset'; -import {YfmArgv} from '../../models'; -import {lintPage} from '../../resolvers'; +import {ArgvService, PluginService, PresetService, TocService} from '~/services'; +import {TocServiceData} from '~/services/tocs'; +import {PresetStorage} from '~/services/preset'; +import {YfmArgv} from '~/models'; +import {lintPage} from '~/resolvers'; +import {RevisionContext} from '~/context/context'; let processedPages = new Subject(); @@ -15,9 +16,15 @@ interface ProcessLinterWorkerOptions { argvConfig: YfmArgv; navigationPaths: TocServiceData['navigationPaths']; presetStorage: PresetStorage; + context: RevisionContext; } -async function run({argvConfig, presetStorage, navigationPaths}: ProcessLinterWorkerOptions) { +async function run({ + argvConfig, + presetStorage, + navigationPaths, + context, +}: ProcessLinterWorkerOptions) { ArgvService.set(argvConfig); PresetService.setPresetStorage(presetStorage); TocService.setNavigationPaths(navigationPaths); @@ -27,6 +34,7 @@ async function run({argvConfig, presetStorage, navigationPaths}: ProcessLinterWo lintPage({ inputPath: pathToFile, fileExtension: extname(pathToFile), + context, onFinish: () => { processedPages.next(pathToFile); }, diff --git a/tests/e2e/__snapshots__/include-toc.test.ts.snap b/tests/e2e/__snapshots__/include-toc.test.ts.snap index ec4d957b..946f4009 100644 --- a/tests/e2e/__snapshots__/include-toc.test.ts.snap +++ b/tests/e2e/__snapshots__/include-toc.test.ts.snap @@ -1,133 +1,8 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Include toc Nested toc inclusions with mixed including modes 1`] = ` -"[ - "product1/_includes/inc.md", - "product1/article1.md", - "product1/toc.yaml", - "product2/overlay1/_includes/inc.md", - "product2/overlay1/article1.md", - "product2/overlay2/_includes/inc.md", - "product2/overlay2/article1.md", - "product2/overlay3/_includes/inc.md", - "product2/overlay3/article1.md", - "product2/p2.md", - "product2/toc.yaml" -]" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 2`] = ` -"--- -sourcePath: product1/core/_includes/inc.md ---- -This is the core include." -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 3`] = ` -"--- -sourcePath: product1/core/article1.md ---- -This is the core content of Article 1. - -{% include [x](_includes/inc.md) %} -" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 4`] = ` -"title: Product 1 title -items: - - name: Article1 - href: article1.md -base: product1 -deepBase: 1 -" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 5`] = ` -"--- -sourcePath: product2/core/_includes/inc.md ---- -This is the core include. -" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 6`] = ` -"--- -sourcePath: product2/overlay1/product/article1.md ---- -This is the overlay content of Article 1 for product 2. - -{% include [x](_includes/inc.md) %} -" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 7`] = ` -"--- -sourcePath: product2/core/_includes/inc.md ---- -This is the core include. -" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 8`] = ` -"--- -sourcePath: product2/overlay2/product/article1.md ---- -This is the overlay number #2 of Article 1 content for product 2. - -{% include [x](_includes/inc.md) %} -" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 9`] = ` -"--- -sourcePath: product2/core/_includes/inc.md ---- -This is the core include. -" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 10`] = ` -"--- -sourcePath: product2/core/article1.md ---- -This is the core content of Article 1. - -{% include [x](_includes/inc.md) %} -" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 11`] = ` -"This is the product 2 specific article. - -Check here link to [Article1 overlay 1](overlay1/article1.md) -Check here link to [Article1 overlay 2](overlay2/article1.md)" -`; - -exports[`Include toc Nested toc inclusions with mixed including modes 12`] = ` -"title: Product 2 title -items: - - name: P2 Article - href: p2.md - - name: Overlay1 - items: - - name: Article1 - href: overlay1/article1.md - - name: Overlay 2 - items: - - name: Article1 - href: overlay2/article1.md - - name: Overlay 3 - items: - - name: Article1 - href: overlay3/article1.md -base: product2 -deepBase: 1 -" -`; - exports[`Include toc Toc is included in link mode 1`] = ` "[ + ".revision.meta.json", "a1.md", "folder1/a1.md", "folder1/folder2/a1.md", @@ -169,6 +44,7 @@ deepBase: 0 exports[`Include toc Toc is included inline, not as a new section 1`] = ` "[ + ".revision.meta.json", ".yfm", "file1.md", "fileA.md", @@ -298,6 +174,7 @@ deepBase: 0 exports[`Include toc Toc with expressions 1`] = ` "[ + ".revision.meta.json", "a1.md", "index.yaml", "toc.yaml" diff --git a/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap b/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap index cf882252..7bdbafab 100644 --- a/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap +++ b/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap @@ -2,17 +2,16 @@ exports[`Allow load custom resources md2html single page with custom resources 1`] = ` "[ + ".revision.meta.json", "_assets/script/test1.js", "_assets/style/test.css", "_bundle/search-async-0", "_bundle/app-css-1", "_bundle/app-js-1", - "_bundle/search-css-2", "_bundle/search-js-0", "_bundle/search-js-1", "_bundle/search-css-0", "_bundle/search-js-2", - "_bundle/search-css-1", "index.html", "page.html", "project/config.html", @@ -250,9 +249,6 @@ exports[`Allow load custom resources md2html single page with custom resources 5 window.STATIC_CONTENT = false window.__DATA__ = {"data":{"leading":false,"html":"

Lorem -

/n -
-

Lorem

/n","headings":[],"meta":{"style":["_assets/style/test.css"],"script":["_assets/script/test1.js"]},"toc":{"title":"Documentation","href":"index.yaml","items":[{"name":"Documentation","href":"#_page","id":"Documentation-RANDOM"},{"name":"Config","href":"#_project_config","id":"Config-RANDOM"}],"base":".","deepBase":0,"singlePage":true}},"router":{"pathname":"single-page.html"},"lang":"ru","langs":["ru"]};