Merge pull request #1646 from DanielXMoore/speed

Multithreaded compilation via Node workers and `threads` option or `CIVET_THREADS` environment variable; enable Node compiler cache; improve unplugin caching

build/build.sh

@@ -23,7 +23,11 @@ rm -rf dist/unplugin/source
 
 # cli
 BIN="dist/civet"
-echo "#!/usr/bin/env node" | cat - dist/cli.js > "$BIN"
+(
+  echo "#!/usr/bin/env node"
+  echo '"use strict"'
+  echo "try { require('node:module').enableCompileCache() } catch {}"
+) | cat - dist/cli.js > "$BIN"
 echo "cli()" >> "$BIN"
 chmod +x "$BIN"
 rm dist/cli.js

build/esbuild.civet

@@ -1,8 +1,18 @@
+try { require('node:module').enableCompileCache() } catch {}
 esbuild := require "esbuild"
 heraPlugin := require "@danielx/hera/esbuild-plugin"
 // Need to use the packaged version because we may not have built our own yet
-civetPlugin := require "../node_modules/@danielx/civet/dist/esbuild-plugin.js"
+civetOldPlugin := require "../node_modules/@danielx/civet/dist/esbuild-plugin.js"
 civetUnplugin := require("../node_modules/@danielx/civet/dist/unplugin/esbuild.js").default
+civetPlugin := civetUnplugin
+  ts: "civet"
+  cache: true
+  config: null
+civetPluginEmit := civetUnplugin
+  ts: 'civet'
+  emitDeclaration: true
+  declarationExtension: ".d.ts"
+  config: null
 
 path := require "path"
 {access} := require "fs/promises"
@@ -56,6 +66,13 @@ rewriteCivetImports := {
         external: true
 }
 
+defineDirname := (esm: boolean) =>
+  if esm
+    "__dirname": "undefined"
+  else
+    "__dirname": 'undefined'
+    "import.meta.url": '""'
+
 // Files that need civet imports re-written
 // since they aren't actually bundled
 for name of ["cli", "esbuild-plugin"]
@@ -67,7 +84,7 @@ for name of ["cli", "esbuild-plugin"]
     outfile: `dist/${name}.js`
     plugins: [
       rewriteCivetImports
-      civetPlugin()
+      civetPlugin
     ]
     external: [
       '../package.json'
@@ -92,7 +109,7 @@ for name of ["config", "babel-plugin"]
       outExtension: ".js": if format is "esm" then ".mjs" else ".js"
       plugins: [
         rewriteCivetImports
-        civetPlugin()
+        civetPlugin
       ]
       footer:
         // Rewrite default export as CJS exports object,
@@ -101,7 +118,7 @@ for name of ["config", "babel-plugin"]
     }).catch -> process.exit 1
 
 // esm needs to be a module for import.meta
-for name of ["esm"]
+for name of ["esm", "node-worker"]
   build({
     entryPoints: [`source/${name}.civet`]
     bundle: true
@@ -110,7 +127,7 @@ for name of ["esm"]
     outfile: `dist/${name}.mjs`
     plugins: [
       rewriteCivetImports
-      civetPlugin()
+      civetPlugin
     ]
   }).catch -> process.exit 1
 
@@ -124,8 +141,11 @@ for esm of [false, true]
     plugins: [
       resolveExtensions
       heraPlugin module: true
-      civetPlugin()
+      civetPlugin
     ]
+    define: unless esm
+      "import.meta.url": '""' // avoid warning; eliminated by `dropLabels`
+    dropLabels: ['ESM_ONLY'] unless esm
   }).catch -> process.exit 1
 
 // Browser build
@@ -140,11 +160,20 @@ build({
     'node:module': './source/browser-shim.civet'
     'node:path': './source/browser-shim.civet'
     'node:vm': './source/browser-shim.civet'
-  external: ['node:fs']
+  external:
+    . 'node:fs'
+    . 'node:url' // won't actually be imported, via `dropLabels`
+    . 'node:worker_threads' // won't actually be imported, via `define`
+  define:
+    "process.env.CIVET_THREADS": '0'
+    "import.meta.url": '""' // avoid warning; eliminated by `dropLabels`
+  dropLabels: ['ESM_ONLY']
+  minifySyntax: true // eliminate `if (false)` from `define` setting
+  minify
   plugins: [
     resolveExtensions
     heraPlugin module: true
-    civetPlugin()
+    civetOldPlugin() // currently necessary for `alias` to work
   ]
 }).catch -> process.exit 1
 
@@ -158,7 +187,7 @@ build({
   target: "esNext"
   outfile: 'dist/bun-civet.mjs'
   plugins: [
-    civetPlugin()
+    civetPlugin
   ]
 }).catch -> process.exit 1
 
@@ -185,11 +214,8 @@ for format of ["esm", "cjs"]
     outExtension: ".js": if format is "esm" then ".mjs" else ".js"
     plugins: [
       rewriteCivetImports
-      civetUnplugin
-        ts: 'civet'
-        emitDeclaration: format is "esm"  // only run TypeScript once
-        declarationExtension: ".d.ts"
-        config: null
+      // only run TypeScript once
+      if format is "esm" then civetPluginEmit else civetPlugin
     ]
   }).catch -> process.exit 1
 
@@ -206,7 +232,5 @@ for format of ["esm", "cjs"]
     outExtension: ".js": if format is "esm" then ".mjs" else ".js"
     plugins: [
       civetPlugin
-        emitDeclaration: format is "esm"
-        declarationExtension: ".d.ts"
     ]
   }).catch -> process.exit 1

civet.dev/config.md

@@ -209,3 +209,10 @@ civet --config custom-config.civet ...
 # Disable config files
 civet --no-config ...
 ```
+
+## Compiler Options
+
+In addition to the "parse options" described above, there are a few
+top-level options (above `parseOptions`):
+
+- `threads`: Use specified number of Node worker threads to compile Civet files faster. Default: `0` (don't use threads), or `CIVET_THREADS` environment variable if set.

package.json

@@ -100,9 +100,9 @@
     "@prettier/sync": "^0.5.2",
     "@types/assert": "^1.5.6",
     "@types/mocha": "^10.0.8",
-    "@types/node": "^20.12.2",
+    "@types/node": "^22.10.2",
     "c8": "^7.12.0",
-    "esbuild": "0.20.0",
+    "esbuild": "0.24.0",
     "marked": "^4.2.4",
     "mocha": "^10.7.3",
     "prettier": "^3.2.5",

source/main.civet

@@ -8,6 +8,7 @@ import type { BlockStatement } from ./parser/types.civet
 export type { ASTError, BlockStatement } from ./parser/types.civet
 
 import StateCache from "./state-cache.civet"
+import { WorkerPool } from "./worker-pool.civet"
 
 export class ParseErrors extends Error
   name = "ParseErrors"
@@ -90,13 +91,30 @@ export type CompilerOptions
     comptime?: boolean
     globals?: string[]
     symbols?: string[]
-  // Specifying an empty array will prevent ParseErrors from being thrown
+  /** Specifying an empty array will prevent ParseErrors from being thrown */
   errors?: ParseError[]
+  /** Number of parallel threads to compile with (Node only) */
+  threads?: number
 
 type CompileOutput<T extends CompilerOptions> =
   T extends { ast: true } ? BlockStatement : T extends { sourceMap: true } ? { code: string, sourceMap: ReturnType<typeof SourceMap> } : string
 
+let workerPool: WorkerPool?
+
 export function compile<const T extends CompilerOptions>(src: string, options?: T): T extends { sync: true } ? CompileOutput<T> : Promise<CompileOutput<T>>
+  // `CIVET_THREADS=0` (including in browser build) forces no threads
+  unless process.env.CIVET_THREADS == 0
+    threads := parseInt options?.threads ?? process.env.CIVET_THREADS, 10
+    if threads is 0 // explicit 0 terminates existing threads
+      workerPool?.setThreads 0
+    else if not isNaN(threads) and threads > 0 and not options.sync
+      if workerPool?
+        workerPool.setThreads threads
+      else
+        workerPool = new WorkerPool threads
+      // Prevent worker from recursively spawning its own worker
+      return workerPool.run 'compile', src, {...options, threads: 0}
+
   unless options
     options = {} as T
   else

source/node-worker.civet

@@ -0,0 +1,20 @@
+{ parentPort } from node:worker_threads
+
+module from node:module
+try module.enableCompileCache()
+
+async do
+  // import dynamically to use compile cache
+  { compile } from ./main.civet
+
+  parentPort!.on 'message', {id:: number, op:: string, args:: any[]} =>
+    try
+      let result
+      switch op
+        when "compile"
+          result = await (compile as any) ...args
+        else
+          throw `Unknown operation: ${op}`
+      parentPort!.postMessage {id, result}
+    catch error
+      parentPort!.postMessage {id, error}

source/unplugin/README.md

@@ -210,7 +210,11 @@ interface PluginOptions {
     Note that some bundlers require additional plugins to handle TS.
     For example, for Webpack, you would need to install `ts-loader` and add it to your webpack config.
     Unfortunately, Rollup's TypeScript plugin is incompatible with this plugin, so you need to set `ts` to another option.
-- `cache`: Cache compilation results based on file's mtime (Useful for longer running processes like `watch` or `serve`).
+- `cache`: Cache compilation results based on file's mtime.
+  Useful when bundling the same source files for both CommonJS and ESM,
+  or for longer running processes like `watch` or `serve`.  Default: `true`.
+- `threads`: Use specified number of Node worker threads to
+  compile Civet files faster. Default: `0` (don't use threads), or `CIVET_THREADS` environment variable if set.
 - `config`: Civet config filename to load, or `null` to avoid looking for the
   default config filenames in the project root directory.
   See [Civet config](https://civet.dev/config).

source/unplugin/unplugin.civet

@@ -39,14 +39,23 @@ export type PluginOptions
   js?: boolean
   /** @deprecated Use "emitDeclaration" instead */
   dts?: boolean
+  /** Number of parallel threads to compile with (Node only) */
+  threads?: number
   /** Cache compilation results based on file mtime (useful for serve or watch mode) */
   cache?: boolean
   /** config filename, or null to not look for default config file */
   config?: string? | null
   parseOptions?: ParseOptions
 
+type CacheEntry
+  mtime: number
+  result?: TransformResult
+  promise?: Promise<void>
+
 civetExtension := /\.civet$/
-isCivetTranspiled := /(\.civet)(\.[jt]sx)([?#].*)?$/
+// Normally .jsx/.tsx extension should be present, but sometimes
+// (e.g. esbuild's alias feature) loads directly without resolve
+isCivetTranspiled := /(\.civet)(\.[jt]sx)?([?#].*)?$/
 postfixRE := /[?#].*$/s
 isWindows := os.platform() is 'win32'
 windowsSlashRE := /\\/g
@@ -131,7 +140,7 @@ export const rawPlugin: Parameters<typeof createUnplugin<PluginOptions>>[0] =
         : (f) => f.toLowerCase()
     }
 
-  const cache = options.cache ? new Map<string, {mtime: number, result: TransformResult}>() : undefined;
+  cache := new Map<string, CacheEntry> unless options.cache is false
 
   plugin: ReturnType<typeof rawPlugin> & { __virtualModulePrefix?: string } := {
     name: 'unplugin-civet'
@@ -142,12 +151,13 @@ export const rawPlugin: Parameters<typeof createUnplugin<PluginOptions>>[0] =
         ? options.config
         : await findInDir(process.cwd())
       if civetConfigPath
-        compileOptions = await loadConfig(civetConfigPath)
+        compileOptions = await loadConfig civetConfigPath
       // Merge parseOptions, with plugin options taking priority
       compileOptions.parseOptions = {
         ...compileOptions.parseOptions
         ...options.parseOptions
       }
+      compileOptions.threads = options.threads if options.threads?
 
       if transformTS or ts is "tsc"
         ts := await tsPromise!
@@ -423,14 +433,19 @@ export const rawPlugin: Parameters<typeof createUnplugin<PluginOptions>>[0] =
 
       filename := path.resolve rootDir, basename
 
-      let mtime
-      if cache
+      let mtime: number?, cached: CacheEntry?, resolve: =>?
+      if cache?
         mtime = fs.promises.stat(filename) |> await |> .mtimeMs
-        const cached = cache?.get filename
+        cached = cache?.get filename
         if cached and cached.mtime is mtime
-          return cached.result
+          // If the file is currently being compiled, wait for it to finish
+          await cached.promise if cached.promise
+          return cached.result if cached.result
+        // We're the first to compile this file with this mtime
+        promise := new Promise<void> (r): void => resolve = r
+        cache.set filename, cached = {mtime, promise}
+      finally resolve?()
 
-      rawCivetSource := await fs.promises.readFile filename, 'utf-8'
       this.addWatchFile filename
 
       let compiled: string
@@ -439,11 +454,12 @@ export const rawPlugin: Parameters<typeof createUnplugin<PluginOptions>>[0] =
         ...compileOptions
         filename: id
         errors: []
-      } as const
+      }
       function checkErrors
         if civetOptions.errors#
           throw new civet.ParseErrors civetOptions.errors
 
+      rawCivetSource := await fs.promises.readFile filename, 'utf-8'
       ast := await civet.compile rawCivetSource, {
         ...civetOptions
         ast: true
@@ -533,7 +549,9 @@ export const rawPlugin: Parameters<typeof createUnplugin<PluginOptions>>[0] =
       if options.transformOutput
         transformed = await options.transformOutput transformed.code, id
 
-      cache?.set filename, {mtime!, result: transformed}
+      if cached?
+        cached.result = transformed
+        delete cached.promise
 
       return transformed