start building fields & computedFields

packages/core/package.json

@@ -24,6 +24,9 @@
     "effect": "^3.10.2",
     "esbuild": "^0.24.0",
     "glob": "^11.0.0",
+    "rehype-format": "^5.0.1",
+    "rehype-stringify": "^10.0.1",
+    "remark-rehype": "^11.1.1",
     "unified": "^11.0.5",
     "unist": "^0.0.1"
   },
@@ -33,6 +36,7 @@
     "remark-frontmatter": "^5.0.0",
     "remark-parse": "^11.0.0",
     "remark-parse-frontmatter": "^1.0.3",
-    "remark-stringify": "^11.0.0"
+    "remark-stringify": "^11.0.0",
+    "vfile": "^6.0.3"
   }
 }

packages/core/src/ConfigBuilder.ts

@@ -74,7 +74,8 @@ export const make = Effect.gen(function*() {
 
   return {
     config: config.changes.pipe(
-      Stream.filterMap(identity)
+      Stream.filterMap(identity),
+      Stream.debounce("200 millis")
     )
   } as const
 })

packages/core/src/DocumentBuilder.ts

@@ -5,20 +5,36 @@ import * as Console from "effect/Console"
 import * as Context from "effect/Context"
 import * as Effect from "effect/Effect"
 import * as Layer from "effect/Layer"
+import * as Schema from "effect/Schema"
 import * as Stream from "effect/Stream"
 import { ConfigBuilder } from "./ConfigBuilder.js"
+import type * as Document from "./Document.js"
+import type * as Source from "./Source.js"
 
 const make = Effect.gen(function*() {
   const config = yield* ConfigBuilder
 
   const documents = config.config.pipe(
-    Stream.flatMap((config) => Stream.fromIterable(config.documents)),
-    Stream.bindTo("document"),
-    Stream.bind("source", ({ document }) =>
-      document.source.pipe(
-        Stream.catchAllCause((cause) => Stream.drain(Effect.logError(cause)))
-      ) as Stream.Stream<Context.Context<never>>),
-    Stream.runForEach((_) => Console.dir(_.source, { depth: null })),
+    Stream.flatMap((config) =>
+      Stream.fromIterable(config.documents).pipe(
+        Stream.bindTo("document"),
+        Stream.let("decodeFields", ({ document }) => Schema.decodeUnknown(Schema.Struct(document.fields))),
+        Stream.bind("output", ({ document }) =>
+          document.source.pipe(
+            Stream.catchAllCause((cause) => Stream.drain(Effect.logError(cause)))
+          ) as Stream.Stream<Source.Output<unknown>>),
+        Stream.bind("fields", ({ decodeFields, document, output }) =>
+          decodeFields(output.fields).pipe(
+            Effect.flatMap((fields) => resolveComputedFields({ document, output, fields }))
+          ), { concurrency: "unbounded" }),
+        Stream.runForEach((_) =>
+          Console.dir({
+            fields: _.fields
+          }, { depth: null })
+        ),
+        Effect.catchAllCause(Effect.logError)
+      ), { switch: true }),
+    Stream.runDrain,
     Effect.forkScoped
   )
 
@@ -39,3 +55,30 @@ export class DocumentBuilder extends Context.Tag("@effect/contentlayer-core/Docu
     Layer.provide(ConfigBuilder.Live)
   )
 }
+
+const resolveComputedFields = (options: {
+  readonly document: Document.Document<any, any>
+  readonly output: Source.Output<unknown>
+  readonly fields: Record<string, unknown>
+}) =>
+  Effect.reduce(
+    options.document.computedFields,
+    options.fields,
+    (fields, group) =>
+      Effect.forEach(
+        group,
+        (field) =>
+          field.resolve(fields, options.output.context).pipe(
+            Effect.tap(Schema.validate(field.schema))
+          ),
+        { concurrency: group.length }
+      ).pipe(
+        Effect.map((values) => {
+          const newFields = { ...fields }
+          for (let index = 0; index < group.length; index++) {
+            newFields[group[index].name] = values[index]
+          }
+          return newFields
+        })
+      )
+  )

packages/core/src/Source.ts

@@ -4,6 +4,7 @@
 import type * as CommandExecutor from "@effect/platform/CommandExecutor"
 import * as FileSystem from "@effect/platform/FileSystem"
 import * as Path from "@effect/platform/Path"
+import * as Context from "effect/Context"
 import * as Data from "effect/Data"
 import * as Effect from "effect/Effect"
 import * as Stream from "effect/Stream"
@@ -14,7 +15,7 @@ import { ContentlayerError } from "./ContentlayerError.js"
  * @since 1.0.0
  * @category models
  */
-export interface Source<out Meta, out E = never> extends Stream.Stream<Output<Meta>, E, Source.Provided> {}
+export interface Source<out Meta> extends Stream.Stream<Output<Meta>, ContentlayerError, Source.Provided> {}
 
 /**
  * @since 1.0.0
@@ -25,13 +26,19 @@ export declare namespace Source {
    * @since 1.0.0
    * @category models
    */
-  export type Any = Source<any, any> | Source<any>
+  export type Any = Source<any>
 
   /**
    * @since 1.0.0
    * @category models
    */
-  export type Meta<A extends Any> = A extends Source<infer Meta, infer _E> ? Meta : never
+  export type Meta<A extends Any> = A extends Source<infer Meta> ? Meta : never
+
+  /**
+   * @since 1.0.0
+   * @category models
+   */
+  export type MetaContext<A extends Any> = A extends Source<infer Meta> ? Context.Context<Meta> : never
 
   /**
    * @since 1.0.0
@@ -61,13 +68,23 @@ export class Output<out Meta> extends Data.Class<{
   readonly content: Effect.Effect<string>
   readonly contentUint8Array: Effect.Effect<Uint8Array>
   readonly fields: Record<string, unknown>
-  readonly meta: Meta
+  readonly context: Context.Context<Meta>
 }> {
   /**
    * @since 1.0.0
    */
   readonly [OutputTypeId]: OutputTypeId = OutputTypeId
 
+  /**
+   * @since 1.0.0
+   */
+  addMeta<I, S>(tag: Context.Tag<I, S>, value: S): Output<Meta | I> {
+    return new Output({
+      ...this,
+      context: Context.add(this.context, tag, value)
+    })
+  }
+
   /**
    * @since 1.0.0
    */
@@ -99,18 +116,21 @@ export class Output<out Meta> extends Data.Class<{
  * @since 1.0.0
  * @category file system
  */
-export interface FileSystemMeta extends Path.Path.Parsed {
-  readonly path: string
-  readonly name: string
-}
+class FileSystemMeta extends Context.Tag("@effect/contentlayer-core/Source/FileSystemMeta")<
+  FileSystemMeta,
+  Path.Path.Parsed & {
+    readonly path: string
+    readonly name: string
+  }
+>() {}
 
 /**
  * @since 1.0.0
  * @category file system
  */
 export const fileSystem = (options: {
   readonly paths: ReadonlyArray<string>
-}): Source<FileSystemMeta, ContentlayerError> =>
+}): Source<FileSystemMeta> =>
   Effect.gen(function*() {
     const fs = yield* FileSystem.FileSystem
     const path_ = yield* Path.Path
@@ -131,11 +151,11 @@ export const fileSystem = (options: {
         stream: Stream.orDie(fs.stream(path)),
         content: Effect.orDie(fs.readFileString(path)),
         contentUint8Array: Effect.orDie(fs.readFile(path)),
-        meta: {
+        context: FileSystemMeta.context({
           ...path_.parse(path),
           path,
           name: path_.basename(path)
-        },
+        }),
         fields: {}
       })
 

packages/core/src/SourcePlugin.ts

@@ -1,17 +1,31 @@
 /**
  * @since 1.0.0
  */
+import * as Context from "effect/Context"
 import * as Effect from "effect/Effect"
 import * as Stream from "effect/Stream"
+import rehypeFormat from "rehype-format"
+import rehypeStringify from "rehype-stringify"
 import remarkFrontmatter from "remark-frontmatter"
 import remarkParse from "remark-parse"
 import remarkParseFrontmatter from "remark-parse-frontmatter"
+import remarkRehype from "remark-rehype"
 import remarkStringify from "remark-stringify"
 import * as Unified from "unified"
 import type * as Unist from "unist"
+import type { VFile } from "vfile"
 import { ContentlayerError } from "./ContentlayerError.js"
 import type * as Source from "./Source.js"
 
+/**
+ * @since 1.0.0
+ * @category unified
+ */
+export class UnifiedOutput extends Context.Tag("@effect/contentlayer-core/SourcePlugin/UnifiedOutput")<
+  UnifiedOutput,
+  VFile
+>() {}
+
 /**
  * @since 1.0.0
  * @category unified
@@ -21,18 +35,18 @@ export const unified = <
   HeadTree extends Unist.Node,
   TailTree extends Unist.Node,
   CompileTree extends Unist.Node,
-  Out extends Unified.CompileResults,
-  EX = never
+  Out extends Unified.CompileResults
 >(options: {
   readonly processor:
     | Unified.Processor<ParseTree, HeadTree, TailTree, CompileTree, Out>
-    | Effect.Effect<Unified.Processor<ParseTree, HeadTree, TailTree, CompileTree, Out>, EX, Source.Source.Provided>
-  readonly metadata: (_: Record<string, any>) => Record<string, any>
+    | Effect.Effect<
+      Unified.Processor<ParseTree, HeadTree, TailTree, CompileTree, Out>,
+      ContentlayerError,
+      Source.Source.Provided
+    >
+  readonly extractFields: (vfile: VFile) => Record<string, any>
 }) =>
-<In, InErr>(source: Source.Source<In, InErr>): Source.Source<
-  In,
-  EX | InErr | ContentlayerError
-> =>
+<In>(source: Source.Source<In>): Source.Source<In | UnifiedOutput> =>
   (Effect.isEffect(options.processor) ? options.processor : Effect.succeed(options.processor)).pipe(
     Effect.map((processor) =>
       source.pipe(
@@ -49,10 +63,9 @@ export const unified = <
                 })
             }),
             Effect.map((vfile) =>
-              output.addFields({
-                body: vfile.value,
-                ...options.metadata(vfile.data)
-              })
+              output
+                .addMeta(UnifiedOutput, vfile)
+                .addFields(options.extractFields(vfile))
             )
           )
         )
@@ -71,5 +84,29 @@ export const unifiedRemark = unified({
     .use(remarkStringify)
     .use(remarkFrontmatter)
     .use(remarkParseFrontmatter),
-  metadata: (data) => data.frontmatter ?? {}
+  extractFields: (vfile) => ({
+    ...vfile.data,
+    ...vfile.data?.frontmatter as any,
+    body: vfile.value
+  })
+})
+
+/**
+ * @since 1.0.0
+ * @category unified
+ */
+export const unifiedRemarkRehype = unified({
+  processor: Unified.unified()
+    .use(remarkParse)
+    .use(remarkStringify)
+    .use(remarkFrontmatter)
+    .use(remarkParseFrontmatter)
+    .use(remarkRehype)
+    .use(rehypeFormat)
+    .use(rehypeStringify),
+  extractFields: (vfile) => ({
+    ...vfile.data,
+    ...vfile.data?.frontmatter as any,
+    body: vfile.value
+  })
 })