custom nodes creation + stdlib cleanup + fork any node preps (#175)

.github/workflows/main.yaml

@@ -23,7 +23,7 @@ jobs:
         name: Install pnpm
         id: pnpm-install
         with:
-          version: 8
+          version: 9
           run_install: false
 
       - name: Get pnpm store directory
@@ -41,15 +41,15 @@ jobs:
             ${{ runner.os }}-pnpm-store-
       - name: Install dependencies
         run: pnpm install
+      - name: Lint
+        run: pnpm run lint
       - name: Build
         run: pnpm run build
         env:
           NEXT_PUBLIC_SUPABASE_URL: ${{ vars.NEXT_PUBLIC_SUPABASE_URL }}
           NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }}
       - name: Test
         run: xvfb-run -a pnpm run test
-      - name: Lint
-        run: pnpm run lint
       - name: If release, Publish to NPM
         if: github.ref == 'refs/heads/main' && startsWith(github.event.head_commit.message, 'Prepare for v')
         run: |

core/knip.json

@@ -12,6 +12,7 @@
     ],
     "project": [
         "**/*.{js,ts,tsx}",
-        "!dist/**/*"
+        "!dist/**/*",
+        "!src/misc/**/*"
     ]
 }

core/src/misc/custom-code-node-from-code.ts

@@ -0,0 +1,35 @@
+import { isCodeNode, isMacroNode, MacroNode, Node } from "../";
+import { transpileFile } from "./transpile-file/transpile-file";
+import { improvedMacroToOldMacro } from "./improved-macros.ts/improved-macros";
+
+export function customCodeNodeFromCode(
+  code: string,
+  suffixId?: string
+): Node | MacroNode<any> {
+  const transpiledCode = transpileFile("", code);
+
+  // Wrap the transpiled code to handle default export
+  const wrappedCode = `(function () {
+      const __imports = arguments[0];
+      const __exports = {};
+      ${transpiledCode}
+
+      return __exports;
+  })(arguments)
+  `;
+
+  const result = new Function(wrappedCode)({});
+
+  if (isCodeNode(result.default) || isMacroNode(result.default)) {
+    if (result.default.icon) {
+      const macro = improvedMacroToOldMacro(result.default) as MacroNode<any>;
+      macro.id = `${macro.id}${suffixId ? `-${suffixId}` : ""}`;
+      return macro;
+    }
+    const node = result.default as Node;
+    node.id = `${node.id}${suffixId ? `-${suffixId}` : ""}`;
+    return node;
+  } else {
+    throw new Error("Invalid node type");
+  }
+}

core/src/misc/improved-macros.ts/improved-macro-utils.ts

@@ -0,0 +1,189 @@
+import {
+  MacroConfigurableValue,
+  InputPin,
+  nodeInput,
+  MacroNode,
+  MacroEditorFieldDefinition,
+} from "../..";
+
+function extractInputNameAndPath(match: string): {
+  inputName: string;
+  path: string[];
+} {
+  const cleaned = match.replace(/[{}]/g, "").trim();
+  const parts = cleaned.split(".");
+  return {
+    inputName: parts[0],
+    path: parts.slice(1),
+  };
+}
+
+export function extractInputsFromValue(
+  val: MacroConfigurableValue,
+  key: string
+): Record<string, InputPin> {
+  const inputs = {};
+
+  function extractFromValue(value: any) {
+    if (typeof value === "string") {
+      const matches = value.match(/({{.*?}})/g);
+      if (matches) {
+        for (const match of matches) {
+          const { inputName } = extractInputNameAndPath(match);
+          inputs[inputName] = nodeInput();
+        }
+      }
+    }
+  }
+
+  if (val.type === "string") {
+    extractFromValue(val.value);
+  } else if (val.type === "dynamic") {
+    return { [key]: nodeInput() };
+  } else {
+    try {
+      const jsonString = JSON.stringify(val.value);
+      const matches = jsonString.match(/({{.*?}})/g);
+      if (matches) {
+        for (const match of matches) {
+          const { inputName } = extractInputNameAndPath(match);
+          inputs[inputName] = nodeInput();
+        }
+      }
+    } catch (error) {
+      console.error("Error stringifying value:", error);
+    }
+  }
+
+  return inputs;
+}
+
+export function replaceInputsInValue(
+  inputs: Record<string, any>,
+  value: MacroConfigurableValue,
+  fieldName: string,
+  ignoreMissingInputs: boolean = true
+): MacroConfigurableValue["value"] {
+  if (value.type === "string") {
+    return value.value.replace(/({{.*?}})/g, (match) => {
+      const { inputName, path } = extractInputNameAndPath(match);
+      let result = inputs[inputName];
+      for (const key of path) {
+        if (result && typeof result === "object" && key in result) {
+          result = result[key];
+        } else {
+          return ignoreMissingInputs ? match : "";
+        }
+      }
+      return result !== undefined ? result : match;
+    });
+  }
+
+  if (value.type === "dynamic") {
+    return inputs[fieldName];
+  }
+
+  const jsonString = JSON.stringify(value.value);
+  const replacedJsonString = jsonString.replace(/({{.*?}})/g, (match) => {
+    const { inputName, path } = extractInputNameAndPath(match);
+    let result = inputs[inputName];
+    for (const key of path) {
+      if (result && typeof result === "object" && key in result) {
+        result = result[key];
+      } else {
+        return match; // Return original match if path is invalid
+      }
+    }
+    return result !== undefined ? result : match;
+  });
+
+  try {
+    return JSON.parse(replacedJsonString);
+  } catch (error) {
+    console.error("Error parsing replaced JSON:", error);
+    return value;
+  }
+}
+
+export function renderConfigurableValue(
+  value: MacroConfigurableValue,
+  fieldName: string
+) {
+  if (value.type === "dynamic") {
+    return `{{${fieldName}}}`;
+  } else return `${value.value}`;
+}
+
+export function generateConfigEditor<Config>(
+  config: Config,
+  overrides?: Partial<Record<keyof Config, any>>
+): MacroNode<Config>["editorConfig"] {
+  const fields = Object.keys(config).map((key) => {
+    const value = config[key];
+    const override = overrides && overrides[key];
+    let fieldType: MacroEditorFieldDefinition["type"];
+
+    if (override) {
+      fieldType = override.type || (typeof value as any);
+    } else {
+      switch (typeof value) {
+        case "string":
+          fieldType = "string";
+          break;
+        case "number":
+          fieldType = "number";
+          break;
+        case "boolean":
+          fieldType = "boolean";
+          break;
+        case "object":
+          fieldType = "json";
+          break;
+        default:
+          fieldType = "string";
+          break;
+      }
+    }
+
+    return {
+      type: fieldType,
+      configKey: key,
+      label:
+        override?.label ||
+        key
+          .replace(/([A-Z])/g, " $1")
+          .replace(/^./, (str) => str.toUpperCase()),
+    };
+  });
+
+  return {
+    type: "structured",
+    fields: fields as MacroEditorFieldDefinition[],
+  };
+}
+
+export function renderDerivedString(displayName: string, config: any) {
+  const string = displayName?.replace(/{{.*?}}/g, (match) => {
+    const { inputName } = extractInputNameAndPath(match);
+    const value = config[inputName];
+    const isMacroConfigurableValue =
+      value && typeof value === "object" && "type" in value && "value" in value;
+    if (isMacroConfigurableValue) {
+      if (value.type === "dynamic") {
+        return match;
+      } else {
+        return value.value;
+      }
+    }
+    return match;
+  });
+
+  // Format time values in the final string
+  return string?.replace(/(\d+)ms/g, (match, p1) => {
+    const num = parseInt(p1, 10);
+    if (num >= 1000) {
+      return num / 1000 + "s";
+    }
+    return match;
+  });
+}

core/src/misc/improved-macros.ts/improved-macros.spec.ts

@@ -0,0 +1,70 @@
+import { assert } from "chai";
+import { spiedOutput } from "../../test-utils";
+
+import { improvedMacroToOldMacro, ImprovedMacroNode } from "./improved-macros";
+import {
+  extractInputsFromValue,
+  replaceInputsInValue,
+} from "./improved-macro-utils";
+import { eventually } from "../../";
+import {
+  MacroConfigurableValue,
+  macroConfigurableValue,
+  nodeOutput,
+  dynamicNodeInput,
+} from "../..";
+
+describe("ImprovedMacros", () => {
+  describe("SimpleMacro with dot notation", () => {
+    it("processes input with dot notation template", async () => {
+      // Define a simple macro node
+      const SimpleMacro: ImprovedMacroNode<{
+        message: MacroConfigurableValue;
+      }> = {
+        id: "SimpleMacro",
+        defaultConfig: {
+          message: macroConfigurableValue(
+            "string",
+            "Hello, {{person.name}}! Your age is {{person.age}}."
+          ),
+        },
+        inputs: (config) => extractInputsFromValue(config.message, "message"),
+        outputs: {
+          result: nodeOutput(),
+        },
+        run: (inputs, outputs, ctx) => {
+          const message = replaceInputsInValue(
+            inputs,
+            ctx.context.config.message,
+            "message"
+          );
+
+          outputs.result.next(message);
+        },
+      };
+
+      const macro = improvedMacroToOldMacro(SimpleMacro);
+
+      const definition = macro.definitionBuilder(macro.defaultData);
+      assert.deepEqual(Object.keys(definition.inputs), ["person"]);
+      assert.deepEqual(Object.keys(definition.outputs), ["result"]);
+
+      const runFn = macro.runFnBuilder(macro.defaultData);
+
+      const [spy, result] = spiedOutput();
+
+      const input = dynamicNodeInput();
+      const testPerson = { name: "Alice", age: 30 };
+      runFn({ person: testPerson }, { result }, {
+        context: { config: macro.defaultData },
+      } as any);
+
+      input.subject.next(testPerson);
+
+      await eventually(() => {
+        assert.equal(spy.callCount, 1);
+        assert.equal(spy.lastCall.args[0], "Hello, Alice! Your age is 30.");
+      });
+    });
+  });
+});