Support generating Python SDK with Pyodide

cspell.yaml

@@ -247,6 +247,11 @@ words:
   - xplat
   - xxsubtype
   - yamls
+  - pyodide
+  - pyimport
+  - NODEFS
+  - deps
+  - pyyaml
 ignorePaths:
   - "**/node_modules/**"
   - "**/dist/**"

packages/http-client-python/emitter/src/emitter.ts

@@ -5,13 +5,15 @@ import {
   SdkServiceOperation,
 } from "@azure-tools/typespec-client-generator-core";
 import { EmitContext } from "@typespec/compiler";
-import { execSync } from "child_process";
+import { exec } from "child_process";
 import fs from "fs";
 import path, { dirname } from "path";
+import { loadPyodide } from "pyodide";
 import { fileURLToPath } from "url";
 import { emitCodeModel } from "./code-model.js";
 import { saveCodeModelAsYaml } from "./external-process.js";
 import { PythonEmitterOptions, PythonSdkContext } from "./lib.js";
+import { runPython3 } from "./run-python3.js";
 import { removeUnderscoresFromNamespace } from "./utils.js";
 
 export function getModelsMode(context: SdkContext): "dpg" | "none" {
@@ -78,50 +80,116 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
   const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..");
   const outputDir = context.emitterOutputDir;
   const yamlMap = emitCodeModel(sdkContext);
-  addDefaultOptions(sdkContext);
   const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap);
-  let venvPath = path.join(root, "venv");
-  if (fs.existsSync(path.join(venvPath, "bin"))) {
-    venvPath = path.join(venvPath, "bin", "python");
-  } else if (fs.existsSync(path.join(venvPath, "Scripts"))) {
-    venvPath = path.join(venvPath, "Scripts", "python.exe");
-  } else {
-    throw new Error("Virtual environment doesn't exist.");
-  }
-  const commandArgs = [
-    venvPath,
-    `${root}/eng/scripts/setup/run_tsp.py`,
-    `--output-folder=${outputDir}`,
-    `--cadl-file=${yamlPath}`,
-  ];
+  addDefaultOptions(sdkContext);
   const resolvedOptions = sdkContext.emitContext.options;
-  if (resolvedOptions["packaging-files-config"]) {
-    const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map(
-      ([key, value]) => {
-        return `${key}:${value}`;
-      },
-    );
-    commandArgs.push(`--packaging-files-config='${keyValuePairs.join("|")}'`);
-    resolvedOptions["packaging-files-config"] = undefined;
-  }
-  if (
-    resolvedOptions["package-pprint-name"] !== undefined &&
-    !resolvedOptions["package-pprint-name"].startsWith('"')
-  ) {
-    resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`;
-  }
+  if (resolvedOptions["use-pyodide"]) {
+    const commandArgs: Record<string, string> = {};
+    if (resolvedOptions["packaging-files-config"]) {
+      const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map(
+        ([key, value]) => {
+          return `${key}:${value}`;
+        },
+      );
+      commandArgs["packaging-files-config"] = keyValuePairs.join("|");
+      resolvedOptions["packaging-files-config"] = undefined;
+    }
+    if (
+      resolvedOptions["package-pprint-name"] !== undefined &&
+      !resolvedOptions["package-pprint-name"].startsWith('"')
+    ) {
+      resolvedOptions["package-pprint-name"] = `${resolvedOptions["package-pprint-name"]}`;
+    }
 
-  for (const [key, value] of Object.entries(resolvedOptions)) {
-    commandArgs.push(`--${key}=${value}`);
-  }
-  if (sdkContext.arm === true) {
-    commandArgs.push("--azure-arm=true");
-  }
-  if (resolvedOptions.flavor === "azure") {
-    commandArgs.push("--emit-cross-language-definition-file=true");
-  }
-  commandArgs.push("--from-typespec=true");
-  if (!program.compilerOptions.noEmit && !program.hasError()) {
-    execSync(commandArgs.join(" "));
+    for (const [key, value] of Object.entries(resolvedOptions)) {
+      commandArgs[key] = value;
+    }
+    if (sdkContext.arm === true) {
+      commandArgs["azure-arm"] = "true";
+    }
+    if (resolvedOptions.flavor === "azure") {
+      commandArgs["emit-cross-language-definition-file"] = "true";
+    }
+    commandArgs["from-typespec"] = "true";
+
+    if (!program.compilerOptions.noEmit && !program.hasError()) {
+      const outputFolder = path.relative(root, outputDir);
+      if (!fs.existsSync(outputFolder)) {
+        fs.mkdirSync(outputFolder, { recursive: true });
+      }
+      const pyodide = await loadPyodide({ indexURL: path.join(root, "node_modules", "pyodide") });
+      pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, ".");
+      await pyodide.loadPackage("setuptools");
+      await pyodide.loadPackage("tomli");
+      await pyodide.loadPackage("docutils");
+      await pyodide.loadPackage("micropip");
+      const micropip = pyodide.pyimport("micropip");
+      await micropip.install("emfs:generator/dist/pygen-0.1.0-py3-none-any.whl");
+      const yamlRelativePath = path.relative(root, yamlPath);
+      const globals = pyodide.toPy({ outputFolder, yamlRelativePath, commandArgs });
+      const python = `
+        async def main():
+          import warnings
+          with warnings.catch_warnings():
+            warnings.simplefilter("ignore", SyntaxWarning)
+            from pygen import m2r, preprocess, codegen, black
+
+          m2r.M2R(output_folder=outputFolder, cadl_file=yamlRelativePath, **commandArgs).process()
+          preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlRelativePath, **commandArgs).process()
+          codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlRelativePath, **commandArgs).process()
+          black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process()
+
+        await main()
+        `;
+      await pyodide.runPythonAsync(python, { globals });
+    }
+  } else {
+    let venvPath = path.join(root, "venv");
+    if (!fs.existsSync(venvPath)) {
+      await runPython3("./eng/scripts/setup/install.py");
+      await runPython3("./eng/scripts/setup/prepare.py");
+    }
+    if (fs.existsSync(path.join(venvPath, "bin"))) {
+      venvPath = path.join(venvPath, "bin", "python");
+    } else if (fs.existsSync(path.join(venvPath, "Scripts"))) {
+      venvPath = path.join(venvPath, "Scripts", "python.exe");
+    } else {
+      throw new Error("Virtual environment doesn't exist.");
+    }
+    const commandArgs = [
+      venvPath,
+      `${root}/eng/scripts/setup/run_tsp.py`,
+      `--output-folder=${outputDir}`,
+      `--cadl-file=${yamlPath}`,
+    ];
+    if (resolvedOptions["packaging-files-config"]) {
+      const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map(
+        ([key, value]) => {
+          return `${key}:${value}`;
+        },
+      );
+      commandArgs.push(`--packaging-files-config='${keyValuePairs.join("|")}'`);
+      resolvedOptions["packaging-files-config"] = undefined;
+    }
+    if (
+      resolvedOptions["package-pprint-name"] !== undefined &&
+      !resolvedOptions["package-pprint-name"].startsWith('"')
+    ) {
+      resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`;
+    }
+
+    for (const [key, value] of Object.entries(resolvedOptions)) {
+      commandArgs.push(`--${key}=${value}`);
+    }
+    if (sdkContext.arm === true) {
+      commandArgs.push("--azure-arm=true");
+    }
+    if (resolvedOptions.flavor === "azure") {
+      commandArgs.push("--emit-cross-language-definition-file=true");
+    }
+    commandArgs.push("--from-typespec=true");
+    if (!program.compilerOptions.noEmit && !program.hasError()) {
+      await exec(commandArgs.join(" "));
+    }
   }
 }

packages/http-client-python/emitter/src/lib.ts

@@ -17,6 +17,7 @@ export interface PythonEmitterOptions {
   debug?: boolean;
   flavor?: "azure";
   "examples-dir"?: string;
+  "use-pyodide"?: boolean;
 }
 
 export interface PythonSdkContext<TServiceOperation extends SdkServiceOperation>
@@ -43,6 +44,7 @@ const EmitterOptionsSchema: JSONSchemaType<PythonEmitterOptions> = {
     debug: { type: "boolean", nullable: true },
     flavor: { type: "string", nullable: true },
     "examples-dir": { type: "string", nullable: true, format: "absolute-path" },
+    "use-pyodide": { type: "boolean", nullable: true },
   },
   required: [],
 };

packages/http-client-python/emitter/src/run-python3.ts

@@ -0,0 +1,20 @@
+// This script wraps logic in @azure-tools/extension to resolve
+// the path to Python 3 so that a Python script file can be run
+// from an npm script in package.json.  It uses the same Python 3
+// path resolution algorithm as AutoRest so that the behavior
+// is fully consistent (and also supports AUTOREST_PYTHON_EXE).
+//
+// Invoke it like so: "tsx run-python3.ts script.py"
+
+import cp from "child_process";
+import { patchPythonPath } from "./system-requirements.js";
+
+export async function runPython3(...args: string[]) {
+  const command = await patchPythonPath(["python", ...args], {
+    version: ">=3.8",
+    environmentVariable: "AUTOREST_PYTHON_EXE",
+  });
+  cp.execSync(command.join(" "), {
+    stdio: [0, 1, 2],
+  });
+}