From 277e1996651edb4c160ce5a0a08646b21b57c497 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 17 Oct 2024 14:07:53 -0700 Subject: [PATCH 01/46] Support generating with Pyodide --- .../http-client-python/emitter/src/emitter.ts | 163 +++++++++++++----- .../http-client-python/emitter/src/lib.ts | 2 + .../setup => emitter/src}/run-python3.ts | 7 +- .../src}/system-requirements.ts | 0 .../eng/scripts/ci/regenerate.ts | 6 + packages/http-client-python/package-lock.json | 35 +++- packages/http-client-python/package.json | 5 +- 7 files changed, 166 insertions(+), 52 deletions(-) rename packages/http-client-python/{eng/scripts/setup => emitter/src}/run-python3.ts (77%) rename packages/http-client-python/{eng/scripts/setup => emitter/src}/system-requirements.ts (100%) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 28df53da1f..50483c9598 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -13,6 +13,9 @@ import { emitCodeModel } from "./code-model.js"; import { saveCodeModelAsYaml } from "./external-process.js"; import { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; import { removeUnderscoresFromNamespace } from "./utils.js"; +import { loadPyodide } from "pyodide"; +import jsyaml from "js-yaml"; +import { runPython3 } from "./run-python3.js"; export function getModelsMode(context: SdkContext): "dpg" | "none" { const specifiedModelsMode = context.emitContext.options["models-mode"]; @@ -79,49 +82,125 @@ export async function $onEmit(context: EmitContext) { const outputDir = context.emitterOutputDir; const yamlMap = emitCodeModel(sdkContext); addDefaultOptions(sdkContext); - const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap); - let venvPath = path.join(root, "venv"); - if (fs.existsSync(path.join(venvPath, "bin"))) { - venvPath = path.join(venvPath, "bin", "python"); - } else if (fs.existsSync(path.join(venvPath, "Scripts"))) { - venvPath = path.join(venvPath, "Scripts", "python.exe"); - } else { - throw new Error("Virtual environment doesn't exist."); - } - const commandArgs = [ - venvPath, - `${root}/eng/scripts/setup/run_tsp.py`, - `--output-folder=${outputDir}`, - `--cadl-file=${yamlPath}`, - ]; const resolvedOptions = sdkContext.emitContext.options; - if (resolvedOptions["packaging-files-config"]) { - const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map( - ([key, value]) => { - return `${key}:${value}`; - }, - ); - commandArgs.push(`--packaging-files-config='${keyValuePairs.join("|")}'`); - resolvedOptions["packaging-files-config"] = undefined; - } - if ( - resolvedOptions["package-pprint-name"] !== undefined && - !resolvedOptions["package-pprint-name"].startsWith('"') - ) { - resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`; - } + if (resolvedOptions["use-pyodide"]) { + const commandArgs: Record = {} + if (resolvedOptions["packaging-files-config"]) { + const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map(([key, value]) => { + return `${key}:${value}`; + }); + commandArgs["packaging-files-config"] = keyValuePairs.join("|"); + resolvedOptions["packaging-files-config"] = undefined; + } + if ( + resolvedOptions["package-pprint-name"] !== undefined && + !resolvedOptions["package-pprint-name"].startsWith('"') + ) { + resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`; + } - for (const [key, value] of Object.entries(resolvedOptions)) { - commandArgs.push(`--${key}=${value}`); - } - if (sdkContext.arm === true) { - commandArgs.push("--azure-arm=true"); - } - if (resolvedOptions.flavor === "azure") { - commandArgs.push("--emit-cross-language-definition-file=true"); - } - commandArgs.push("--from-typespec=true"); - if (!program.compilerOptions.noEmit && !program.hasError()) { - execSync(commandArgs.join(" ")); + for (const [key, value] of Object.entries(resolvedOptions)) { + commandArgs[key] = value; + } + if (sdkContext.arm === true) { + commandArgs["azure-arm"]="true"; + } + if (resolvedOptions.flavor === "azure") { + commandArgs["emit-cross-language-definition-file"]="true"; + } + commandArgs["from-typespec"] = "true"; + + if (!program.compilerOptions.noEmit && !program.hasError()) { + const outputFolder = path.relative(root, outputDir) + if (!fs.existsSync(outputFolder)) { + fs.mkdirSync(outputFolder, { recursive: true }); + } + let pyodide = await loadPyodide({indexURL: path.join(root, "node_modules", "pyodide")}); + pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, "."); + const yamlStr = jsyaml.dump(yamlMap); + if (!fs.existsSync("temp")) { + fs.mkdirSync("temp"); + } + const yamlPath = path.join("temp", "yamldata.yaml"); + pyodide.FS.writeFile(yamlPath, yamlStr); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + await micropip.install( + [ + "black", + "click", + "docutils", + "Jinja2", + "m2r2", + "MarkupSafe", + "pathspec", + "platformdirs", + "PyYAML", + "tomli", + "setuptools", + ] + ); + await micropip.install("https://github.com/microsoft/typespec/releases/download/pygen%401.0.0/pygen-0.1.0-py3-none-any.whl"); + const globals = pyodide.toPy({outputFolder, yamlPath, commandArgs}); + const python = ` + async def main(): + from pygen import m2r, preprocess, codegen, black + + m2r.M2R(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() + preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() + codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() + black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process() + + await main() + ` + await pyodide.runPythonAsync(python, {globals}); + } + } else { + await runPython3("./eng/scripts/setup/install.py"); + await runPython3("./eng/scripts/setup/prepare.py"); + let venvPath = path.join(root, "venv"); + if (fs.existsSync(path.join(venvPath, "bin"))) { + venvPath = path.join(venvPath, "bin", "python"); + } else if (fs.existsSync(path.join(venvPath, "Scripts"))) { + venvPath = path.join(venvPath, "Scripts", "python.exe"); + } else { + throw new Error("Virtual environment doesn't exist."); + } + const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap); + const commandArgs = [ + venvPath, + `${root}/eng/scripts/setup/run_tsp.py`, + `--output-folder=${outputDir}`, + `--cadl-file=${yamlPath}`, + ]; + if (resolvedOptions["packaging-files-config"]) { + const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map( + ([key, value]) => { + return `${key}:${value}`; + }, + ); + commandArgs.push(`--packaging-files-config='${keyValuePairs.join("|")}'`); + resolvedOptions["packaging-files-config"] = undefined; + } + if ( + resolvedOptions["package-pprint-name"] !== undefined && + !resolvedOptions["package-pprint-name"].startsWith('"') + ) { + resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`; + } + + for (const [key, value] of Object.entries(resolvedOptions)) { + commandArgs.push(`--${key}=${value}`); + } + if (sdkContext.arm === true) { + commandArgs.push("--azure-arm=true"); + } + if (resolvedOptions.flavor === "azure") { + commandArgs.push("--emit-cross-language-definition-file=true"); + } + commandArgs.push("--from-typespec=true"); + if (!program.compilerOptions.noEmit && !program.hasError()) { + execSync(commandArgs.join(" ")); + } } } diff --git a/packages/http-client-python/emitter/src/lib.ts b/packages/http-client-python/emitter/src/lib.ts index 91aed8fb4b..d83619f371 100644 --- a/packages/http-client-python/emitter/src/lib.ts +++ b/packages/http-client-python/emitter/src/lib.ts @@ -17,6 +17,7 @@ export interface PythonEmitterOptions { debug?: boolean; flavor?: "azure"; "examples-dir"?: string; + "use-pyodide"?: boolean; } export interface PythonSdkContext @@ -43,6 +44,7 @@ const EmitterOptionsSchema: JSONSchemaType = { debug: { type: "boolean", nullable: true }, flavor: { type: "string", nullable: true }, "examples-dir": { type: "string", nullable: true, format: "absolute-path" }, + "use-pyodide": { type: "boolean", nullable: true }, }, required: [], }; diff --git a/packages/http-client-python/eng/scripts/setup/run-python3.ts b/packages/http-client-python/emitter/src/run-python3.ts similarity index 77% rename from packages/http-client-python/eng/scripts/setup/run-python3.ts rename to packages/http-client-python/emitter/src/run-python3.ts index 6de4922a40..e4df6e8a71 100644 --- a/packages/http-client-python/eng/scripts/setup/run-python3.ts +++ b/packages/http-client-python/emitter/src/run-python3.ts @@ -9,7 +9,7 @@ import cp from "child_process"; import { patchPythonPath } from "./system-requirements.js"; -async function runPython3(...args: string[]) { +export async function runPython3(...args: string[]) { const command = await patchPythonPath(["python", ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE", @@ -18,8 +18,3 @@ async function runPython3(...args: string[]) { stdio: [0, 1, 2], }); } - -runPython3(...process.argv.slice(2)).catch((err) => { - console.error(err.toString()); // eslint-disable-line no-console - process.exit(1); -}); diff --git a/packages/http-client-python/eng/scripts/setup/system-requirements.ts b/packages/http-client-python/emitter/src/system-requirements.ts similarity index 100% rename from packages/http-client-python/eng/scripts/setup/system-requirements.ts rename to packages/http-client-python/emitter/src/system-requirements.ts diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index 629c29c411..000a3a9400 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -16,6 +16,7 @@ const argv = parseArgs({ pluginDir: { type: "string" }, emitterName: { type: "string" }, generatedFolder: { type: "string" }, + pyodide: { type: "boolean" }, }, }); @@ -158,12 +159,14 @@ interface RegenerateFlagsInput { flavor?: string; debug?: boolean; name?: string; + pyodide?: boolean; } interface RegenerateFlags { flavor: string; debug: boolean; name?: string; + pyodide?: boolean; } const SpecialFlags: Record> = { @@ -242,6 +245,9 @@ function addOptions( const emitterConfigs: EmitterConfig[] = []; for (const config of getEmitterOption(spec)) { const options: Record = { ...config }; + if (flags.pyodide) { + options["use-pyodide"] = "true"; + } options["flavor"] = flags.flavor; for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) { options[k] = v; diff --git a/packages/http-client-python/package-lock.json b/packages/http-client-python/package-lock.json index 51dd110ce6..c33a70de3f 100644 --- a/packages/http-client-python/package-lock.json +++ b/packages/http-client-python/package-lock.json @@ -7,10 +7,10 @@ "": { "name": "@typespec/http-client-python", "version": "0.3.1", - "hasInstallScript": true, "license": "MIT", "dependencies": { "js-yaml": "~4.1.0", + "pyodide": "0.26.2", "semver": "~7.6.2" }, "devDependencies": { @@ -5642,6 +5642,18 @@ "node": ">=6" } }, + "node_modules/pyodide": { + "version": "0.26.2", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.26.2.tgz", + "integrity": "sha512-8VCRdFX83gBsWs6XP2rhG8HMaB+JaVyyav4q/EMzoV8fXH8HN6T5IISC92SNma6i1DRA3SVXA61S1rJcB8efgA==", + "license": "Apache-2.0", + "dependencies": { + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/qs": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", @@ -7633,6 +7645,27 @@ "node": ">=8" } }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xml-formatter": { "version": "3.6.3", "resolved": "https://registry.npmjs.org/xml-formatter/-/xml-formatter-3.6.3.tgz", diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index e679a11886..7e33906be0 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -35,8 +35,6 @@ "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", - "install": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", - "prepare": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", "ci:generator": "tsx ./eng/scripts/ci/run-ci.ts", @@ -64,7 +62,8 @@ }, "dependencies": { "js-yaml": "~4.1.0", - "semver": "~7.6.2" + "semver": "~7.6.2", + "pyodide": "0.26.2" }, "devDependencies": { "@azure-tools/typespec-azure-resource-manager": "~0.47.0", From f7dc98aea4130d6cad445da76fb6c78ba6e7eb77 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 17 Oct 2024 14:57:33 -0700 Subject: [PATCH 02/46] Fix spell check --- cspell.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cspell.yaml b/cspell.yaml index 434b41a6e7..c9a1036a8a 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -245,6 +245,10 @@ words: - xplat - xxsubtype - yamls + - pyodide + - pyimport + - NODEFS + - yamldata ignorePaths: - "**/node_modules/**" - "**/dist/**" From 98df4ba8224c6ef95cb6f2856ebe38f194b483c2 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 17 Oct 2024 14:57:40 -0700 Subject: [PATCH 03/46] Fix format --- .../http-client-python/emitter/src/emitter.ts | 100 +++++++++--------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 50483c9598..8d6407c455 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -7,15 +7,15 @@ import { import { EmitContext } from "@typespec/compiler"; import { execSync } from "child_process"; import fs from "fs"; +import jsyaml from "js-yaml"; import path, { dirname } from "path"; +import { loadPyodide } from "pyodide"; import { fileURLToPath } from "url"; import { emitCodeModel } from "./code-model.js"; import { saveCodeModelAsYaml } from "./external-process.js"; import { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; -import { removeUnderscoresFromNamespace } from "./utils.js"; -import { loadPyodide } from "pyodide"; -import jsyaml from "js-yaml"; import { runPython3 } from "./run-python3.js"; +import { removeUnderscoresFromNamespace } from "./utils.js"; export function getModelsMode(context: SdkContext): "dpg" | "none" { const specifiedModelsMode = context.emitContext.options["models-mode"]; @@ -84,65 +84,67 @@ export async function $onEmit(context: EmitContext) { addDefaultOptions(sdkContext); const resolvedOptions = sdkContext.emitContext.options; if (resolvedOptions["use-pyodide"]) { - const commandArgs: Record = {} + const commandArgs: Record = {}; if (resolvedOptions["packaging-files-config"]) { - const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map(([key, value]) => { - return `${key}:${value}`; - }); - commandArgs["packaging-files-config"] = keyValuePairs.join("|"); - resolvedOptions["packaging-files-config"] = undefined; + const keyValuePairs = Object.entries(resolvedOptions["packaging-files-config"]).map( + ([key, value]) => { + return `${key}:${value}`; + }, + ); + commandArgs["packaging-files-config"] = keyValuePairs.join("|"); + resolvedOptions["packaging-files-config"] = undefined; } if ( - resolvedOptions["package-pprint-name"] !== undefined && - !resolvedOptions["package-pprint-name"].startsWith('"') + resolvedOptions["package-pprint-name"] !== undefined && + !resolvedOptions["package-pprint-name"].startsWith('"') ) { - resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`; + resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`; } for (const [key, value] of Object.entries(resolvedOptions)) { - commandArgs[key] = value; + commandArgs[key] = value; } if (sdkContext.arm === true) { - commandArgs["azure-arm"]="true"; + commandArgs["azure-arm"] = "true"; } if (resolvedOptions.flavor === "azure") { - commandArgs["emit-cross-language-definition-file"]="true"; + commandArgs["emit-cross-language-definition-file"] = "true"; } commandArgs["from-typespec"] = "true"; if (!program.compilerOptions.noEmit && !program.hasError()) { - const outputFolder = path.relative(root, outputDir) - if (!fs.existsSync(outputFolder)) { - fs.mkdirSync(outputFolder, { recursive: true }); - } - let pyodide = await loadPyodide({indexURL: path.join(root, "node_modules", "pyodide")}); - pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, "."); - const yamlStr = jsyaml.dump(yamlMap); - if (!fs.existsSync("temp")) { - fs.mkdirSync("temp"); - } - const yamlPath = path.join("temp", "yamldata.yaml"); - pyodide.FS.writeFile(yamlPath, yamlStr); - await pyodide.loadPackage("micropip"); - const micropip = pyodide.pyimport("micropip"); - await micropip.install( - [ - "black", - "click", - "docutils", - "Jinja2", - "m2r2", - "MarkupSafe", - "pathspec", - "platformdirs", - "PyYAML", - "tomli", - "setuptools", - ] - ); - await micropip.install("https://github.com/microsoft/typespec/releases/download/pygen%401.0.0/pygen-0.1.0-py3-none-any.whl"); - const globals = pyodide.toPy({outputFolder, yamlPath, commandArgs}); - const python = ` + const outputFolder = path.relative(root, outputDir); + if (!fs.existsSync(outputFolder)) { + fs.mkdirSync(outputFolder, { recursive: true }); + } + let pyodide = await loadPyodide({ indexURL: path.join(root, "node_modules", "pyodide") }); + pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, "."); + const yamlStr = jsyaml.dump(yamlMap); + if (!fs.existsSync("temp")) { + fs.mkdirSync("temp"); + } + const yamlPath = path.join("temp", "yamldata.yaml"); + pyodide.FS.writeFile(yamlPath, yamlStr); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + await micropip.install([ + "black", + "click", + "docutils", + "Jinja2", + "m2r2", + "MarkupSafe", + "pathspec", + "platformdirs", + "PyYAML", + "tomli", + "setuptools", + ]); + await micropip.install( + "https://github.com/microsoft/typespec/releases/download/pygen%401.0.0/pygen-0.1.0-py3-none-any.whl", + ); + const globals = pyodide.toPy({ outputFolder, yamlPath, commandArgs }); + const python = ` async def main(): from pygen import m2r, preprocess, codegen, black @@ -152,8 +154,8 @@ export async function $onEmit(context: EmitContext) { black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process() await main() - ` - await pyodide.runPythonAsync(python, {globals}); + `; + await pyodide.runPythonAsync(python, { globals }); } } else { await runPython3("./eng/scripts/setup/install.py"); From b41d46c14281a9a7e54d399af3efa624d48b7e6b Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 17 Oct 2024 15:08:14 -0700 Subject: [PATCH 04/46] Fix lint --- packages/http-client-python/emitter/src/emitter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 8d6407c455..dbe197f78a 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -117,7 +117,7 @@ export async function $onEmit(context: EmitContext) { if (!fs.existsSync(outputFolder)) { fs.mkdirSync(outputFolder, { recursive: true }); } - let pyodide = await loadPyodide({ indexURL: path.join(root, "node_modules", "pyodide") }); + const pyodide = await loadPyodide({ indexURL: path.join(root, "node_modules", "pyodide") }); pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, "."); const yamlStr = jsyaml.dump(yamlMap); if (!fs.existsSync("temp")) { From f8a14d689a3cc8ddf0caa9100ad58baa08ecf4e7 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 18 Oct 2024 10:45:36 -0700 Subject: [PATCH 05/46] Use pygen wheel from fork --- packages/http-client-python/emitter/src/emitter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index dbe197f78a..ae6c310cd1 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -141,7 +141,7 @@ export async function $onEmit(context: EmitContext) { "setuptools", ]); await micropip.install( - "https://github.com/microsoft/typespec/releases/download/pygen%401.0.0/pygen-0.1.0-py3-none-any.whl", + "https://github.com/YalinLi0312/typespec/releases/download/pygen%40v0.1.0/pygen-0.1.0-py3-none-any.whl", ); const globals = pyodide.toPy({ outputFolder, yamlPath, commandArgs }); const python = ` From 60896776713b73f7d5f4a70c958f2ec6fa60068d Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 18 Oct 2024 16:44:20 -0700 Subject: [PATCH 06/46] Align dependencies versions --- .../http-client-python/emitter/src/emitter.ts | 6 +++--- .../generator/requirements.txt | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index ae6c310cd1..7ee63139e9 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -130,9 +130,9 @@ export async function $onEmit(context: EmitContext) { await micropip.install([ "black", "click", - "docutils", - "Jinja2", - "m2r2", + "docutils==0.21.2", + "Jinja2==3.1.4", + "m2r2==0.3.3.post2", "MarkupSafe", "pathspec", "platformdirs", diff --git a/packages/http-client-python/generator/requirements.txt b/packages/http-client-python/generator/requirements.txt index bebbbb5a64..f5cbc53ec6 100644 --- a/packages/http-client-python/generator/requirements.txt +++ b/packages/http-client-python/generator/requirements.txt @@ -1,12 +1,12 @@ -black==24.4.0 -click==8.1.3 -docutils==0.19 +black==24.10.0 +click==8.1.7 +docutils==0.21.2 Jinja2==3.1.4 -m2r2==0.3.3 -MarkupSafe==2.1.2 +m2r2==0.3.3.post2 +MarkupSafe==2.1.5 mistune==0.8.4 -pathspec==0.11.1 -platformdirs==3.2.0 +pathspec==0.12.1 +platformdirs==4.3.6 PyYAML==6.0.1 tomli==2.0.1 -setuptools==69.2.0 +setuptools==69.5.1 From db80270237fd73a1d5136574b82462fbb72f5efc Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Wed, 23 Oct 2024 11:06:56 -0700 Subject: [PATCH 07/46] Align dev requirements version --- packages/http-client-python/generator/dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/dev_requirements.txt b/packages/http-client-python/generator/dev_requirements.txt index 76e1bc490b..568aa0eac6 100644 --- a/packages/http-client-python/generator/dev_requirements.txt +++ b/packages/http-client-python/generator/dev_requirements.txt @@ -8,6 +8,6 @@ colorama==0.4.6 debugpy==1.8.2 pytest==8.3.2 coverage==7.6.1 -black==24.8.0 +black==24.10.0 ptvsd==4.3.2 types-PyYAML==6.0.12.8 From 18422e84054945415b76a183f4f118cf52ff5193 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 24 Oct 2024 14:31:18 -0700 Subject: [PATCH 08/46] Update run-python3.ts --- packages/http-client-python/emitter/src/run-python3.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/emitter/src/run-python3.ts b/packages/http-client-python/emitter/src/run-python3.ts index e4df6e8a71..650ee9e155 100644 --- a/packages/http-client-python/emitter/src/run-python3.ts +++ b/packages/http-client-python/emitter/src/run-python3.ts @@ -14,7 +14,13 @@ export async function runPython3(...args: string[]) { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE", }); - cp.execSync(command.join(" "), { - stdio: [0, 1, 2], + cp.exec(command.join(" "), + (error: cp.ExecException | null, stdout: string, stderr: string) => { + if (error) { + console.error(`Error: ${error.message}`); // eslint-disable-line no-console + console.error(`stderr: ${stderr}`); // eslint-disable-line no-console + process.exit(1); + } + console.log(`stdout: ${stdout}`); // eslint-disable-line no-console }); } From 4f7594fab1802f5139605da1ff929c49efaab2a3 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 25 Oct 2024 16:47:30 -0700 Subject: [PATCH 09/46] Wait for runPython3 subprocess complete --- .../emitter/src/run-python3.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/http-client-python/emitter/src/run-python3.ts b/packages/http-client-python/emitter/src/run-python3.ts index 650ee9e155..2f94df45a5 100644 --- a/packages/http-client-python/emitter/src/run-python3.ts +++ b/packages/http-client-python/emitter/src/run-python3.ts @@ -7,6 +7,7 @@ // Invoke it like so: "tsx run-python3.ts script.py" import cp from "child_process"; +import util from "util"; import { patchPythonPath } from "./system-requirements.js"; export async function runPython3(...args: string[]) { @@ -14,13 +15,14 @@ export async function runPython3(...args: string[]) { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE", }); - cp.exec(command.join(" "), - (error: cp.ExecException | null, stdout: string, stderr: string) => { - if (error) { - console.error(`Error: ${error.message}`); // eslint-disable-line no-console - console.error(`stderr: ${stderr}`); // eslint-disable-line no-console - process.exit(1); + const execPromise = util.promisify(cp.exec); + try { + const { stdout, stderr } = await execPromise(command.join(" ")); + console.log('Output:', stdout); + if (stderr) { + console.error('Error:', stderr); } - console.log(`stdout: ${stdout}`); // eslint-disable-line no-console - }); + } catch (error) { + console.error('Execution error:', error); + } } From b7464b62dc8bd2ed92418f15c01406814a5a8ed7 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 25 Oct 2024 16:49:14 -0700 Subject: [PATCH 10/46] Run install and prepare only in first generate --- packages/http-client-python/emitter/src/emitter.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 7ee63139e9..3a9097a9eb 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -158,9 +158,11 @@ export async function $onEmit(context: EmitContext) { await pyodide.runPythonAsync(python, { globals }); } } else { - await runPython3("./eng/scripts/setup/install.py"); - await runPython3("./eng/scripts/setup/prepare.py"); let venvPath = path.join(root, "venv"); + if (!fs.existsSync(path.join(venvPath))) { + await runPython3("./eng/scripts/setup/install.py"); + await runPython3("./eng/scripts/setup/prepare.py"); + } if (fs.existsSync(path.join(venvPath, "bin"))) { venvPath = path.join(venvPath, "bin", "python"); } else if (fs.existsSync(path.join(venvPath, "Scripts"))) { From e66c54121600cb96dc2705a203651326760b7f95 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 25 Oct 2024 16:50:29 -0700 Subject: [PATCH 11/46] Ignore SyntaxWarning from mistune 0.8.4 --- packages/http-client-python/emitter/src/emitter.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 3a9097a9eb..2b7b2f5242 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -146,6 +146,9 @@ export async function $onEmit(context: EmitContext) { const globals = pyodide.toPy({ outputFolder, yamlPath, commandArgs }); const python = ` async def main(): + import warnings + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) from pygen import m2r, preprocess, codegen, black m2r.M2R(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() From cd69beb270d2a6c85e1b7a3999e69a7ac38ddf0b Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 29 Oct 2024 16:05:08 -0700 Subject: [PATCH 12/46] Fix lint --- packages/http-client-python/emitter/src/run-python3.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/http-client-python/emitter/src/run-python3.ts b/packages/http-client-python/emitter/src/run-python3.ts index 2f94df45a5..087a1c779a 100644 --- a/packages/http-client-python/emitter/src/run-python3.ts +++ b/packages/http-client-python/emitter/src/run-python3.ts @@ -18,11 +18,11 @@ export async function runPython3(...args: string[]) { const execPromise = util.promisify(cp.exec); try { const { stdout, stderr } = await execPromise(command.join(" ")); - console.log('Output:', stdout); + console.log("Output:", stdout); // eslint-disable-line no-console if (stderr) { - console.error('Error:', stderr); + console.error("Error:", stderr); // eslint-disable-line no-console } } catch (error) { - console.error('Execution error:', error); + console.error("Execution error:", error); // eslint-disable-line no-console } } From 04f6507961ffaa773f72c8a1ba7fcf7d7db45797 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 29 Oct 2024 16:05:43 -0700 Subject: [PATCH 13/46] Run codegen command async --- .../http-client-python/emitter/src/emitter.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 2b7b2f5242..cfb7b512fd 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -5,7 +5,7 @@ import { SdkServiceOperation, } from "@azure-tools/typespec-client-generator-core"; import { EmitContext } from "@typespec/compiler"; -import { execSync } from "child_process"; +import { exec } from "child_process"; import fs from "fs"; import jsyaml from "js-yaml"; import path, { dirname } from "path"; @@ -146,15 +146,15 @@ export async function $onEmit(context: EmitContext) { const globals = pyodide.toPy({ outputFolder, yamlPath, commandArgs }); const python = ` async def main(): - import warnings - with warnings.catch_warnings(): - warnings.simplefilter("ignore", SyntaxWarning) + import warnings + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) from pygen import m2r, preprocess, codegen, black - m2r.M2R(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() - preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() - codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() - black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process() + m2r.M2R(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() + preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() + codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() + black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process() await main() `; @@ -207,7 +207,7 @@ export async function $onEmit(context: EmitContext) { } commandArgs.push("--from-typespec=true"); if (!program.compilerOptions.noEmit && !program.hasError()) { - execSync(commandArgs.join(" ")); + await exec(commandArgs.join(" ")); } } } From ac1c1bde90bfcfcd90ef5379787d94184f0472f1 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 7 Nov 2024 12:51:13 -0800 Subject: [PATCH 14/46] Prepare venv and install pygen deps in build step --- .../http-client-python/emitter/src/emitter.ts | 5 ---- .../src => eng/scripts/setup}/run-python3.ts | 23 ++++++++----------- .../scripts/setup}/system-requirements.ts | 0 packages/http-client-python/package.json | 4 +++- 4 files changed, 13 insertions(+), 19 deletions(-) rename packages/http-client-python/{emitter/src => eng/scripts/setup}/run-python3.ts (50%) rename packages/http-client-python/{emitter/src => eng/scripts/setup}/system-requirements.ts (100%) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index cfb7b512fd..c892050ff4 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -14,7 +14,6 @@ import { fileURLToPath } from "url"; import { emitCodeModel } from "./code-model.js"; import { saveCodeModelAsYaml } from "./external-process.js"; import { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; -import { runPython3 } from "./run-python3.js"; import { removeUnderscoresFromNamespace } from "./utils.js"; export function getModelsMode(context: SdkContext): "dpg" | "none" { @@ -162,10 +161,6 @@ export async function $onEmit(context: EmitContext) { } } else { let venvPath = path.join(root, "venv"); - if (!fs.existsSync(path.join(venvPath))) { - await runPython3("./eng/scripts/setup/install.py"); - await runPython3("./eng/scripts/setup/prepare.py"); - } if (fs.existsSync(path.join(venvPath, "bin"))) { venvPath = path.join(venvPath, "bin", "python"); } else if (fs.existsSync(path.join(venvPath, "Scripts"))) { diff --git a/packages/http-client-python/emitter/src/run-python3.ts b/packages/http-client-python/eng/scripts/setup/run-python3.ts similarity index 50% rename from packages/http-client-python/emitter/src/run-python3.ts rename to packages/http-client-python/eng/scripts/setup/run-python3.ts index 087a1c779a..bce337b969 100644 --- a/packages/http-client-python/emitter/src/run-python3.ts +++ b/packages/http-client-python/eng/scripts/setup/run-python3.ts @@ -7,22 +7,19 @@ // Invoke it like so: "tsx run-python3.ts script.py" import cp from "child_process"; -import util from "util"; -import { patchPythonPath } from "./system-requirements.js"; +import { patchPythonPath } from "./system-requirements"; -export async function runPython3(...args: string[]) { +async function runPython3(...args: string[]) { const command = await patchPythonPath(["python", ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE", }); - const execPromise = util.promisify(cp.exec); - try { - const { stdout, stderr } = await execPromise(command.join(" ")); - console.log("Output:", stdout); // eslint-disable-line no-console - if (stderr) { - console.error("Error:", stderr); // eslint-disable-line no-console - } - } catch (error) { - console.error("Execution error:", error); // eslint-disable-line no-console - } + cp.execSync(command.join(" "), { + stdio: [0, 1, 2], + }); } + +runPython3(...process.argv.slice(2)).catch((err) => { + console.error(err.toString()); // eslint-disable-line no-console + process.exit(1); +}); diff --git a/packages/http-client-python/emitter/src/system-requirements.ts b/packages/http-client-python/eng/scripts/setup/system-requirements.ts similarity index 100% rename from packages/http-client-python/emitter/src/system-requirements.ts rename to packages/http-client-python/eng/scripts/setup/system-requirements.ts diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 86d25e916d..804ec91725 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -30,11 +30,13 @@ }, "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", - "build": "tsc -p ./emitter/tsconfig.build.json", + "build": "tsc -p ./emitter/tsconfig.build.json && npm run install_pygen_deps && npm run install_pygen_dev_deps", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", + "install_pygen_deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", + "install_pygen_dev_deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", "ci:generator": "tsx ./eng/scripts/ci/run-ci.ts", From 5015c10714436489bd2d1bb7495be24263563bcf Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 7 Nov 2024 17:39:29 -0800 Subject: [PATCH 15/46] Update package.json --- packages/http-client-python/package.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 5fd84020db..bca53a1b17 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -30,13 +30,11 @@ }, "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", - "build": "tsc -p ./emitter/tsconfig.build.json && npm run install_pygen_deps && npm run install_pygen_dev_deps", + "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", - "install_pygen_deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", - "install_pygen_dev_deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", "ci:generator": "tsx ./eng/scripts/ci/run-ci.ts", From 947cc5613e3c158c87e20d344eadb413b639c4d9 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 12:15:22 -0800 Subject: [PATCH 16/46] Run build with --verbose --- packages/http-client-python/package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index bca53a1b17..0655f67035 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -30,7 +30,10 @@ }, "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", - "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", + "venv:install-reqs": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", + "venv:install-dev-reqs": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", + "build-python-wheel": "tsx ./eng/scripts/setup/run-python3.ts build_pygen_wheel.py", + "build": "tsc -p ./emitter/tsconfig.build.json && npm run venv:install-reqs && npm run venv:install-dev-reqs && npm run build-python-wheel --verbose", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", From ae1aca4eb5afeac82ed74e0a3b9ea3dca6faa5dc Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 15:34:10 -0800 Subject: [PATCH 17/46] Build and use local pygen wheel --- cspell.yaml | 1 + packages/http-client-python/emitter/src/emitter.ts | 4 +--- .../eng/scripts/setup/run-python3.ts | 2 +- .../eng/scripts/setup/venvtools.py | 13 +++++++++++++ .../generator/dev_requirements.txt | 1 + packages/http-client-python/package-lock.json | 1 + packages/http-client-python/package.json | 2 +- 7 files changed, 19 insertions(+), 5 deletions(-) diff --git a/cspell.yaml b/cspell.yaml index 6bd1fe96d0..cf044b82a8 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -251,6 +251,7 @@ words: - pyimport - NODEFS - yamldata + - reqs ignorePaths: - "**/node_modules/**" - "**/dist/**" diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index c892050ff4..064718f2a0 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -138,10 +138,8 @@ export async function $onEmit(context: EmitContext) { "PyYAML", "tomli", "setuptools", + "emfs:generator/dist/pygen-0.1.0-py3-none-any.whl", ]); - await micropip.install( - "https://github.com/YalinLi0312/typespec/releases/download/pygen%40v0.1.0/pygen-0.1.0-py3-none-any.whl", - ); const globals = pyodide.toPy({ outputFolder, yamlPath, commandArgs }); const python = ` async def main(): diff --git a/packages/http-client-python/eng/scripts/setup/run-python3.ts b/packages/http-client-python/eng/scripts/setup/run-python3.ts index bce337b969..6de4922a40 100644 --- a/packages/http-client-python/eng/scripts/setup/run-python3.ts +++ b/packages/http-client-python/eng/scripts/setup/run-python3.ts @@ -7,7 +7,7 @@ // Invoke it like so: "tsx run-python3.ts script.py" import cp from "child_process"; -import { patchPythonPath } from "./system-requirements"; +import { patchPythonPath } from "./system-requirements.js"; async function runPython3(...args: string[]) { const command = await patchPythonPath(["python", ...args], { diff --git a/packages/http-client-python/eng/scripts/setup/venvtools.py b/packages/http-client-python/eng/scripts/setup/venvtools.py index c79ce4946b..75ff23c6d7 100644 --- a/packages/http-client-python/eng/scripts/setup/venvtools.py +++ b/packages/http-client-python/eng/scripts/setup/venvtools.py @@ -85,3 +85,16 @@ def python_run(venv_context, module, command=None, *, additional_dir="."): except subprocess.CalledProcessError as err: print(err) sys.exit(1) + +def build_wheel(venv_context, command, *, additional_dir="."): + try: + cmd_line = [venv_context.env_exe] + command + print("Executing: {}".format(" ".join(cmd_line))) + subprocess.run( + cmd_line, + cwd=_ROOT_DIR / additional_dir, + check=True, + ) + except subprocess.CalledProcessError as err: + print(err) + sys.exit(1) diff --git a/packages/http-client-python/generator/dev_requirements.txt b/packages/http-client-python/generator/dev_requirements.txt index 3938c53a7c..903f828e21 100644 --- a/packages/http-client-python/generator/dev_requirements.txt +++ b/packages/http-client-python/generator/dev_requirements.txt @@ -11,3 +11,4 @@ coverage==7.6.1 black==24.10.0 ptvsd==4.3.2 types-PyYAML==6.0.12.8 +wheel==0.44.0 diff --git a/packages/http-client-python/package-lock.json b/packages/http-client-python/package-lock.json index bce99213fc..97be78b58e 100644 --- a/packages/http-client-python/package-lock.json +++ b/packages/http-client-python/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "js-yaml": "~4.1.0", + "pyodide": "0.26.2", "semver": "~7.6.2", "tsx": "~4.19.1" }, diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 0655f67035..c99d2dac2e 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -32,7 +32,7 @@ "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", "venv:install-reqs": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", "venv:install-dev-reqs": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", - "build-python-wheel": "tsx ./eng/scripts/setup/run-python3.ts build_pygen_wheel.py", + "build-python-wheel": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py", "build": "tsc -p ./emitter/tsconfig.build.json && npm run venv:install-reqs && npm run venv:install-dev-reqs && npm run build-python-wheel --verbose", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", From ebfa971e54c8928a5a17e0c7edda03f7ba426db9 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 17:48:19 -0800 Subject: [PATCH 18/46] Revert to use remote pygen wheel --- cspell.yaml | 1 - packages/http-client-python/emitter/src/emitter.ts | 4 +++- .../http-client-python/eng/scripts/setup/run-python3.ts | 2 +- packages/http-client-python/generator/dev_requirements.txt | 1 - packages/http-client-python/package.json | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cspell.yaml b/cspell.yaml index cf044b82a8..6bd1fe96d0 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -251,7 +251,6 @@ words: - pyimport - NODEFS - yamldata - - reqs ignorePaths: - "**/node_modules/**" - "**/dist/**" diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 064718f2a0..aed8b21263 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -138,8 +138,10 @@ export async function $onEmit(context: EmitContext) { "PyYAML", "tomli", "setuptools", - "emfs:generator/dist/pygen-0.1.0-py3-none-any.whl", ]); + await micropip.install( + "https://github.com/YalinLi0312/typespec/releases/download/pygen%40v0.1.0/pygen-0.1.0-py3-none-any.whl", + ); const globals = pyodide.toPy({ outputFolder, yamlPath, commandArgs }); const python = ` async def main(): diff --git a/packages/http-client-python/eng/scripts/setup/run-python3.ts b/packages/http-client-python/eng/scripts/setup/run-python3.ts index 6de4922a40..bce337b969 100644 --- a/packages/http-client-python/eng/scripts/setup/run-python3.ts +++ b/packages/http-client-python/eng/scripts/setup/run-python3.ts @@ -7,7 +7,7 @@ // Invoke it like so: "tsx run-python3.ts script.py" import cp from "child_process"; -import { patchPythonPath } from "./system-requirements.js"; +import { patchPythonPath } from "./system-requirements"; async function runPython3(...args: string[]) { const command = await patchPythonPath(["python", ...args], { diff --git a/packages/http-client-python/generator/dev_requirements.txt b/packages/http-client-python/generator/dev_requirements.txt index 903f828e21..3938c53a7c 100644 --- a/packages/http-client-python/generator/dev_requirements.txt +++ b/packages/http-client-python/generator/dev_requirements.txt @@ -11,4 +11,3 @@ coverage==7.6.1 black==24.10.0 ptvsd==4.3.2 types-PyYAML==6.0.12.8 -wheel==0.44.0 diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index c99d2dac2e..84d5fb615b 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -30,14 +30,14 @@ }, "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", - "venv:install-reqs": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", - "venv:install-dev-reqs": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "build-python-wheel": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py", - "build": "tsc -p ./emitter/tsconfig.build.json && npm run venv:install-reqs && npm run venv:install-dev-reqs && npm run build-python-wheel --verbose", + "build": "tsc -p ./emitter/tsconfig.build.json && npm run install_pygen_deps && npm run install_pygen_dev_deps", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", + "install_pygen_deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", + "install_pygen_dev_deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", "ci:generator": "tsx ./eng/scripts/ci/run-ci.ts", From 222167ed8e87183464fbacf5eee49bdc63787bf2 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 19:12:56 -0800 Subject: [PATCH 19/46] Build with --verbose --- packages/http-client-python/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 84d5fb615b..668cc94ffe 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -31,7 +31,7 @@ "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", "build-python-wheel": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py", - "build": "tsc -p ./emitter/tsconfig.build.json && npm run install_pygen_deps && npm run install_pygen_dev_deps", + "build": "tsc -p ./emitter/tsconfig.build.json && npm run install_pygen_deps --verbose && npm run install_pygen_dev_deps --verbose", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", From 8d0a423bf14665fbab25d9be927b56fd08018d59 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 21:15:06 -0800 Subject: [PATCH 20/46] Update dev_requirements.txt --- packages/http-client-python/generator/dev_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/http-client-python/generator/dev_requirements.txt b/packages/http-client-python/generator/dev_requirements.txt index 3938c53a7c..903f828e21 100644 --- a/packages/http-client-python/generator/dev_requirements.txt +++ b/packages/http-client-python/generator/dev_requirements.txt @@ -11,3 +11,4 @@ coverage==7.6.1 black==24.10.0 ptvsd==4.3.2 types-PyYAML==6.0.12.8 +wheel==0.44.0 From 379d9a85b01b4ed7c11f69a7d78335818f3c49ff Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 21:30:34 -0800 Subject: [PATCH 21/46] Use black 24.8.0 --- packages/http-client-python/generator/dev_requirements.txt | 2 +- packages/http-client-python/generator/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/dev_requirements.txt b/packages/http-client-python/generator/dev_requirements.txt index 903f828e21..3920cfa6b6 100644 --- a/packages/http-client-python/generator/dev_requirements.txt +++ b/packages/http-client-python/generator/dev_requirements.txt @@ -8,7 +8,7 @@ colorama==0.4.6 debugpy==1.8.2 pytest==8.3.2 coverage==7.6.1 -black==24.10.0 +black==24.8.0 ptvsd==4.3.2 types-PyYAML==6.0.12.8 wheel==0.44.0 diff --git a/packages/http-client-python/generator/requirements.txt b/packages/http-client-python/generator/requirements.txt index f5cbc53ec6..39e092bcb4 100644 --- a/packages/http-client-python/generator/requirements.txt +++ b/packages/http-client-python/generator/requirements.txt @@ -1,4 +1,4 @@ -black==24.10.0 +black==24.8.0 click==8.1.7 docutils==0.21.2 Jinja2==3.1.4 From 69487a2ac8dfead668d792f61105a9034573d3de Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 21:36:10 -0800 Subject: [PATCH 22/46] Build and use local pygen wheel --- packages/http-client-python/emitter/src/emitter.ts | 4 +--- packages/http-client-python/package.json | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index aed8b21263..064718f2a0 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -138,10 +138,8 @@ export async function $onEmit(context: EmitContext) { "PyYAML", "tomli", "setuptools", + "emfs:generator/dist/pygen-0.1.0-py3-none-any.whl", ]); - await micropip.install( - "https://github.com/YalinLi0312/typespec/releases/download/pygen%40v0.1.0/pygen-0.1.0-py3-none-any.whl", - ); const globals = pyodide.toPy({ outputFolder, yamlPath, commandArgs }); const python = ` async def main(): diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 668cc94ffe..01ce8763e2 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -31,7 +31,7 @@ "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", "build-python-wheel": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py", - "build": "tsc -p ./emitter/tsconfig.build.json && npm run install_pygen_deps --verbose && npm run install_pygen_dev_deps --verbose", + "build": "tsc -p ./emitter/tsconfig.build.json && npm run install_pygen_deps && npm run install_pygen_dev_deps && npm run build_pygen_wheel", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", From 0620a37e23b08542826a9fa834a3af3bd573a765 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 21:45:43 -0800 Subject: [PATCH 23/46] Update build cmd --- cspell.yaml | 1 + .../http-client-python/eng/scripts/setup/run-python3.ts | 2 +- packages/http-client-python/package.json | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cspell.yaml b/cspell.yaml index 6bd1fe96d0..d9cc592b62 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -251,6 +251,7 @@ words: - pyimport - NODEFS - yamldata + - deps ignorePaths: - "**/node_modules/**" - "**/dist/**" diff --git a/packages/http-client-python/eng/scripts/setup/run-python3.ts b/packages/http-client-python/eng/scripts/setup/run-python3.ts index bce337b969..6de4922a40 100644 --- a/packages/http-client-python/eng/scripts/setup/run-python3.ts +++ b/packages/http-client-python/eng/scripts/setup/run-python3.ts @@ -7,7 +7,7 @@ // Invoke it like so: "tsx run-python3.ts script.py" import cp from "child_process"; -import { patchPythonPath } from "./system-requirements"; +import { patchPythonPath } from "./system-requirements.js"; async function runPython3(...args: string[]) { const command = await patchPythonPath(["python", ...args], { diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 01ce8763e2..50844f006c 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -31,13 +31,13 @@ "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", "build-python-wheel": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py", - "build": "tsc -p ./emitter/tsconfig.build.json && npm run install_pygen_deps && npm run install_pygen_dev_deps && npm run build_pygen_wheel", + "build": "tsc -p ./emitter/tsconfig.build.json && npm run venv:install-deps && npm run venv:install-dev-deps && npm run build-python-wheel", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", - "install_pygen_deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", - "install_pygen_dev_deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", + "venv:install-deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", + "venv:install-dev-deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", "ci:generator": "tsx ./eng/scripts/ci/run-ci.ts", From efdddcdcf428499d02073e74bf89b1c6e89d81e0 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 22:01:48 -0800 Subject: [PATCH 24/46] Create build_pygen_wheel.py --- .../eng/scripts/setup/build_pygen_wheel.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py new file mode 100644 index 0000000000..2011ca2026 --- /dev/null +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys + +if not sys.version_info >= (3, 8, 0): + raise Exception("Autorest for Python extension requires Python 3.8 at least") + +try: + import pip +except ImportError: + raise Exception("Your Python installation doesn't have pip available") + +try: + import venv +except ImportError: + raise Exception("Your Python installation doesn't have venv available") + + +# Now we have pip and Py >= 3.8, go to work + +from pathlib import Path + +from venvtools import build_wheel + +_ROOT_DIR = Path(__file__).parent.parent.parent.parent + + +def main(): + venv_path = _ROOT_DIR / "venv" + if venv_path.exists(): + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + build_wheel(venv_context, ["setup.py", "bdist_wheel"], additional_dir="generator") + else: + raise Exception("Please run 'npm install' first to create a Python virtual environment.") + +if __name__ == "__main__": + main() From 7abd521c8db2d04ea02a96c09b69cea1e6c22375 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 22:06:00 -0800 Subject: [PATCH 25/46] Use docutils 0.20.1 --- packages/http-client-python/generator/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/requirements.txt b/packages/http-client-python/generator/requirements.txt index 39e092bcb4..d112c8465c 100644 --- a/packages/http-client-python/generator/requirements.txt +++ b/packages/http-client-python/generator/requirements.txt @@ -1,6 +1,6 @@ black==24.8.0 click==8.1.7 -docutils==0.21.2 +docutils==0.20.1 Jinja2==3.1.4 m2r2==0.3.3.post2 MarkupSafe==2.1.5 From 27f520ca5119a8a8ef01f0b03ee5dadea52b2809 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 23:05:38 -0800 Subject: [PATCH 26/46] Update build python wheel cmd --- .../eng/scripts/setup/build_pygen_wheel.py | 4 ++-- .../eng/scripts/setup/venvtools.py | 13 ------------- .../generator/dev_requirements.txt | 1 + packages/http-client-python/package.json | 2 +- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index 2011ca2026..d4974a198e 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -25,7 +25,7 @@ from pathlib import Path -from venvtools import build_wheel +from venvtools import python_run _ROOT_DIR = Path(__file__).parent.parent.parent.parent @@ -35,7 +35,7 @@ def main(): if venv_path.exists(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) - build_wheel(venv_context, ["setup.py", "bdist_wheel"], additional_dir="generator") + python_run(venv_context, "build", ["--wheel"], additional_dir="generator") else: raise Exception("Please run 'npm install' first to create a Python virtual environment.") diff --git a/packages/http-client-python/eng/scripts/setup/venvtools.py b/packages/http-client-python/eng/scripts/setup/venvtools.py index 75ff23c6d7..c79ce4946b 100644 --- a/packages/http-client-python/eng/scripts/setup/venvtools.py +++ b/packages/http-client-python/eng/scripts/setup/venvtools.py @@ -85,16 +85,3 @@ def python_run(venv_context, module, command=None, *, additional_dir="."): except subprocess.CalledProcessError as err: print(err) sys.exit(1) - -def build_wheel(venv_context, command, *, additional_dir="."): - try: - cmd_line = [venv_context.env_exe] + command - print("Executing: {}".format(" ".join(cmd_line))) - subprocess.run( - cmd_line, - cwd=_ROOT_DIR / additional_dir, - check=True, - ) - except subprocess.CalledProcessError as err: - print(err) - sys.exit(1) diff --git a/packages/http-client-python/generator/dev_requirements.txt b/packages/http-client-python/generator/dev_requirements.txt index 3920cfa6b6..2cffff3f3b 100644 --- a/packages/http-client-python/generator/dev_requirements.txt +++ b/packages/http-client-python/generator/dev_requirements.txt @@ -12,3 +12,4 @@ black==24.8.0 ptvsd==4.3.2 types-PyYAML==6.0.12.8 wheel==0.44.0 +build==1.2.2 diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 50844f006c..b216a4b810 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -36,7 +36,7 @@ "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", - "venv:install-deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", + "venv:install-deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", "venv:install-dev-deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", From 90b037d9b41954848346614d94c8c6d5d44a44e3 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 8 Nov 2024 23:08:35 -0800 Subject: [PATCH 27/46] Update dev_requirements.txt --- packages/http-client-python/generator/dev_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/http-client-python/generator/dev_requirements.txt b/packages/http-client-python/generator/dev_requirements.txt index 2cffff3f3b..7daf480069 100644 --- a/packages/http-client-python/generator/dev_requirements.txt +++ b/packages/http-client-python/generator/dev_requirements.txt @@ -11,5 +11,4 @@ coverage==7.6.1 black==24.8.0 ptvsd==4.3.2 types-PyYAML==6.0.12.8 -wheel==0.44.0 build==1.2.2 From edd6db7503b4c132c6d17fcd4a448eb0b48c0694 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 12 Nov 2024 11:33:48 -0800 Subject: [PATCH 28/46] test --- packages/http-client-python/eng/scripts/setup/install.py | 6 ++++++ packages/http-client-python/package.json | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/install.py b/packages/http-client-python/eng/scripts/setup/install.py index 8b10f58bde..17bc6558b5 100644 --- a/packages/http-client-python/eng/scripts/setup/install.py +++ b/packages/http-client-python/eng/scripts/setup/install.py @@ -47,6 +47,12 @@ def main(): ["install", "-r", f"{_ROOT_DIR}/generator/requirements.txt"], ) python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/generator"]) + python_run( + venv_context, + "pip", + ["install", "-r", f"{_ROOT_DIR}/generator/dev_requirements.txt"], + ) + python_run(venv_context, "build", ["--wheel"], additional_dir="generator") if __name__ == "__main__": diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 30e32ae7ec..1680c00c7c 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -30,14 +30,11 @@ }, "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", - "build-python-wheel": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py", - "build": "tsc -p ./emitter/tsconfig.build.json && npm run venv:install-deps && npm run venv:install-dev-deps && npm run build-python-wheel", + "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", - "venv:install-deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", - "venv:install-dev-deps": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", "ci:generator": "tsx ./eng/scripts/ci/run-ci.ts", From 4bec00b6362a422278b809a3fe5eeb3c53d41b8e Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 12 Nov 2024 12:01:36 -0800 Subject: [PATCH 29/46] test --- .../eng/scripts/setup/build_pygen_wheel.py | 14 ++++++++------ .../eng/scripts/setup/install.py | 6 ------ packages/http-client-python/package-lock.json | 5 +++-- packages/http-client-python/package.json | 4 +++- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index d4974a198e..c43105d785 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -32,12 +32,14 @@ def main(): venv_path = _ROOT_DIR / "venv" - if venv_path.exists(): - env_builder = venv.EnvBuilder(with_pip=True) - venv_context = env_builder.ensure_directories(venv_path) - python_run(venv_context, "build", ["--wheel"], additional_dir="generator") - else: - raise Exception("Please run 'npm install' first to create a Python virtual environment.") + venv_preexists = venv_path.exists() + + assert venv_preexists # Otherwise install was not done + + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + python_run(venv_context, "build", ["--wheel"], additional_dir="generator") + if __name__ == "__main__": main() diff --git a/packages/http-client-python/eng/scripts/setup/install.py b/packages/http-client-python/eng/scripts/setup/install.py index 17bc6558b5..8b10f58bde 100644 --- a/packages/http-client-python/eng/scripts/setup/install.py +++ b/packages/http-client-python/eng/scripts/setup/install.py @@ -47,12 +47,6 @@ def main(): ["install", "-r", f"{_ROOT_DIR}/generator/requirements.txt"], ) python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/generator"]) - python_run( - venv_context, - "pip", - ["install", "-r", f"{_ROOT_DIR}/generator/dev_requirements.txt"], - ) - python_run(venv_context, "build", ["--wheel"], additional_dir="generator") if __name__ == "__main__": diff --git a/packages/http-client-python/package-lock.json b/packages/http-client-python/package-lock.json index 97be78b58e..07f2a49e17 100644 --- a/packages/http-client-python/package-lock.json +++ b/packages/http-client-python/package-lock.json @@ -1,12 +1,13 @@ { "name": "@typespec/http-client-python", - "version": "0.3.7", + "version": "0.3.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@typespec/http-client-python", - "version": "0.3.7", + "version": "0.3.8", + "hasInstallScript": true, "license": "MIT", "dependencies": { "js-yaml": "~4.1.0", diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 1680c00c7c..0a737f6438 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -30,7 +30,9 @@ }, "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", - "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", + "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py", + "install": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", + "prepare": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", From 0b56a1f8c433b4442e2d672a4eb4207709e001ca Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 12 Nov 2024 12:56:55 -0800 Subject: [PATCH 30/46] test --- .../eng/scripts/setup/build_pygen_wheel.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index c43105d785..e3432f5c22 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -6,6 +6,7 @@ # license information. # -------------------------------------------------------------------------- import sys +import os if not sys.version_info >= (3, 8, 0): raise Exception("Autorest for Python extension requires Python 3.8 at least") @@ -38,7 +39,15 @@ def main(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) - python_run(venv_context, "build", ["--wheel"], additional_dir="generator") + print(venv_context.env_exe) + print(os.path.exists(venv_context.env_exe)) + + for root, dirs, files in os.walk(venv_path): + for dir_name in dirs: + print(os.path.join(root, dir_name)) + for file in files: + print(file) + # python_run(venv_context, "build", ["--wheel"], additional_dir="generator") if __name__ == "__main__": From 5e601e07ec8ea181b7e7dce0592ada0615ed9a22 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 12 Nov 2024 15:12:11 -0800 Subject: [PATCH 31/46] test --- .../eng/scripts/setup/build_pygen_wheel.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index e3432f5c22..2a3bdebc58 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -42,12 +42,7 @@ def main(): print(venv_context.env_exe) print(os.path.exists(venv_context.env_exe)) - for root, dirs, files in os.walk(venv_path): - for dir_name in dirs: - print(os.path.join(root, dir_name)) - for file in files: - print(file) - # python_run(venv_context, "build", ["--wheel"], additional_dir="generator") + python_run(venv_context, "build", ["--wheel"], additional_dir="generator") if __name__ == "__main__": From cb25b96eb2f8baf7e843f619b750cfd21bbd324b Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 12 Nov 2024 15:55:11 -0800 Subject: [PATCH 32/46] Build wheel in a separate venv --- .../eng/scripts/setup/build_pygen_wheel.py | 18 +++++++----------- packages/http-client-python/package.json | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index 2a3bdebc58..8df9f32f55 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -6,7 +6,6 @@ # license information. # -------------------------------------------------------------------------- import sys -import os if not sys.version_info >= (3, 8, 0): raise Exception("Autorest for Python extension requires Python 3.8 at least") @@ -26,22 +25,19 @@ from pathlib import Path -from venvtools import python_run +from venvtools import ExtendedEnvBuilder, python_run _ROOT_DIR = Path(__file__).parent.parent.parent.parent def main(): - venv_path = _ROOT_DIR / "venv" - venv_preexists = venv_path.exists() - - assert venv_preexists # Otherwise install was not done - - env_builder = venv.EnvBuilder(with_pip=True) - venv_context = env_builder.ensure_directories(venv_path) - print(venv_context.env_exe) - print(os.path.exists(venv_context.env_exe)) + venv_path = _ROOT_DIR / "venv_build_wheel" + env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) + env_builder.create(venv_path) + venv_context = env_builder.context + python_run(venv_context, "pip", ["install", "-U", "pip"]) + python_run(venv_context, "pip", ["install", "build"]) python_run(venv_context, "build", ["--wheel"], additional_dir="generator") diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 0a737f6438..998f61ce3a 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -30,7 +30,7 @@ }, "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", - "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py", + "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py && rimraf ./venv_build_wheel", "install": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", "prepare": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", From 8c98d9bc4c93169f89679e81feff11d078b0aa49 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Wed, 13 Nov 2024 12:48:03 -0800 Subject: [PATCH 33/46] Install python deps for pyodide in "npm install" --- .../http-client-python/emitter/src/emitter.ts | 38 ++++++------------- .../eng/scripts/setup/build_pygen_wheel.py | 5 --- .../eng/scripts/setup/install-pyodide-deps.ts | 27 +++++++++++++ packages/http-client-python/package.json | 4 +- 4 files changed, 40 insertions(+), 34 deletions(-) create mode 100644 packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 064718f2a0..985807f4bb 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -7,7 +7,6 @@ import { import { EmitContext } from "@typespec/compiler"; import { exec } from "child_process"; import fs from "fs"; -import jsyaml from "js-yaml"; import path, { dirname } from "path"; import { loadPyodide } from "pyodide"; import { fileURLToPath } from "url"; @@ -80,6 +79,7 @@ export async function $onEmit(context: EmitContext) { const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const outputDir = context.emitterOutputDir; const yamlMap = emitCodeModel(sdkContext); + const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap); addDefaultOptions(sdkContext); const resolvedOptions = sdkContext.emitContext.options; if (resolvedOptions["use-pyodide"]) { @@ -97,7 +97,7 @@ export async function $onEmit(context: EmitContext) { resolvedOptions["package-pprint-name"] !== undefined && !resolvedOptions["package-pprint-name"].startsWith('"') ) { - resolvedOptions["package-pprint-name"] = `"${resolvedOptions["package-pprint-name"]}"`; + resolvedOptions["package-pprint-name"] = `${resolvedOptions["package-pprint-name"]}`; } for (const [key, value] of Object.entries(resolvedOptions)) { @@ -118,29 +118,14 @@ export async function $onEmit(context: EmitContext) { } const pyodide = await loadPyodide({ indexURL: path.join(root, "node_modules", "pyodide") }); pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, "."); - const yamlStr = jsyaml.dump(yamlMap); - if (!fs.existsSync("temp")) { - fs.mkdirSync("temp"); - } - const yamlPath = path.join("temp", "yamldata.yaml"); - pyodide.FS.writeFile(yamlPath, yamlStr); + await pyodide.loadPackage("setuptools"); + await pyodide.loadPackage("tomli"); + await pyodide.loadPackage("docutils"); await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); - await micropip.install([ - "black", - "click", - "docutils==0.21.2", - "Jinja2==3.1.4", - "m2r2==0.3.3.post2", - "MarkupSafe", - "pathspec", - "platformdirs", - "PyYAML", - "tomli", - "setuptools", - "emfs:generator/dist/pygen-0.1.0-py3-none-any.whl", - ]); - const globals = pyodide.toPy({ outputFolder, yamlPath, commandArgs }); + await micropip.install("emfs:generator/dist/pygen-0.1.0-py3-none-any.whl"); + const yamlRelativePath = path.relative(root, yamlPath); + const globals = pyodide.toPy({ outputFolder, yamlRelativePath, commandArgs }); const python = ` async def main(): import warnings @@ -148,9 +133,9 @@ export async function $onEmit(context: EmitContext) { warnings.simplefilter("ignore", SyntaxWarning) from pygen import m2r, preprocess, codegen, black - m2r.M2R(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() - preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() - codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlPath, **commandArgs).process() + m2r.M2R(output_folder=outputFolder, cadl_file=yamlRelativePath, **commandArgs).process() + preprocess.PreProcessPlugin(output_folder=outputFolder, cadl_file=yamlRelativePath, **commandArgs).process() + codegen.CodeGenerator(output_folder=outputFolder, cadl_file=yamlRelativePath, **commandArgs).process() black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process() await main() @@ -166,7 +151,6 @@ export async function $onEmit(context: EmitContext) { } else { throw new Error("Virtual environment doesn't exist."); } - const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap); const commandArgs = [ venvPath, `${root}/eng/scripts/setup/run_tsp.py`, diff --git a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py index 8df9f32f55..79f85e086a 100644 --- a/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py +++ b/packages/http-client-python/eng/scripts/setup/build_pygen_wheel.py @@ -15,11 +15,6 @@ except ImportError: raise Exception("Your Python installation doesn't have pip available") -try: - import venv -except ImportError: - raise Exception("Your Python installation doesn't have venv available") - # Now we have pip and Py >= 3.8, go to work diff --git a/packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts b/packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts new file mode 100644 index 0000000000..617b06d17d --- /dev/null +++ b/packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts @@ -0,0 +1,27 @@ +import { loadPyodide } from "pyodide"; +import path, { dirname } from "path"; +import { fileURLToPath } from "url"; + +async function installPyodideDeps() { + const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); + const pyodide = await loadPyodide({ indexURL: path.join(root, "node_modules", "pyodide") }); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + await micropip.install([ + "black", + "click", + "docutils==0.21.2", + "Jinja2==3.1.4", + "m2r2==0.3.3.post2", + "MarkupSafe", + "pathspec", + "platformdirs", + "pyyaml", + "tomli", + "setuptools", + ]); +} + +installPyodideDeps() + .then(() => console.log("Successfully installed all required Python packages in Pyodide!")) + .catch((error) => console.error(`Installation in Pyodide failed: ${error.message}`)); diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 998f61ce3a..fb5b755bc8 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -31,12 +31,12 @@ "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py && rimraf ./venv_build_wheel", - "install": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py", - "prepare": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", + "install": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py && tsx ./eng/scripts/setup/install-pyodide-deps.ts", + "prepare": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", "ci:generator": "tsx ./eng/scripts/ci/run-ci.ts", From 3c2c8f57fad3a5410895a8441a93a9e77f5369a9 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Wed, 13 Nov 2024 12:49:04 -0800 Subject: [PATCH 34/46] Change python check exceptions to warnings --- packages/http-client-python/eng/scripts/setup/install.py | 6 +++--- packages/http-client-python/eng/scripts/setup/prepare.py | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/install.py b/packages/http-client-python/eng/scripts/setup/install.py index 8b10f58bde..9ff745c9be 100644 --- a/packages/http-client-python/eng/scripts/setup/install.py +++ b/packages/http-client-python/eng/scripts/setup/install.py @@ -8,17 +8,17 @@ import sys if not sys.version_info >= (3, 8, 0): - raise Exception("Autorest for Python extension requires Python 3.8 at least") + raise Warning("Autorest for Python extension requires Python 3.8 at least") try: import pip except ImportError: - raise Exception("Your Python installation doesn't have pip available") + raise Warning("Your Python installation doesn't have pip available") try: import venv except ImportError: - raise Exception("Your Python installation doesn't have venv available") + raise Warning("Your Python installation doesn't have venv available") # Now we have pip and Py >= 3.8, go to work diff --git a/packages/http-client-python/eng/scripts/setup/prepare.py b/packages/http-client-python/eng/scripts/setup/prepare.py index 4c2c46bc03..433635dc6b 100644 --- a/packages/http-client-python/eng/scripts/setup/prepare.py +++ b/packages/http-client-python/eng/scripts/setup/prepare.py @@ -6,11 +6,9 @@ # license information. # -------------------------------------------------------------------------- import sys -import os -import argparse if not sys.version_info >= (3, 8, 0): - raise Exception("Autorest for Python extension requires Python 3.8 at least") + raise Warning("Autorest for Python extension requires Python 3.8 at least") from pathlib import Path import venv From 49f459b6d8b1153ee05464d9d15bc2c2830714fd Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Wed, 13 Nov 2024 12:50:13 -0800 Subject: [PATCH 35/46] Add pyodide tests --- .../eng/scripts/Generate-WithPyodide.ps1 | 11 +++++++++++ .../eng/scripts/Test-Packages.ps1 | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 packages/http-client-python/eng/scripts/Generate-WithPyodide.ps1 diff --git a/packages/http-client-python/eng/scripts/Generate-WithPyodide.ps1 b/packages/http-client-python/eng/scripts/Generate-WithPyodide.ps1 new file mode 100644 index 0000000000..c56126bd74 --- /dev/null +++ b/packages/http-client-python/eng/scripts/Generate-WithPyodide.ps1 @@ -0,0 +1,11 @@ +#Requires -Version 7.0 + +Import-Module "$PSScriptRoot\Generation.psm1" -DisableNameChecking -Force; + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..' '..') + +Write-Host "Building project ..." +& npm run build + +Write-Host "Regenerating project with Pyodide ..." +& npm run regenerate -- --pyodide diff --git a/packages/http-client-python/eng/scripts/Test-Packages.ps1 b/packages/http-client-python/eng/scripts/Test-Packages.ps1 index 0df7f33952..edb8f86200 100644 --- a/packages/http-client-python/eng/scripts/Test-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Test-Packages.ps1 @@ -34,7 +34,7 @@ try { Set-StrictMode -Version 1 # run E2E Test for TypeSpec emitter - Write-Host "Generating test projects ..." + Write-Host "Generating test projects with venv ..." & "$packageRoot/eng/scripts/Generate.ps1" Write-Host 'Code generation is completed.' @@ -44,7 +44,20 @@ try { Write-Host 'Done. No code generation differences detected.' } catch { - Write-Error 'Generated code is not up to date. Please run: eng/Generate.ps1' + Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate.ps1' + } + + Write-Host "Generating test projects with pyodide ..." + & "$packageRoot/eng/scripts/Generate-WithPyodide.ps1" + Write-Host 'Code generation is completed.' + + try { + Write-Host 'Checking for differences in generated code...' + & "$packageRoot/eng/scripts/Check-GitChanges.ps1" + Write-Host 'Done. No code generation differences detected.' + } + catch { + Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate-WithPyodide.ps1' } try { From 68e09413d1e3c04bbdba40a8b8ca5b360c6d28e2 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Wed, 13 Nov 2024 14:24:55 -0800 Subject: [PATCH 36/46] test --- .../eng/scripts/Test-Packages.ps1 | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/http-client-python/eng/scripts/Test-Packages.ps1 b/packages/http-client-python/eng/scripts/Test-Packages.ps1 index edb8f86200..7e2ec74dea 100644 --- a/packages/http-client-python/eng/scripts/Test-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Test-Packages.ps1 @@ -34,18 +34,18 @@ try { Set-StrictMode -Version 1 # run E2E Test for TypeSpec emitter - Write-Host "Generating test projects with venv ..." - & "$packageRoot/eng/scripts/Generate.ps1" - Write-Host 'Code generation is completed.' + # Write-Host "Generating test projects with venv ..." + # & "$packageRoot/eng/scripts/Generate.ps1" + # Write-Host 'Code generation is completed.' - try { - Write-Host 'Checking for differences in generated code...' - & "$packageRoot/eng/scripts/Check-GitChanges.ps1" - Write-Host 'Done. No code generation differences detected.' - } - catch { - Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate.ps1' - } + # try { + # Write-Host 'Checking for differences in generated code...' + # & "$packageRoot/eng/scripts/Check-GitChanges.ps1" + # Write-Host 'Done. No code generation differences detected.' + # } + # catch { + # Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate.ps1' + # } Write-Host "Generating test projects with pyodide ..." & "$packageRoot/eng/scripts/Generate-WithPyodide.ps1" From d5511542cbf9296023243f98bb0db6df66da5c8c Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 14 Nov 2024 10:01:16 -0800 Subject: [PATCH 37/46] Uncomment test on venv --- cspell.yaml | 1 - .../eng/scripts/Test-Packages.ps1 | 26 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cspell.yaml b/cspell.yaml index d9cc592b62..e53f658e04 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -250,7 +250,6 @@ words: - pyodide - pyimport - NODEFS - - yamldata - deps ignorePaths: - "**/node_modules/**" diff --git a/packages/http-client-python/eng/scripts/Test-Packages.ps1 b/packages/http-client-python/eng/scripts/Test-Packages.ps1 index 7e2ec74dea..2830de243a 100644 --- a/packages/http-client-python/eng/scripts/Test-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Test-Packages.ps1 @@ -34,19 +34,6 @@ try { Set-StrictMode -Version 1 # run E2E Test for TypeSpec emitter - # Write-Host "Generating test projects with venv ..." - # & "$packageRoot/eng/scripts/Generate.ps1" - # Write-Host 'Code generation is completed.' - - # try { - # Write-Host 'Checking for differences in generated code...' - # & "$packageRoot/eng/scripts/Check-GitChanges.ps1" - # Write-Host 'Done. No code generation differences detected.' - # } - # catch { - # Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate.ps1' - # } - Write-Host "Generating test projects with pyodide ..." & "$packageRoot/eng/scripts/Generate-WithPyodide.ps1" Write-Host 'Code generation is completed.' @@ -60,6 +47,19 @@ try { Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate-WithPyodide.ps1' } + Write-Host "Generating test projects with venv ..." + & "$packageRoot/eng/scripts/Generate.ps1" + Write-Host 'Code generation is completed.' + + try { + Write-Host 'Checking for differences in generated code...' + & "$packageRoot/eng/scripts/Check-GitChanges.ps1" + Write-Host 'Done. No code generation differences detected.' + } + catch { + Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate.ps1' + } + try { Write-Host "Pip List" & pip list From 790911a7941d6120af9ec82cb7373fe27db64cc4 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 14 Nov 2024 10:05:11 -0800 Subject: [PATCH 38/46] Update dev_requirements.txt --- packages/http-client-python/generator/dev_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/http-client-python/generator/dev_requirements.txt b/packages/http-client-python/generator/dev_requirements.txt index 7daf480069..94772f7a8d 100644 --- a/packages/http-client-python/generator/dev_requirements.txt +++ b/packages/http-client-python/generator/dev_requirements.txt @@ -11,4 +11,3 @@ coverage==7.6.1 black==24.8.0 ptvsd==4.3.2 types-PyYAML==6.0.12.8 -build==1.2.2 From 783e899f509b244fbc3885f26f09eae7d043f008 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Thu, 14 Nov 2024 10:24:45 -0800 Subject: [PATCH 39/46] Fix spell, format, lint --- cspell.yaml | 1 + .../eng/scripts/setup/install-pyodide-deps.ts | 40 +++++++++---------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/cspell.yaml b/cspell.yaml index e53f658e04..103fcbc4cf 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -251,6 +251,7 @@ words: - pyimport - NODEFS - deps + - pyyaml ignorePaths: - "**/node_modules/**" - "**/dist/**" diff --git a/packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts b/packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts index 617b06d17d..2662c38773 100644 --- a/packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts +++ b/packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts @@ -1,27 +1,27 @@ -import { loadPyodide } from "pyodide"; import path, { dirname } from "path"; +import { loadPyodide } from "pyodide"; import { fileURLToPath } from "url"; async function installPyodideDeps() { - const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); - const pyodide = await loadPyodide({ indexURL: path.join(root, "node_modules", "pyodide") }); - await pyodide.loadPackage("micropip"); - const micropip = pyodide.pyimport("micropip"); - await micropip.install([ - "black", - "click", - "docutils==0.21.2", - "Jinja2==3.1.4", - "m2r2==0.3.3.post2", - "MarkupSafe", - "pathspec", - "platformdirs", - "pyyaml", - "tomli", - "setuptools", - ]); + const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); + const pyodide = await loadPyodide({ indexURL: path.join(root, "node_modules", "pyodide") }); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + await micropip.install([ + "black", + "click", + "docutils==0.21.2", + "Jinja2==3.1.4", + "m2r2==0.3.3.post2", + "MarkupSafe", + "pathspec", + "platformdirs", + "pyyaml", + "tomli", + "setuptools", + ]); } installPyodideDeps() - .then(() => console.log("Successfully installed all required Python packages in Pyodide!")) - .catch((error) => console.error(`Installation in Pyodide failed: ${error.message}`)); + .then(() => console.log("Successfully installed all required Python packages in Pyodide!")) // eslint-disable-line no-console + .catch((error) => console.error(`Installation in Pyodide failed: ${error.message}`)); // eslint-disable-line no-console From d04db9c0ca6a4d857b365c961fff08cc33cc64c2 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 19 Nov 2024 11:29:20 -0800 Subject: [PATCH 40/46] Create venv when it's not found in emitter --- .../http-client-python/emitter/src/emitter.ts | 4 +- .../emitter/src/run-python3.ts | 20 ++ .../emitter/src/system-requirements.ts | 261 ++++++++++++++++++ 3 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 packages/http-client-python/emitter/src/run-python3.ts create mode 100644 packages/http-client-python/emitter/src/system-requirements.ts diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 985807f4bb..0bc43084a1 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -14,6 +14,7 @@ import { emitCodeModel } from "./code-model.js"; import { saveCodeModelAsYaml } from "./external-process.js"; import { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; import { removeUnderscoresFromNamespace } from "./utils.js"; +import { runPython3 } from "./run-python3.js"; export function getModelsMode(context: SdkContext): "dpg" | "none" { const specifiedModelsMode = context.emitContext.options["models-mode"]; @@ -149,7 +150,8 @@ export async function $onEmit(context: EmitContext) { } else if (fs.existsSync(path.join(venvPath, "Scripts"))) { venvPath = path.join(venvPath, "Scripts", "python.exe"); } else { - throw new Error("Virtual environment doesn't exist."); + await runPython3("./eng/scripts/setup/install.py"); + await runPython3("./eng/scripts/setup/prepare.py"); } const commandArgs = [ venvPath, diff --git a/packages/http-client-python/emitter/src/run-python3.ts b/packages/http-client-python/emitter/src/run-python3.ts new file mode 100644 index 0000000000..e4df6e8a71 --- /dev/null +++ b/packages/http-client-python/emitter/src/run-python3.ts @@ -0,0 +1,20 @@ +// This script wraps logic in @azure-tools/extension to resolve +// the path to Python 3 so that a Python script file can be run +// from an npm script in package.json. It uses the same Python 3 +// path resolution algorithm as AutoRest so that the behavior +// is fully consistent (and also supports AUTOREST_PYTHON_EXE). +// +// Invoke it like so: "tsx run-python3.ts script.py" + +import cp from "child_process"; +import { patchPythonPath } from "./system-requirements.js"; + +export async function runPython3(...args: string[]) { + const command = await patchPythonPath(["python", ...args], { + version: ">=3.8", + environmentVariable: "AUTOREST_PYTHON_EXE", + }); + cp.execSync(command.join(" "), { + stdio: [0, 1, 2], + }); +} diff --git a/packages/http-client-python/emitter/src/system-requirements.ts b/packages/http-client-python/emitter/src/system-requirements.ts new file mode 100644 index 0000000000..7f12ff5b5a --- /dev/null +++ b/packages/http-client-python/emitter/src/system-requirements.ts @@ -0,0 +1,261 @@ +import { ChildProcess, spawn, SpawnOptions } from "child_process"; +import { coerce, satisfies } from "semver"; + +/* + * Copied from @autorest/system-requirements + */ + +const execute = ( + command: string, + cmdlineargs: Array, + options: MoreOptions = {}, +): Promise => { + return new Promise((resolve, reject) => { + const cp = spawn(command, cmdlineargs, { ...options, stdio: "pipe", shell: true }); + if (options.onCreate) { + options.onCreate(cp); + } + + options.onStdOutData && cp.stdout.on("data", options.onStdOutData); + options.onStdErrData && cp.stderr.on("data", options.onStdErrData); + + let err = ""; + let out = ""; + let all = ""; + cp.stderr.on("data", (chunk) => { + err += chunk; + all += chunk; + }); + cp.stdout.on("data", (chunk) => { + out += chunk; + all += chunk; + }); + + cp.on("error", (err) => { + reject(err); + }); + cp.on("close", (code, signal) => + resolve({ + stdout: out, + stderr: err, + log: all, + error: code ? new Error("Process Failed.") : null, + code, + }), + ); + }); +}; + +const versionIsSatisfied = (version: string, requirement: string): boolean => { + const cleanedVersion = coerce(version); + if (!cleanedVersion) { + throw new Error(`Invalid version ${version}.`); + } + return satisfies(cleanedVersion, requirement, true); +}; + +/** + * Validate the provided system requirement resolution is satisfying the version requirement if applicable. + * @param resolution Command resolution. + * @param actualVersion Version for that resolution. + * @param requirement Requirement. + * @returns the resolution if it is valid or an @see SystemRequirementError if not. + */ +const validateVersionRequirement = ( + resolution: SystemRequirementResolution, + actualVersion: string, + requirement: SystemRequirement, +): SystemRequirementResolution | SystemRequirementError => { + if (!requirement.version) { + return resolution; // No version requirement. + } + + try { + if (versionIsSatisfied(actualVersion, requirement.version)) { + return resolution; + } + return { + ...resolution, + error: true, + message: `'${resolution.command}' version is '${actualVersion}' but doesn't satisfy requirement '${requirement.version}'. Please update.`, + actualVersion: actualVersion, + neededVersion: requirement.version, + }; + } catch { + return { + ...resolution, + error: true, + message: `Couldn't parse the version ${actualVersion}. This is not a valid semver version.`, + actualVersion: actualVersion, + neededVersion: requirement.version, + }; + } +}; + +const tryPython = async ( + requirement: SystemRequirement, + command: string, + additionalArgs: string[] = [], +): Promise => { + const resolution: SystemRequirementResolution = { + name: PythonRequirement, + command, + additionalArgs: additionalArgs.length > 0 ? additionalArgs : undefined, + }; + + try { + const result = await execute(command, [ + ...additionalArgs, + "-c", + `"${PRINT_PYTHON_VERSION_SCRIPT}"`, + ]); + return validateVersionRequirement(resolution, result.stdout.trim(), requirement); + } catch (e) { + return { + error: true, + ...resolution, + message: `'${command}' command line is not found in the path. Make sure to have it installed.`, + }; + } +}; + +/** + * Returns the path to the executable as asked in the requirement. + * @param requirement System requirement definition. + * @returns If the requirement provide an environment variable for the path returns the value of that environment variable. undefined otherwise. + */ +const getExecutablePath = (requirement: SystemRequirement): string | undefined => + requirement.environmentVariable && process.env[requirement.environmentVariable]; + +const createPythonErrorMessage = ( + requirement: SystemRequirement, + errors: SystemRequirementError[], +): SystemRequirementError => { + const versionReq = requirement.version ?? "*"; + const lines = [ + `Couldn't find a valid python interpreter satisfying the requirement (version: ${versionReq}). Tried:`, + ...errors.map((x) => ` - ${x.command} (${x.message})`), + ]; + + return { + error: true, + name: "python", + command: "python", + message: lines.join("\n"), + }; +}; + +const resolvePythonRequirement = async ( + requirement: SystemRequirement, +): Promise => { + // Hardcoding AUTOREST_PYTHON_EXE is for backward compatibility + const path = getExecutablePath(requirement) ?? process.env["AUTOREST_PYTHON_EXE"]; + if (path) { + return await tryPython(requirement, path); + } + + const errors: SystemRequirementError[] = []; + // On windows try `py` executable with `-3` flag. + if (process.platform === "win32") { + const pyResult = await tryPython(requirement, "py", ["-3"]); + if ("error" in pyResult) { + errors.push(pyResult); + } else { + return pyResult; + } + } + + const python3Result = await tryPython(requirement, "python3"); + if ("error" in python3Result) { + errors.push(python3Result); + } else { + return python3Result; + } + + const pythonResult = await tryPython(requirement, "python"); + if ("error" in pythonResult) { + errors.push(pythonResult); + } else { + return pythonResult; + } + + return createPythonErrorMessage(requirement, errors); +}; + +/** + * @param command list of the command and arguments. First item in array must be a python exe @see KnownPythonExe. (e.g. ["python", "my_python_file.py"] + * @param requirement + */ +export const patchPythonPath = async ( + command: PythonCommandLine, + requirement: SystemRequirement, +): Promise => { + const [_, ...args] = command; + const resolution = await resolvePythonRequirement(requirement); + if ("error" in resolution) { + throw new Error(`Failed to find compatible python version. ${resolution.message}`); + } + return [resolution.command, ...(resolution.additionalArgs ?? []), ...args]; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TYPES +const PythonRequirement = "python"; +const PRINT_PYTHON_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))"; + +type KnownPythonExe = "python.exe" | "python3.exe" | "python" | "python3"; +type PythonCommandLine = [KnownPythonExe, ...string[]]; + +interface MoreOptions extends SpawnOptions { + onCreate?(cp: ChildProcess): void; + onStdOutData?(chunk: any): void; + onStdErrData?(chunk: any): void; +} + +interface SystemRequirement { + version?: string; + /** + * Name of an environment variable where the user could provide the path to the exe. + * @example "AUTOREST_PYTHON_PATH" + */ + environmentVariable?: string; +} + +interface SystemRequirementResolution { + /** + * Name of the requirement. + * @example python, java, etc. + */ + name: string; + + /** + * Name of the command + * @example python3, /home/my_user/python39/python, java, etc. + */ + command: string; + + /** + * List of additional arguments to pass to this command. + * @example '-3' for 'py' to specify to use python 3 + */ + additionalArgs?: string[]; +} + +interface ExecResult { + stdout: string; + stderr: string; + + /** + * Union of stdout and stderr. + */ + log: string; + error: Error | null; + code: number | null; +} + +interface SystemRequirementError extends SystemRequirementResolution { + error: true; + message: string; + neededVersion?: string; + actualVersion?: string; +} From 6dd488c6743c2a7b91336015e80866a35e8ffee0 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 19 Nov 2024 11:35:22 -0800 Subject: [PATCH 41/46] Update warning info --- packages/http-client-python/eng/scripts/setup/install.py | 6 +++--- packages/http-client-python/eng/scripts/setup/prepare.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/install.py b/packages/http-client-python/eng/scripts/setup/install.py index 9ff745c9be..fac9ca17b8 100644 --- a/packages/http-client-python/eng/scripts/setup/install.py +++ b/packages/http-client-python/eng/scripts/setup/install.py @@ -8,17 +8,17 @@ import sys if not sys.version_info >= (3, 8, 0): - raise Warning("Autorest for Python extension requires Python 3.8 at least") + raise Warning("Autorest for Python extension requires Python 3.8 at least. If you don't have Python, please run with --pyodide.") try: import pip except ImportError: - raise Warning("Your Python installation doesn't have pip available") + raise Warning("Your Python installation doesn't have pip available. If you don't have Python, please run with --pyodide.") try: import venv except ImportError: - raise Warning("Your Python installation doesn't have venv available") + raise Warning("Your Python installation doesn't have venv available. If you don't have Python, please run with --pyodide.") # Now we have pip and Py >= 3.8, go to work diff --git a/packages/http-client-python/eng/scripts/setup/prepare.py b/packages/http-client-python/eng/scripts/setup/prepare.py index 433635dc6b..dbb92e1c2e 100644 --- a/packages/http-client-python/eng/scripts/setup/prepare.py +++ b/packages/http-client-python/eng/scripts/setup/prepare.py @@ -8,7 +8,7 @@ import sys if not sys.version_info >= (3, 8, 0): - raise Warning("Autorest for Python extension requires Python 3.8 at least") + raise Warning("Autorest for Python extension requires Python 3.8 at least. If you don't have Python, please run with --pyodide.") from pathlib import Path import venv From 5110c8d02ef63de4e6ea3fb7eb20a712bd4e4ca7 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Tue, 19 Nov 2024 16:17:52 -0800 Subject: [PATCH 42/46] Fix format --- packages/http-client-python/emitter/src/emitter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 0bc43084a1..5f13b46b52 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -13,8 +13,8 @@ import { fileURLToPath } from "url"; import { emitCodeModel } from "./code-model.js"; import { saveCodeModelAsYaml } from "./external-process.js"; import { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; -import { removeUnderscoresFromNamespace } from "./utils.js"; import { runPython3 } from "./run-python3.js"; +import { removeUnderscoresFromNamespace } from "./utils.js"; export function getModelsMode(context: SdkContext): "dpg" | "none" { const specifiedModelsMode = context.emitContext.options["models-mode"]; From 86bf69243c4b13c08e5e81920780aa4aa3464f59 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 22 Nov 2024 08:48:11 -0800 Subject: [PATCH 43/46] Update emitter.ts --- packages/http-client-python/emitter/src/emitter.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 5f13b46b52..7a4cc22bdc 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -145,13 +145,16 @@ export async function $onEmit(context: EmitContext) { } } else { let venvPath = path.join(root, "venv"); + if (!fs.existsSync(venvPath)) { + await runPython3("./eng/scripts/setup/install.py"); + await runPython3("./eng/scripts/setup/prepare.py"); + } if (fs.existsSync(path.join(venvPath, "bin"))) { venvPath = path.join(venvPath, "bin", "python"); } else if (fs.existsSync(path.join(venvPath, "Scripts"))) { venvPath = path.join(venvPath, "Scripts", "python.exe"); } else { - await runPython3("./eng/scripts/setup/install.py"); - await runPython3("./eng/scripts/setup/prepare.py"); + throw new Error("Virtual environment doesn't exist."); } const commandArgs = [ venvPath, From 9be5ab7470a0d032c7f1ab1b301500c0c6857a29 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 22 Nov 2024 13:05:42 -0800 Subject: [PATCH 44/46] Cleanup scripts --- .../http-client-python/eng/scripts/setup/build.ts | 13 +++++++++++++ .../setup/{install-pyodide-deps.ts => install.ts} | 0 packages/http-client-python/package.json | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 packages/http-client-python/eng/scripts/setup/build.ts rename packages/http-client-python/eng/scripts/setup/{install-pyodide-deps.ts => install.ts} (100%) diff --git a/packages/http-client-python/eng/scripts/setup/build.ts b/packages/http-client-python/eng/scripts/setup/build.ts new file mode 100644 index 0000000000..1ca2cd5a7c --- /dev/null +++ b/packages/http-client-python/eng/scripts/setup/build.ts @@ -0,0 +1,13 @@ +import { exec } from 'child_process'; + +// Define the command you want to run +const command = "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py && rimraf ./venv_build_wheel" + +// Execute the command +exec(command, (error, stdout, stderr) => { + if (error) { + console.error(`Error executing command: ${error.message}`); + return; + } + console.log(`Command output:\n${stdout}`); +}); diff --git a/packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts b/packages/http-client-python/eng/scripts/setup/install.ts similarity index 100% rename from packages/http-client-python/eng/scripts/setup/install-pyodide-deps.ts rename to packages/http-client-python/eng/scripts/setup/install.ts diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 07556aacf5..16403933d8 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -30,12 +30,12 @@ }, "scripts": { "clean": "rimraf ./dist ./temp ./emitter/temp ./generator/test/azure/generated ./generator/test/unbranded/generated ./venv", - "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py && rimraf ./venv_build_wheel", + "build": "tsc -p ./emitter/tsconfig.build.json && tsx ./eng/scripts/setup/build.ts", "watch": "tsc -p ./emitter/tsconfig.build.json --watch", "lint": "eslint . --max-warnings=0", "lint:py": "tsx ./eng/scripts/ci/lint.ts --folderName generator", "format": "pnpm -w format:dir packages/http-client-python && tsx ./eng/scripts/ci/format.ts", - "install": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py && tsx ./eng/scripts/setup/install-pyodide-deps.ts", + "install": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/install.py && tsx ./eng/scripts/setup/install.ts", "prepare": "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/prepare.py", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", "ci": "npm run test:emitter && npm run ci:generator --", From 0f53dfb61c94f9e72049e359e7912e09ba6aef09 Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 22 Nov 2024 13:16:50 -0800 Subject: [PATCH 45/46] Fix format and lint --- packages/http-client-python/eng/scripts/setup/build.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/build.ts b/packages/http-client-python/eng/scripts/setup/build.ts index 1ca2cd5a7c..2ed768f472 100644 --- a/packages/http-client-python/eng/scripts/setup/build.ts +++ b/packages/http-client-python/eng/scripts/setup/build.ts @@ -1,13 +1,14 @@ -import { exec } from 'child_process'; +import { exec } from "child_process"; // Define the command you want to run -const command = "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py && rimraf ./venv_build_wheel" +const command = + "tsx ./eng/scripts/setup/run-python3.ts ./eng/scripts/setup/build_pygen_wheel.py && rimraf ./venv_build_wheel"; // Execute the command exec(command, (error, stdout, stderr) => { if (error) { - console.error(`Error executing command: ${error.message}`); + console.error(`Error executing command: ${error.message}`); // eslint-disable-line no-console return; } - console.log(`Command output:\n${stdout}`); + console.log(`Command output:\n${stdout}`); // eslint-disable-line no-console }); From 22ab1a8341d644ec1dd021a97e82a45fe021e62c Mon Sep 17 00:00:00 2001 From: Yalin Li Date: Fri, 22 Nov 2024 14:13:12 -0800 Subject: [PATCH 46/46] Fix a bug in diff check in ci --- .../eng/scripts/Test-Packages.ps1 | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/http-client-python/eng/scripts/Test-Packages.ps1 b/packages/http-client-python/eng/scripts/Test-Packages.ps1 index 2830de243a..3b604f2792 100644 --- a/packages/http-client-python/eng/scripts/Test-Packages.ps1 +++ b/packages/http-client-python/eng/scripts/Test-Packages.ps1 @@ -38,14 +38,8 @@ try { & "$packageRoot/eng/scripts/Generate-WithPyodide.ps1" Write-Host 'Code generation is completed.' - try { - Write-Host 'Checking for differences in generated code...' - & "$packageRoot/eng/scripts/Check-GitChanges.ps1" - Write-Host 'Done. No code generation differences detected.' - } - catch { - Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate-WithPyodide.ps1' - } + # force add updates + Invoke-LoggedCommand "git add $packageRoot/generator/test -f" Write-Host "Generating test projects with venv ..." & "$packageRoot/eng/scripts/Generate.ps1" @@ -60,6 +54,9 @@ try { Write-Error 'Generated code is not up to date. Please run: eng/scripts/Generate.ps1' } + # reset force updates + Invoke-LoggedCommand "git reset ." + try { Write-Host "Pip List" & pip list