diff --git a/Cargo.lock b/Cargo.lock index 41c74cda26..17b152ce3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,7 +1700,6 @@ dependencies = [ "indoc", "itertools 0.11.0", "reqwest", - "schemars", "serde 1.0.202", "serde_json", "serde_with 3.8.1", @@ -4864,6 +4863,17 @@ dependencies = [ "regex-syntax 0.8.3", ] +[[package]] +name = "globwalk" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" +dependencies = [ + "bitflags 1.3.2", + "ignore", + "walkdir", +] + [[package]] name = "glow" version = "0.13.1" @@ -6608,6 +6618,7 @@ dependencies = [ "serde 1.0.202", "serde_json", "tempfile", + "tera", "tokio", ] @@ -9607,31 +9618,6 @@ dependencies = [ "user-facing-errors", ] -[[package]] -name = "schemars" -version = "0.8.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f55c82c700538496bdc329bb4918a81f87cc8888811bd123cf325a0f2f8d309" -dependencies = [ - "dyn-clone", - "indexmap 1.9.3", - "schemars_derive", - "serde 1.0.202", - "serde_json", -] - -[[package]] -name = "schemars_derive" -version = "0.8.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83263746fe5e32097f06356968a077f96089739c927a61450efa069905eec108" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 2.0.65", -] - [[package]] name = "scoped-tls" version = "1.0.1" @@ -9941,17 +9927,6 @@ dependencies = [ "syn 2.0.65", ] -[[package]] -name = "serde_derive_internals" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330f01ce65a3a5fe59a60c82f3c9a024b573b8a6e875bd233fe5f934e71d54e3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.65", -] - [[package]] name = "serde_json" version = "1.0.117" @@ -11409,6 +11384,22 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tera" +version = "1.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970dff17c11e884a4a09bc76e3a17ef71e01bb13447a11e85226e254fe6d10b8" +dependencies = [ + "globwalk", + "lazy_static 1.4.0", + "pest", + "pest_derive", + "regex", + "serde 1.0.202", + "serde_json", + "unic-segment", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -12240,12 +12231,14 @@ dependencies = [ name = "typegraph_core" version = "0.4.2" dependencies = [ + "anyhow", "common", "enum_dispatch", "graphql-parser 0.4.0", "indexmap 2.2.6", "indoc", "insta", + "metagen", "once_cell", "ordered-float 4.2.0", "paste", @@ -12387,6 +12380,15 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" +[[package]] +name = "unic-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" +dependencies = [ + "unic-ucd-segment", +] + [[package]] name = "unic-ucd-ident" version = "0.9.0" @@ -12398,6 +12400,17 @@ dependencies = [ "unic-ucd-version", ] +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + [[package]] name = "unic-ucd-version" version = "0.9.0" diff --git a/examples/deploy/deploy.mjs b/examples/deploy/deploy.mjs index c88b22aee7..00c5440ca5 100644 --- a/examples/deploy/deploy.mjs +++ b/examples/deploy/deploy.mjs @@ -62,7 +62,7 @@ const tg = await typegraph({ }, ), // Wasm - testWasmAdd: wasm.fromWasm( + testWasmAdd: wasm.fromExport( t.struct({ a: t.float(), b: t.float() }), t.integer(), { wasm: "wasm/rust.wasm", func: "add" }, diff --git a/examples/deploy/deploy.py b/examples/deploy/deploy.py index 2d69a2daf1..eadf1d43f9 100644 --- a/examples/deploy/deploy.py +++ b/examples/deploy/deploy.py @@ -59,7 +59,7 @@ def deploy_example_python(g: Graph): name="sayHello", ), # Wasm - testWasmAdd=wasm.from_wasm( + testWasmAdd=wasm.from_export( t.struct({"a": t.float(), "b": t.float()}), t.integer(), func="add", diff --git a/libs/common/Cargo.toml b/libs/common/Cargo.toml index cfd3520d6f..ac6555f126 100644 --- a/libs/common/Cargo.toml +++ b/libs/common/Cargo.toml @@ -8,7 +8,6 @@ anyhow.workspace = true base64 = "0.21.5" flate2 = "1.0.28" indexmap.workspace = true -schemars = { version = "0.8.16", features = ["derive", "preserve_order"], optional = true } serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } serde_with = "3.4.0" @@ -20,6 +19,3 @@ itertools = "0.11.0" colored = "2.0.4" indoc.workspace = true thiserror.workspace = true - -[features] -codegen = ["dep:schemars"] diff --git a/libs/metagen/Cargo.toml b/libs/metagen/Cargo.toml index 93a513633a..8105828c15 100644 --- a/libs/metagen/Cargo.toml +++ b/libs/metagen/Cargo.toml @@ -8,7 +8,7 @@ common.workspace = true log.workspace = true serde.workspace = true serde_json.workspace = true -tokio = { workspace = true, features =["rt-multi-thread"]} +tokio = { workspace = true, features =["rt-multi-thread"], optional = true } indexmap.workspace = true reqwest = { workspace = true, features = ["json"] } garde = { version = "0.18", features = ["derive"] } @@ -18,7 +18,11 @@ once_cell.workspace = true pretty_assertions = "1.4.0" color-eyre.workspace = true # indoc.workspace = true +tera = { version = "1", default-features = false } [dev-dependencies] -tokio = { workspace = true, features =["full"]} +tokio = { workspace = true, features =["full"] } tempfile.workspace = true + +[features] +multithreaded = ["dep:tokio"] diff --git a/libs/metagen/src/lib.rs b/libs/metagen/src/lib.rs index 075b926662..28c12dac3f 100644 --- a/libs/metagen/src/lib.rs +++ b/libs/metagen/src/lib.rs @@ -24,6 +24,7 @@ mod interlude { mod config; mod mdk; +mod mdk_python; mod mdk_rust; #[cfg(test)] mod tests; @@ -62,6 +63,11 @@ pub trait InputResolver { ) -> impl std::future::Future> + Send; } +/// This type plays the "dispatcher" role to the command object +pub trait InputResolverSync { + fn resolve(&self, order: GeneratorInputOrder) -> anyhow::Result; +} + #[derive(Debug)] pub struct GeneratedFile { // pub path: PathBuf, @@ -85,30 +91,61 @@ trait Plugin: Send + Sync { ) -> anyhow::Result; } +type PluginOutputResult = Result, anyhow::Error>; + +#[derive(Clone)] +struct GeneratorRunner { + pub op: fn(&Path, serde_json::Value) -> PluginOutputResult, +} + +impl GeneratorRunner { + pub fn exec(&self, workspace_path: &Path, value: serde_json::Value) -> PluginOutputResult { + (self.op)(workspace_path, value) + } +} + +thread_local! { + static GENERATORS: HashMap = HashMap::from([ + // builtin generators + ( + "mdk_rust".to_string(), + GeneratorRunner { + op: |workspace_path: &Path, val| { + let config = mdk_rust::MdkRustGenConfig::from_json(val, workspace_path)?; + let generator = mdk_rust::Generator::new(config)?; + Ok(Box::new(generator)) + }, + }, + ), + ( + "mdk_python".to_string(), + GeneratorRunner { + op: |workspace_path: &Path, val| { + let config = mdk_python::MdkPythonGenConfig::from_json(val, workspace_path)?; + let generator = mdk_python::Generator::new(config)?; + Ok(Box::new(generator)) + }, + }, + ), + ]); +} + +impl GeneratorRunner { + pub fn get(name: &str) -> Option { + GENERATORS.with(|m| m.get(name).cloned()) + } +} + /// This function makes use of a JoinSet to process /// items in parallel. This makes using actix workers in InputResolver /// is a no no. +#[cfg(feature = "multithreaded")] pub async fn generate_target( config: &config::Config, target_name: &str, workspace_path: PathBuf, resolver: impl InputResolver + Send + Sync + Clone + 'static, ) -> anyhow::Result { - let generators = [ - // builtin generators - ( - "mdk_rust".to_string(), - // initialize the impl - &|workspace_path: &Path, val| { - let config = mdk_rust::MdkRustGenConfig::from_json(val, workspace_path)?; - let generator = mdk_rust::Generator::new(config)?; - Ok::<_, anyhow::Error>(Box::new(generator) as Box) - }, - ), - ] - .into_iter() - .collect::>(); - let target_conf = config .targets .get(target_name) @@ -118,11 +155,10 @@ pub async fn generate_target( for (gen_name, config) in &target_conf.0 { let config = config.to_owned(); - let get_gen_fn = generators - .get(&gen_name[..]) - .with_context(|| format!("generator {gen_name:?} not found in config"))?; + let get_gen_op = GeneratorRunner::get(gen_name) + .with_context(|| format!("generator \"{gen_name}\" not found in config"))?; - let gen_impl = get_gen_fn(&workspace_path, config)?; + let gen_impl = get_gen_op.exec(&workspace_path, config)?; let bill = gen_impl.bill_of_inputs(); let mut resolve_set = tokio::task::JoinSet::new(); @@ -154,6 +190,61 @@ pub async fn generate_target( out.insert(path, (gen_name.clone(), buf)); } } + let out: HashMap = out + .into_iter() + .map(|(path, (_, buf))| (path, buf)) + .collect(); + Ok(GeneratorOutput(out)) +} + +pub fn generate_target_sync( + config: &config::Config, + target_name: &str, + workspace_path: PathBuf, + resolver: impl InputResolverSync + Send + Sync + Clone + 'static, +) -> anyhow::Result { + let target_conf = config + .targets + .get(target_name) + .with_context(|| format!("target \"{target_name}\" not found in config"))?; + + let mut generate_set = vec![]; + for (gen_name, config) in &target_conf.0 { + let config = config.to_owned(); + + let get_gen_op = GeneratorRunner::get(gen_name) + .with_context(|| format!("generator \"{gen_name}\" not found in config"))?; + + let gen_impl = get_gen_op.exec(&workspace_path, config)?; + let bill = gen_impl.bill_of_inputs(); + + let resolve_set = bill.into_iter().map(|(name, order)| { + let resolver = resolver.clone(); + Ok::<_, anyhow::Error>((name, resolver.resolve(order))) + }); + + let gen_name: Arc = gen_name[..].into(); + generate_set.push(move || { + let mut inputs = HashMap::new(); + for res in resolve_set { + let (name, input) = res?; + inputs.insert(name, input?); + } + let out = gen_impl.generate(inputs)?; + Ok::<_, anyhow::Error>((gen_name, out)) + }); + } + + let mut out = HashMap::new(); + for res in generate_set { + let (gen_name, files) = res()?; + for (path, buf) in files.0 { + if let Some((src, _)) = out.get(&path) { + anyhow::bail!("generators \"{src}\" and \"{gen_name}\" clashed at \"{path:?}\""); + } + out.insert(path, (gen_name.clone(), buf)); + } + } let out = out .into_iter() .map(|(path, (_, buf))| (path, buf)) diff --git a/libs/metagen/src/mdk_python/mod.rs b/libs/metagen/src/mdk_python/mod.rs new file mode 100644 index 0000000000..5eb505785f --- /dev/null +++ b/libs/metagen/src/mdk_python/mod.rs @@ -0,0 +1,338 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + +use garde::external::compact_str::CompactStringExt; +use heck::ToPascalCase; + +use crate::interlude::*; +use crate::mdk::*; +use crate::*; + +use self::utils::Memo; +use self::utils::TypeGenerated; + +mod types; +mod utils; + +#[derive(Serialize, Deserialize, Debug, garde::Validate)] +pub struct MdkPythonGenConfig { + #[serde(flatten)] + #[garde(dive)] + pub base: crate::config::MdkGeneratorConfigBase, +} + +impl MdkPythonGenConfig { + pub fn from_json(json: serde_json::Value, workspace_path: &Path) -> anyhow::Result { + let mut config: mdk_python::MdkPythonGenConfig = serde_json::from_value(json)?; + config.base.path = workspace_path.join(config.base.path); + config.base.typegraph_path = config + .base + .typegraph_path + .as_ref() + .map(|path| workspace_path.join(path)); + Ok(config) + } +} + +pub struct Generator { + config: MdkPythonGenConfig, +} + +impl Generator { + pub const INPUT_TG: &'static str = "tg_name"; + pub fn new(config: MdkPythonGenConfig) -> Result { + use garde::Validate; + config.validate(&())?; + Ok(Self { config }) + } +} + +impl crate::Plugin for Generator { + fn bill_of_inputs(&self) -> HashMap { + [( + Self::INPUT_TG.to_string(), + if let Some(tg_name) = &self.config.base.typegraph_name { + GeneratorInputOrder::TypegraphFromTypegate { + name: tg_name.clone(), + } + } else if let Some(tg_path) = &self.config.base.typegraph_path { + GeneratorInputOrder::TypegraphFromPath { + path: tg_path.clone(), + name: self.config.base.typegraph_name.clone(), + } + } else { + unreachable!() + }, + )] + .into_iter() + .collect() + } + + fn generate( + &self, + inputs: HashMap, + ) -> anyhow::Result { + // return Ok(GeneratorOutput(Default::default())) + let tg = match inputs + .get(Self::INPUT_TG) + .context("missing generator input")? + { + GeneratorInputResolved::TypegraphFromTypegate { raw } => raw, + GeneratorInputResolved::TypegraphFromPath { raw } => raw, + }; + let mut mergeable_output: IndexMap> = IndexMap::new(); + + let mut tera = tera::Tera::default(); + tera.add_raw_template("main_template", include_str!("static/main.py.jinja"))?; + tera.add_raw_template("types_template", include_str!("static/types.py.jinja"))?; + tera.add_raw_template("struct_template", include_str!("static/struct.py.jinja"))?; + + let stubbed_funs = filter_stubbed_funcs(tg, &["python".to_string()])?; + for fun in &stubbed_funs { + if fun.mat.data.get("mod").is_none() { + continue; + } + let (_, script_path) = get_module_infos(fun, tg)?; + let base_path = self.config.base.path.clone(); + let entry_point_path = if base_path.as_os_str().to_string_lossy().trim() == "" { + self.config + .base + .typegraph_path + .clone() + .map(|p| p.parent().unwrap().to_owned()) // try relto typegraph path first + .unwrap() + .join(script_path) + } else if script_path.is_absolute() { + script_path + } else { + base_path.join(&script_path) + }; + + let required = gen_required_objects(&tera, fun, tg)?; + mergeable_output + .entry(entry_point_path.clone()) + .or_default() + .push(required); + } + + let merged_list = merge_requirements(mergeable_output); + let mut out = HashMap::new(); + for merged_req in merged_list { + let entry_point_path = merged_req.entry_point_path.clone(); + let file_stem = merged_req + .entry_point_path + .file_stem() + .map(|v| v.to_str().to_owned()) + .unwrap() + .with_context(|| "Get file stem") + .unwrap(); + let types_path = merged_req + .entry_point_path + .parent() + .with_context(|| "Get parent path") + .unwrap() + .join(PathBuf::from(format!("{file_stem}_types.py"))); + + out.insert( + entry_point_path, + GeneratedFile { + contents: render_main(&tera, &merged_req, file_stem)?, + overwrite: false, + }, + ); + out.insert( + types_path, + GeneratedFile { + contents: render_types(&tera, &merged_req)?, + overwrite: true, + }, + ); + } + + Ok(GeneratorOutput(out)) + } +} + +fn get_module_infos(fun: &StubbedFunction, tg: &Typegraph) -> anyhow::Result<(String, PathBuf)> { + let idx = serde_json::from_value::( + fun.mat + .data + .get("mod") + .with_context(|| "python mod index") + .unwrap() + .clone(), + )?; + let mod_name = serde_json::from_value::( + fun.mat + .data + .get("name") + .with_context(|| "python mod name") + .unwrap() + .clone(), + )?; + let script_path = serde_json::from_value::( + tg.materializers[idx].data["pythonArtifact"]["path"].clone(), + )?; + let script_path = PathBuf::from(script_path.clone()); + + Ok((mod_name, script_path)) +} + +#[derive(Serialize, Eq, PartialEq, Hash)] +struct FuncDetails { + pub input_name: String, + pub output_name: String, + pub name: String, +} + +/// Objects required per function +struct RequiredObjects { + pub func_details: FuncDetails, + pub top_level_types: Vec, + pub memo: Memo, +} + +/// Objects required per function that refers to the same python module +struct MergedRequiredObjects { + pub funcs: Vec, + pub top_level_types: Vec, + pub memo: Memo, + pub entry_point_path: PathBuf, +} + +fn render_main( + tera: &tera::Tera, + required: &MergedRequiredObjects, + file_stem: &str, +) -> anyhow::Result { + let mut exports = HashSet::new(); + for func in required.funcs.iter() { + exports.insert(format!("typed_{}", func.name)); + exports.insert(func.input_name.clone()); + exports.insert(func.output_name.clone()); + } + + let mut context = tera::Context::new(); + context.insert("funcs", &required.funcs); + context.insert("mod_name", file_stem); + context.insert("imports", &exports.join_compact(", ").to_string()); + + tera.render("main_template", &context) + .map_err(|e| anyhow::Error::new(e).context("Failed to render main template")) +} + +fn render_types(tera: &tera::Tera, required: &MergedRequiredObjects) -> anyhow::Result { + let mut context = tera::Context::new(); + let classes = required + .memo + .types_in_order() + .iter() + .filter_map(|gen| { + if gen.def.is_some() { + Some(serde_json::to_value(gen.to_owned()).unwrap()) + } else { + None + } + }) + .collect::>(); + let types = required + .top_level_types + .iter() + .filter_map(|gen| gen.def.clone()) + .collect::>(); + + context.insert("classes", &classes); + context.insert("types", &types); + context.insert("funcs", &required.funcs); + + tera.render("types_template", &context) + .map_err(|e| anyhow::Error::new(e).context("Failed to render types template")) +} + +fn gen_required_objects( + tera: &tera::Tera, + fun: &StubbedFunction, + tg: &Typegraph, +) -> anyhow::Result { + if let TypeNode::Function { data, .. } = fun.node.clone() { + let mut memo = Memo::new(); + let input = tg.types[data.input as usize].clone(); + let output = tg.types[data.output as usize].clone(); + + let input_name = input.base().title.to_pascal_case(); + let output_name = output.base().title.to_pascal_case(); + + let _input_hint = types::visit_type(tera, &mut memo, &input, tg)?.hint; + let output_hint = types::visit_type(tera, &mut memo, &output, tg)?.hint; + + let (fn_name, _) = get_module_infos(fun, tg)?; + match output { + TypeNode::Object { .. } => { + // output is a top level dataclass + Ok(RequiredObjects { + func_details: FuncDetails { + input_name, + output_name, + name: fn_name, + }, + top_level_types: vec![], + memo, + }) + } + _ => { + // output is a top level inline def + let output_name = format!("Type{output_name}"); + let def = format!("{} = {}", output_name.clone(), output_hint); + let top_level_types = vec![TypeGenerated { + hint: output_hint, + def: Some(def), + }]; + Ok(RequiredObjects { + func_details: FuncDetails { + input_name, + output_name, + name: fn_name, + }, + top_level_types, + memo, + }) + } + } + } else { + panic!("function node was expected") + } +} + +fn merge_requirements( + mergeable: IndexMap>, +) -> Vec { + // merge types and defs that refers the same file + let mut gen_inputs = vec![]; + for (entry_point_path, requirements) in mergeable { + let mut ongoing_merge = MergedRequiredObjects { + entry_point_path, + funcs: vec![], + memo: Memo::new(), + top_level_types: vec![], + }; + let mut types = HashSet::new(); + let mut funcs = HashSet::new(); + + for req in requirements { + // merge classes + ongoing_merge.memo.merge_with(req.memo.clone()); + // merge types + for tpe in req.top_level_types { + types.insert(tpe); + } + // merge funcs + funcs.insert(req.func_details); + } + + ongoing_merge.top_level_types = Vec::from_iter(types.into_iter()); + ongoing_merge.funcs = Vec::from_iter(funcs.into_iter()); + gen_inputs.push(ongoing_merge); + } + + gen_inputs +} diff --git a/libs/metagen/src/mdk_python/static/main.py.jinja b/libs/metagen/src/mdk_python/static/main.py.jinja new file mode 100644 index 0000000000..c2380a2e04 --- /dev/null +++ b/libs/metagen/src/mdk_python/static/main.py.jinja @@ -0,0 +1,8 @@ +from .{{ mod_name }}_types import {{ imports }} + +{% for func in funcs %} +@typed_{{ func.name }} +def {{ func.name }}(inp: {{ func.input_name }}) -> {{ func.output_name }}: + # TODO: write your logic here + raise Exception("{{ func.name }} not implemented") +{% endfor %} diff --git a/libs/metagen/src/mdk_python/static/struct.py.jinja b/libs/metagen/src/mdk_python/static/struct.py.jinja new file mode 100644 index 0000000000..106d9f6668 --- /dev/null +++ b/libs/metagen/src/mdk_python/static/struct.py.jinja @@ -0,0 +1,5 @@ +@dataclass +class {{ class_name }}(Struct): + {% for field in fields -%} + {{ field }} + {% endfor %} diff --git a/libs/metagen/src/mdk_python/static/types.py.jinja b/libs/metagen/src/mdk_python/static/types.py.jinja new file mode 100644 index 0000000000..38126f4769 --- /dev/null +++ b/libs/metagen/src/mdk_python/static/types.py.jinja @@ -0,0 +1,101 @@ +from types import NoneType +from typing import Callable, List, Union, get_origin, ForwardRef +from dataclasses import dataclass, asdict, fields + +FORWARD_REFS = {} + +class Struct: + def try_new(dt_class, val: any): + # Object + ftypes = {f.name: f.type for f in fields(dt_class)} + attrs = {} + for f in val: + fval = val[f] + ftype = ftypes[f] + serialized = False + # Union + if get_origin(ftype) is Union: + try: + attrs[f] = Struct.try_union(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # List + elif get_origin(ftype) is list: + try: + attrs[f] = Struct.try_typed_list(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # Any + if not serialized: + if type(ftype) is str and ftype in FORWARD_REFS: + klass = FORWARD_REFS[ftype] + attrs[f] = Struct.new(klass, fval) + else: + attrs[f] = Struct.new(ftype, fval) + return dt_class(**attrs) + + def try_typed_list(tpe: any, items: any): + hint = tpe.__args__[0] + klass = FORWARD_REFS[hint.__forward_arg__] if type(hint) is ForwardRef else hint + return [Struct.new(klass, v) for v in items] + + def try_union(variants: List[any], val: any): + errors = [] + for variant in variants: + try: + if variant is NoneType: + if val is None: + return None + else: + continue + if get_origin(variant) is list: + if type(val) is list: + return Struct.try_typed_list(variant, val) + else: + continue + klass = FORWARD_REFS[variant.__forward_arg__] + return Struct.try_new(klass, val) + except Exception as e: + errors.append(str(e)) + raise Exception("\n".join(errors)) + + + def new(dt_class: any, val: any): + try: + return Struct.try_new(dt_class, val) + except: + return val + + def repr(self): + return asdict(self) + + +{% for class in classes -%} + +{{ class.def }} +FORWARD_REFS['{{ class.hint }}'] = {{ class.hint }} + +{% endfor -%} + + +{% for def in types -%} +{{ def }} +{% endfor %} + +def __repr(value: any): + if isinstance(value, Struct): + return value.repr() + return value + +{%for func in funcs %} +def typed_{{ func.name }}(user_fn: Callable[[{{ func.input_name }}], {{ func.output_name }}]): + def exported_wrapper(raw_inp): + inp: {{ func.input_name }} = Struct.new({{ func.input_name }}, raw_inp) + out: {{ func.output_name }} = user_fn(inp) + if type(out) is list: + return [__repr(v) for v in out] + return __repr(out) + return exported_wrapper +{% endfor %} diff --git a/libs/metagen/src/mdk_python/types.rs b/libs/metagen/src/mdk_python/types.rs new file mode 100644 index 0000000000..28c1f940a3 --- /dev/null +++ b/libs/metagen/src/mdk_python/types.rs @@ -0,0 +1,121 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + +use anyhow::bail; +use garde::external::compact_str::CompactStringExt; +use heck::ToPascalCase; + +use crate::interlude::*; + +use super::utils::{Memo, TypeGenerated}; + +/// Collect relevant definitions in `memo` if object, return the type in Python +pub fn visit_type( + tera: &tera::Tera, + memo: &mut Memo, + tpe: &TypeNode, + tg: &Typegraph, +) -> anyhow::Result { + memo.incr_weight(); + + let hint = match tpe { + TypeNode::Boolean { .. } => "bool".to_string(), + TypeNode::Float { .. } => "float".to_string(), + TypeNode::Integer { .. } => "int".to_string(), + TypeNode::String { .. } => "str".to_string(), + TypeNode::Object { .. } => { + let class_hint = tpe.base().title.to_pascal_case(); + let hint = match memo.is_allocated(&class_hint) { + true => class_hint, + false => visit_object(tera, memo, tpe, tg)?.hint, + }; + format!("'{hint}'") + } + TypeNode::Optional { data, .. } => { + let item = &tg.types[data.item as usize]; + let item_hint = visit_type(tera, memo, item, tg)?.hint; + format!("Union[{item_hint}, None]") + } + TypeNode::List { data, .. } => { + let item = &tg.types[data.items as usize]; + let item_hint = visit_type(tera, memo, item, tg)?.hint; + format!("List[{item_hint}]") + } + TypeNode::Function { .. } => "".to_string(), + TypeNode::Union { .. } | TypeNode::Either { .. } => { + visit_union_or_either(tera, memo, tpe, tg)?.hint + } + _ => bail!("Unsupported type {:?}", tpe.type_name()), + }; + + memo.decr_weight(); + Ok(hint.into()) +} + +/// Collect relevant definitions in `memo`, return the type in Python +fn visit_object( + tera: &tera::Tera, + memo: &mut Memo, + tpe: &TypeNode, + tg: &Typegraph, +) -> anyhow::Result { + if let TypeNode::Object { base, data } = tpe { + let mut fields_repr = vec![]; + let hint = base.title.clone().to_pascal_case(); + + memo.allocate(hint.clone()); + + for (field, idx) in data.properties.iter() { + let field_tpe = &tg.types[*idx as usize]; + let type_repr = visit_type(tera, memo, field_tpe, tg)?.hint; + fields_repr.push(format!("{field}: {type_repr}")); + } + + let mut context = tera::Context::new(); + context.insert("class_name", &base.title.to_pascal_case()); + context.insert("fields", &fields_repr); + + let code = tera.render("struct_template", &context)?; + let generated = TypeGenerated { + hint: hint.clone(), + def: Some(code), + }; + + memo.insert(hint, generated.clone()); + + Ok(generated) + } else { + panic!("object node was expected, got {:?}", tpe.type_name()) + } +} + +/// Collect relevant definitions in `memo`, return the type in Python +fn visit_union_or_either( + tera: &tera::Tera, + memo: &mut Memo, + tpe: &TypeNode, + tg: &Typegraph, +) -> anyhow::Result { + let mut visit_variants = |variants: &[u32]| -> anyhow::Result { + let mut variants_repr = HashSet::new(); + for idx in variants.iter() { + let field_tpe = &tg.types[*idx as usize]; + let type_repr = visit_type(tera, memo, field_tpe, tg)?.hint; + variants_repr.insert(type_repr); + } + let variant_hints = variants_repr.join_compact(", ").to_string(); + let hint = match variants_repr.len() == 1 { + true => variant_hints, + false => format!("Union[{variant_hints}]"), + }; + Ok(hint.into()) + }; + + if let TypeNode::Union { data, .. } = tpe { + visit_variants(&data.any_of) + } else if let TypeNode::Either { data, .. } = tpe { + visit_variants(&data.one_of) + } else { + panic!("union/either node was expected, got {:?}", tpe.type_name()) + } +} diff --git a/libs/metagen/src/mdk_python/utils.rs b/libs/metagen/src/mdk_python/utils.rs new file mode 100644 index 0000000000..a89e74d306 --- /dev/null +++ b/libs/metagen/src/mdk_python/utils.rs @@ -0,0 +1,111 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + +use indexmap::IndexMap; +use serde::Serialize; + +#[derive(Debug, Eq, Hash, PartialEq, Clone, Serialize)] +pub struct TypeGenerated { + /// Type representation: int, str, class name, .. + pub hint: String, + /// Source-code (for structs only) + pub def: Option, +} + +impl From for TypeGenerated { + fn from(value: String) -> Self { + Self { + hint: value.to_string(), + def: None, + } + } +} + +#[derive(Debug, Clone)] +struct Class { + priority: u32, + type_generated: Option, +} + +#[derive(Clone)] +pub struct Memo { + map: IndexMap, + priority_weight: u32, +} + +impl Memo { + pub fn new() -> Self { + Self { + map: IndexMap::new(), + priority_weight: 1, + } + } + + /// Insert `v` at `k`, if already present, this will also increase the priority by the current weight + pub fn insert(&mut self, k: String, v: TypeGenerated) { + if !self.is_allocated(&k) { + self.allocate(k.clone()); + } + let old = self.map.get(&k).unwrap(); + self.map.insert( + k, + Class { + priority: old.priority + self.priority_weight, + type_generated: Some(v), + }, + ); + } + + /// Allocate for `k` and set priority to 0, if a place is already allocated this will do nothing + pub fn allocate(&mut self, k: String) { + if !self.is_allocated(&k) { + self.map.insert( + k, + Class { + priority: 0, + type_generated: None, + }, + ); + } + } + + pub fn is_allocated(&self, k: &str) -> bool { + self.map.contains_key(k) + } + + pub fn types_in_order(&self) -> Vec { + let mut values = self + .map + .values() + .filter(|c| c.type_generated.is_some()) + .collect::>(); + + // if typing dataclass A depends on dataclass B, then B must be generated first + values.sort_by(|a, b| b.priority.cmp(&a.priority)); + + values + .iter() + .filter_map(|c| c.type_generated.as_ref().cloned()) + .collect() + } + + pub fn incr_weight(&mut self) { + self.priority_weight += 1; + } + + pub fn decr_weight(&mut self) { + if let Some(value) = self.priority_weight.checked_sub(1) { + self.priority_weight = value; + } else { + panic!("invalid state: priority weight overflowed") + } + } + + pub fn merge_with(&mut self, other: Memo) { + for (k, v) in other.map.into_iter() { + if let Some(gen) = v.type_generated { + self.insert(k, gen); + } + } + } +} diff --git a/libs/metagen/src/mdk_rust/mod.rs b/libs/metagen/src/mdk_rust/mod.rs index 51eb6ff12e..041e3553bb 100644 --- a/libs/metagen/src/mdk_rust/mod.rs +++ b/libs/metagen/src/mdk_rust/mod.rs @@ -284,6 +284,7 @@ impl stubs::MyFunc for MyMat { .into() } +#[cfg(feature = "multithreaded")] #[test] fn mdk_rs_e2e() -> anyhow::Result<()> { use crate::tests::*; diff --git a/libs/metagen/src/tests/mod.rs b/libs/metagen/src/tests/mod.rs index 5539f86e4c..7e20f0a291 100644 --- a/libs/metagen/src/tests/mod.rs +++ b/libs/metagen/src/tests/mod.rs @@ -38,6 +38,7 @@ pub struct E2eTestCase { pub build_fn: fn(BuildArgs) -> BoxFuture>, } +#[cfg(feature = "multithreaded")] pub async fn e2e_test(cases: Vec) -> anyhow::Result<()> { // spin_up_typegate for case in cases { diff --git a/meta-cli/Cargo.toml b/meta-cli/Cargo.toml index c15e4513bf..146fcc5f7c 100644 --- a/meta-cli/Cargo.toml +++ b/meta-cli/Cargo.toml @@ -33,7 +33,7 @@ typegate = ["dep:typegate_engine"] typegate_engine = { workspace = true, optional = true } common.workspace = true typescript.workspace = true -metagen.workspace = true +metagen = { workspace = true, features = ["multithreaded"] } # data structures chrono = { version = "0.4.31", features = ["serde"] } diff --git a/meta-cli/src/com/responses.rs b/meta-cli/src/com/responses.rs index f7e4c79854..1ae9d50f1e 100644 --- a/meta-cli/src/com/responses.rs +++ b/meta-cli/src/com/responses.rs @@ -105,6 +105,7 @@ impl SDKResponse { ret } + // TODO: rm once MET-492 lands pub fn codegen(&self) -> Result<()> { let tg = self.as_typegraph()?; let path = self.typegraph_path.clone(); diff --git a/meta-lsp/package.json b/meta-lsp/package.json index 3a62513281..e24ac2e45c 100644 --- a/meta-lsp/package.json +++ b/meta-lsp/package.json @@ -1,48 +1,48 @@ { - "name": "vscode-metatype", - "displayName": "Metatype", - "description": "VSCode extension for Metatype support", - "icon": "logo.png", - "author": "Metatype Team", - "version": "0.4.2", - "repository": { - "type": "git", - "url": "https://github.com/metatypedev/metatype" - }, - "publisher": "metatypedev", - "engines": { - "vscode": "^1.75.0" - }, - "activationEvents": [ - "onLanguage:typescript", - "onLanguage:javascript" - ], - "main": "./vscode-metatype-support/out/extension", - "scripts": { - "compile:ts-server": "pnpm esbuild ./ts-language-server/src/server.ts --bundle --outfile=ts-language-server/out/server.js --packages=external --format=cjs --platform=node", - "dev:ts-server": "cd ts-language-server && pnpm dev", - "compile:vscode": "pnpm esbuild ./vscode-metatype-support/src/extension.ts --bundle --outfile=vscode-metatype-support/out/extension.js --external:vscode --format=cjs --platform=node", - "vscode:prepublish": "pnpm run compile:ts-server && pnpm run compile:vscode", - "lint": "pnpm eslint ./ts-language-server/src ./vscode-metatype-support/src --ext .ts", - "vscode:package": "pnpm vsce package --no-dependencies", - "vscode:publish": "pnpm vsce publish --no-dependencies", - "test:ts-server": "cd ts-language-server && pnpm test", - "clean": "rm -rf ./ts-language-server/out && rm -rf ./vscode-metatype-support/out" - }, - "devDependencies": { - "@types/node": "^16.18.65", - "@types/vscode": "^1.75.0", - "@typescript-eslint/eslint-plugin": "^6.14.0", - "@typescript-eslint/parser": "^6.14.0", - "@vscode/vsce": "^2.22.0", - "esbuild": "^0.19.10", - "eslint": "^8.54.0", - "eslint-config-standard-with-typescript": "^42.0.0", - "eslint-plugin-import": "^2.25.2", - "eslint-plugin-n": "^15.0.0 || ^16.0.0 ", - "eslint-plugin-promise": "^6.0.0", - "typescript": "^5.3.2", - "tsx": "4.7", - "lcov": "1.16" - } + "name": "vscode-metatype", + "displayName": "Metatype", + "description": "VSCode extension for Metatype support", + "icon": "logo.png", + "author": "Metatype Team", + "version": "0.4.2", + "repository": { + "type": "git", + "url": "https://github.com/metatypedev/metatype" + }, + "publisher": "metatypedev", + "engines": { + "vscode": "^1.75.0" + }, + "activationEvents": [ + "onLanguage:typescript", + "onLanguage:javascript" + ], + "main": "./vscode-metatype-support/out/extension", + "scripts": { + "compile:ts-server": "pnpm esbuild ./ts-language-server/src/server.ts --bundle --outfile=ts-language-server/out/server.js --packages=external --format=cjs --platform=node", + "dev:ts-server": "cd ts-language-server && pnpm dev", + "compile:vscode": "pnpm esbuild ./vscode-metatype-support/src/extension.ts --bundle --outfile=vscode-metatype-support/out/extension.js --external:vscode --format=cjs --platform=node", + "vscode:prepublish": "pnpm run compile:ts-server && pnpm run compile:vscode", + "lint": "pnpm eslint ./ts-language-server/src ./vscode-metatype-support/src --ext .ts", + "vscode:package": "pnpm vsce package --no-dependencies", + "vscode:publish": "pnpm vsce publish --no-dependencies", + "test:ts-server": "cd ts-language-server && pnpm test", + "clean": "rm -rf ./ts-language-server/out && rm -rf ./vscode-metatype-support/out" + }, + "devDependencies": { + "@types/node": "^16.18.65", + "@types/vscode": "^1.75.0", + "@typescript-eslint/eslint-plugin": "^6.14.0", + "@typescript-eslint/parser": "^6.14.0", + "@vscode/vsce": "^2.22.0", + "esbuild": "^0.19.10", + "eslint": "^8.54.0", + "eslint-config-standard-with-typescript": "^42.0.0", + "eslint-plugin-import": "^2.25.2", + "eslint-plugin-n": "^15.0.0 || ^16.0.0 ", + "eslint-plugin-promise": "^6.0.0", + "typescript": "^5.3.2", + "tsx": "4.7", + "lcov": "1.16" + } } diff --git a/meta-lsp/ts-language-server/package.json b/meta-lsp/ts-language-server/package.json index ede93cae53..99b4fdfd05 100644 --- a/meta-lsp/ts-language-server/package.json +++ b/meta-lsp/ts-language-server/package.json @@ -1,24 +1,24 @@ { - "name": "typegraph-ts-server", - "description": "TypeScript language server for TypeGraph", - "author": "Metatype Team", - "version": "0.4.2", - "repository": { - "type": "git", - "url": "https://github.com/metatypedev/metatype" - }, - "main": "out/server", - "scripts": { - "lint": "eslint ./src --ext .ts", - "test": "node --test --import=tsx tests/*.test.ts" - }, - "dependencies": { - "vscode-languageserver": "9.0", - "vscode-languageserver-types": "3.17", - "vscode-languageserver-protocol": "3.17", - "vscode-languageserver-textdocument": "1.0", - "tree-sitter": "0.20", - "tree-sitter-typescript": "0.20", - "ts-lsp-client": "1.0" - } + "name": "typegraph-ts-server", + "description": "TypeScript language server for TypeGraph", + "author": "Metatype Team", + "version": "0.4.2", + "repository": { + "type": "git", + "url": "https://github.com/metatypedev/metatype" + }, + "main": "out/server", + "scripts": { + "lint": "eslint ./src --ext .ts", + "test": "node --test --import=tsx tests/*.test.ts" + }, + "dependencies": { + "vscode-languageserver": "9.0", + "vscode-languageserver-types": "3.17", + "vscode-languageserver-protocol": "3.17", + "vscode-languageserver-textdocument": "1.0", + "tree-sitter": "0.20", + "tree-sitter-typescript": "0.20", + "ts-lsp-client": "1.0" + } } diff --git a/meta-lsp/vscode-metatype-support/package.json b/meta-lsp/vscode-metatype-support/package.json index e28f33ebe7..7195809840 100644 --- a/meta-lsp/vscode-metatype-support/package.json +++ b/meta-lsp/vscode-metatype-support/package.json @@ -1,29 +1,29 @@ { - "name": "vscode-metatype-support", - "description": "VSCode extension for Metatype support", - "author": "Metatype Team", - "version": "0.4.2", - "repository": { - "type": "git", - "url": "https://github.com/metatypedev/metatype" - }, - "publisher": "metatypedev", - "engines": { - "vscode": "^1.75.0" - }, - "activationEvents": [ - "onLanguage:typescript", - "onLanguage:javascript" - ], - "main": "./out/extension", - "contributes": {}, - "scripts": { - "vscode:prepublish": "cp ../ts-language-server/out/server.js out/ && pnpm run compile --minify", - "package": "pnpm vsce package --no-dependencies", - "publish": "pnpm vsce publish --no-dependencies", - "lint": "eslint ./src --ext .ts" - }, - "dependencies": { - "vscode-languageclient": "^8.1.0" - } + "name": "vscode-metatype-support", + "description": "VSCode extension for Metatype support", + "author": "Metatype Team", + "version": "0.4.2", + "repository": { + "type": "git", + "url": "https://github.com/metatypedev/metatype" + }, + "publisher": "metatypedev", + "engines": { + "vscode": "^1.75.0" + }, + "activationEvents": [ + "onLanguage:typescript", + "onLanguage:javascript" + ], + "main": "./out/extension", + "contributes": {}, + "scripts": { + "vscode:prepublish": "cp ../ts-language-server/out/server.js out/ && pnpm run compile --minify", + "package": "pnpm vsce package --no-dependencies", + "publish": "pnpm vsce publish --no-dependencies", + "lint": "eslint ./src --ext .ts" + }, + "dependencies": { + "vscode-languageclient": "^8.1.0" + } } diff --git a/typegate/tests/metagen/__snapshots__/metagen_test.ts.snap b/typegate/tests/metagen/__snapshots__/metagen_test.ts.snap new file mode 100644 index 0000000000..0b14e4bc1c --- /dev/null +++ b/typegate/tests/metagen/__snapshots__/metagen_test.ts.snap @@ -0,0 +1,1069 @@ +export const snapshot = {}; + +snapshot[`Metagen within sdk 1`] = ` +[ + { + content: 'package.name = "example_metagen_mdk" +package.edition = "2021" +package.version = "0.0.1" + +[lib] +path = "lib.rs" +crate-type = ["cdylib", "rlib"] + +[dependencies] +anyhow = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +wit-bindgen = "0.22.0" +', + overwrite: false, + path: "./workspace/some/base/path/rust/Cargo.toml", + }, + { + content: \`from types import NoneType +from typing import Callable, List, Union, get_origin, ForwardRef +from dataclasses import dataclass, asdict, fields + +FORWARD_REFS = {} + +class Struct: + def try_new(dt_class, val: any): + # Object + ftypes = {f.name: f.type for f in fields(dt_class)} + attrs = {} + for f in val: + fval = val[f] + ftype = ftypes[f] + serialized = False + # Union + if get_origin(ftype) is Union: + try: + attrs[f] = Struct.try_union(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # List + elif get_origin(ftype) is list: + try: + attrs[f] = Struct.try_typed_list(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # Any + if not serialized: + if type(ftype) is str and ftype in FORWARD_REFS: + klass = FORWARD_REFS[ftype] + attrs[f] = Struct.new(klass, fval) + else: + attrs[f] = Struct.new(ftype, fval) + return dt_class(**attrs) + + def try_typed_list(tpe: any, items: any): + hint = tpe.__args__[0] + klass = FORWARD_REFS[hint.__forward_arg__] if type(hint) is ForwardRef else hint + return [Struct.new(klass, v) for v in items] + + def try_union(variants: List[any], val: any): + errors = [] + for variant in variants: + try: + if variant is NoneType: + if val is None: + return None + else: + continue + if get_origin(variant) is list: + if type(val) is list: + return Struct.try_typed_list(variant, val) + else: + continue + klass = FORWARD_REFS[variant.__forward_arg__] + return Struct.try_new(klass, val) + except Exception as e: + errors.append(str(e)) + raise Exception("\\\\n".join(errors)) + + + def new(dt_class: any, val: any): + try: + return Struct.try_new(dt_class, val) + except: + return val + + def repr(self): + return asdict(self) + + +@dataclass +class Object7(Struct): + name: str + + +FORWARD_REFS['Object7'] = Object7 + +@dataclass +class Student(Struct): + id: int + name: str + peers: Union[List['Student'], None] + + +FORWARD_REFS['Student'] = Student + +@dataclass +class TwoInput(Struct): + name: str + + +FORWARD_REFS['TwoInput'] = TwoInput + +Type8Student = List['Student'] +TypeString6 = str + + +def __repr(value: any): + if isinstance(value, Struct): + return value.repr() + return value + + +def typed_fnOne(user_fn: Callable[[Object7], Type8Student]): + def exported_wrapper(raw_inp): + inp: Object7 = Struct.new(Object7, raw_inp) + out: Type8Student = user_fn(inp) + if type(out) is list: + return [__repr(v) for v in out] + return __repr(out) + return exported_wrapper + +def typed_fnTwo(user_fn: Callable[[TwoInput], TypeString6]): + def exported_wrapper(raw_inp): + inp: TwoInput = Struct.new(TwoInput, raw_inp) + out: TypeString6 = user_fn(inp) + if type(out) is list: + return [__repr(v) for v in out] + return __repr(out) + return exported_wrapper + +\`, + overwrite: true, + path: "./workspace/some/base/path/python/./scripts/same_hit_types.py", + }, + { + content: " +mod mdk; +pub use mdk::*; + +/* +init_mat! { + hook: || { + // initialize global stuff here if you need it + MatBuilder::new() + // register function handlers here + .register_handler(stubs::MyFunc::erased(MyMat)) + } +} + +struct MyMat; + +// FIXME: use actual types from your mdk here +impl stubs::MyFunc for MyMat { + fn handle(&self, input: types::MyFuncIn, _cx: Ctx) -> anyhow::Result { + unimplemented!() + } +} +*/ +", + overwrite: false, + path: "./workspace/some/base/path/rust/lib.rs", + }, + { + content: \`// This file was @generated by metagen and is intended +// to be generated again on subsequent metagen runs. +#![cfg_attr(rustfmt, rustfmt_skip)] + +// gen-static-start +#![allow(unused)] + +pub mod wit { + wit_bindgen::generate!({ + pub_export_macro: true, + + inline: "package metatype:wit-wire; + +interface typegate-wire { + hostcall: func(req: tuple) -> result; +} + +interface mat-wire { + type json-str = string; + + record mat-info { + op-name: string, + mat-title: string, + mat-hash: string, + mat-data-json: string, + } + + record init-args { + metatype-version: string, + expected-ops: list + } + + record init-response { + ok: bool + } + + variant init-error { + version-mismatch(string), + unexpected-mat(mat-info), + other(string) + } + + init: func(args: init-args) -> result; + + record handle-req { + op-name: string, + in-json: json-str, + } + + variant handle-err { + no-handler, + in-json-err(string), + handler-err(string), + } + + handle: func(req: handle-req) -> result; +} + +world wit-wire { + import typegate-wire; + export mat-wire; +} +" + }); +} + +use std::cell::RefCell; +use std::collections::HashMap; + +use anyhow::Context; + +use wit::exports::metatype::wit_wire::mat_wire::*; + +pub type HandlerFn = Box Result>; + +pub struct ErasedHandler { + mat_id: String, + mat_trait: String, + mat_title: String, + handler_fn: HandlerFn, +} + +pub struct MatBuilder { + handlers: HashMap, +} + +impl MatBuilder { + pub fn new() -> Self { + Self { + handlers: Default::default(), + } + } + + pub fn register_handler(mut self, handler: ErasedHandler) -> Self { + self.handlers.insert(handler.mat_trait.clone(), handler); + self + } +} + +pub struct Router { + handlers: HashMap, +} + +impl Router { + pub fn from_builder(builder: MatBuilder) -> Self { + Self { + handlers: builder.handlers, + } + } + + pub fn init(&self, args: InitArgs) -> Result { + static MT_VERSION: &str = "0.4.1-0"; + if args.metatype_version != MT_VERSION { + return Err(InitError::VersionMismatch(MT_VERSION.into())); + } + for info in args.expected_ops { + let mat_trait = stubs::op_to_trait_name(&info.op_name); + if !self.handlers.contains_key(mat_trait) { + return Err(InitError::UnexpectedMat(info)); + } + } + Ok(InitResponse { ok: true }) + } + + pub fn handle(&self, req: HandleReq) -> Result { + let mat_trait = stubs::op_to_trait_name(&req.op_name); + let Some(handler) = self.handlers.get(mat_trait) else { + return Err(HandleErr::NoHandler); + }; + let cx = Ctx { + gql: GraphqlClient {}, + }; + (handler.handler_fn)(&req.in_json, cx) + } +} + +pub type InitCallback = fn() -> anyhow::Result; + +thread_local! { + pub static MAT_STATE: RefCell = panic!("MDK_STATE has not been initialized"); +} + +pub struct Ctx { + gql: GraphqlClient, +} + +pub struct GraphqlClient {} + +#[macro_export] +macro_rules! init_mat { + (hook: \$init_hook:expr) => { + struct MatWireGuest; + use wit::exports::metatype::wit_wire::mat_wire::*; + wit::export!(MatWireGuest with_types_in wit); + + #[allow(unused)] + impl Guest for MatWireGuest { + fn handle(req: HandleReq) -> Result { + MAT_STATE.with(|router| { + let router = router.borrow(); + router.handle(req) + }) + } + + fn init(args: InitArgs) -> Result { + let hook = \$init_hook; + let router = Router::from_builder(hook()); + let resp = router.init(args)?; + MAT_STATE.set(router); + Ok(resp) + } + } + }; +} +// gen-static-end +use types::*; +pub mod types { + use super::*; + #[derive(Debug, serde::Serialize, serde::Deserialize)] + pub struct Object7 { + pub name: String, + } + pub type Student3 = Vec; + pub type Student43 = Option; + #[derive(Debug, serde::Serialize, serde::Deserialize)] + pub struct Student { + pub id: i64, + pub name: String, + pub peers: Student43, + } + pub type Student8 = Vec; + #[derive(Debug, serde::Serialize, serde::Deserialize)] + pub struct TwoInput { + pub name: String, + } +} +use stubs::*; +pub mod stubs { + use super::*; + pub fn op_to_trait_name(op_name: &str) -> &'static str { + match op_name { + _ => panic!("unrecognized op_name: {op_name}"), + } + } +} +\`, + overwrite: true, + path: "./workspace/some/base/path/rust/mdk.rs", + }, + { + content: \`from types import NoneType +from typing import Callable, List, Union, get_origin, ForwardRef +from dataclasses import dataclass, asdict, fields + +FORWARD_REFS = {} + +class Struct: + def try_new(dt_class, val: any): + # Object + ftypes = {f.name: f.type for f in fields(dt_class)} + attrs = {} + for f in val: + fval = val[f] + ftype = ftypes[f] + serialized = False + # Union + if get_origin(ftype) is Union: + try: + attrs[f] = Struct.try_union(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # List + elif get_origin(ftype) is list: + try: + attrs[f] = Struct.try_typed_list(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # Any + if not serialized: + if type(ftype) is str and ftype in FORWARD_REFS: + klass = FORWARD_REFS[ftype] + attrs[f] = Struct.new(klass, fval) + else: + attrs[f] = Struct.new(ftype, fval) + return dt_class(**attrs) + + def try_typed_list(tpe: any, items: any): + hint = tpe.__args__[0] + klass = FORWARD_REFS[hint.__forward_arg__] if type(hint) is ForwardRef else hint + return [Struct.new(klass, v) for v in items] + + def try_union(variants: List[any], val: any): + errors = [] + for variant in variants: + try: + if variant is NoneType: + if val is None: + return None + else: + continue + if get_origin(variant) is list: + if type(val) is list: + return Struct.try_typed_list(variant, val) + else: + continue + klass = FORWARD_REFS[variant.__forward_arg__] + return Struct.try_new(klass, val) + except Exception as e: + errors.append(str(e)) + raise Exception("\\\\n".join(errors)) + + + def new(dt_class: any, val: any): + try: + return Struct.try_new(dt_class, val) + except: + return val + + def repr(self): + return asdict(self) + + +@dataclass +class Object7(Struct): + name: str + + +FORWARD_REFS['Object7'] = Object7 + +@dataclass +class Student(Struct): + id: int + name: str + peers: Union[List['Student'], None] + + +FORWARD_REFS['Student'] = Student + + + +def __repr(value: any): + if isinstance(value, Struct): + return value.repr() + return value + + +def typed_three(user_fn: Callable[[Object7], Student]): + def exported_wrapper(raw_inp): + inp: Object7 = Struct.new(Object7, raw_inp) + out: Student = user_fn(inp) + if type(out) is list: + return [__repr(v) for v in out] + return __repr(out) + return exported_wrapper + +\`, + overwrite: true, + path: "./workspace/some/base/path/python/other_types.py", + }, + { + content: 'from .same_hit_types import Object7, typed_fnOne, typed_fnTwo, Type8Student, TwoInput, TypeString6 + + +@typed_fnOne +def fnOne(inp: Object7) -> Type8Student: + # TODO: write your logic here + raise Exception("fnOne not implemented") + +@typed_fnTwo +def fnTwo(inp: TwoInput) -> TypeString6: + # TODO: write your logic here + raise Exception("fnTwo not implemented") + +', + overwrite: false, + path: "./workspace/some/base/path/python/./scripts/same_hit.py", + }, + { + content: 'from .other_types import typed_three, Object7, Student + + +@typed_three +def three(inp: Object7) -> Student: + # TODO: write your logic here + raise Exception("three not implemented") + +', + overwrite: false, + path: "./workspace/some/base/path/python/other.py", + }, +] +`; + +snapshot[`Metagen within sdk 2`] = ` +[ + { + content: 'package.name = "example_metagen_mdk" +package.edition = "2021" +package.version = "0.0.1" + +[lib] +path = "lib.rs" +crate-type = ["cdylib", "rlib"] + +[dependencies] +anyhow = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +wit-bindgen = "0.22.0" +', + overwrite: false, + path: "./workspace/some/base/path/rust/Cargo.toml", + }, + { + content: \`from types import NoneType +from typing import Callable, List, Union, get_origin, ForwardRef +from dataclasses import dataclass, asdict, fields + +FORWARD_REFS = {} + +class Struct: + def try_new(dt_class, val: any): + # Object + ftypes = {f.name: f.type for f in fields(dt_class)} + attrs = {} + for f in val: + fval = val[f] + ftype = ftypes[f] + serialized = False + # Union + if get_origin(ftype) is Union: + try: + attrs[f] = Struct.try_union(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # List + elif get_origin(ftype) is list: + try: + attrs[f] = Struct.try_typed_list(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # Any + if not serialized: + if type(ftype) is str and ftype in FORWARD_REFS: + klass = FORWARD_REFS[ftype] + attrs[f] = Struct.new(klass, fval) + else: + attrs[f] = Struct.new(ftype, fval) + return dt_class(**attrs) + + def try_typed_list(tpe: any, items: any): + hint = tpe.__args__[0] + klass = FORWARD_REFS[hint.__forward_arg__] if type(hint) is ForwardRef else hint + return [Struct.new(klass, v) for v in items] + + def try_union(variants: List[any], val: any): + errors = [] + for variant in variants: + try: + if variant is NoneType: + if val is None: + return None + else: + continue + if get_origin(variant) is list: + if type(val) is list: + return Struct.try_typed_list(variant, val) + else: + continue + klass = FORWARD_REFS[variant.__forward_arg__] + return Struct.try_new(klass, val) + except Exception as e: + errors.append(str(e)) + raise Exception("\\\\n".join(errors)) + + + def new(dt_class: any, val: any): + try: + return Struct.try_new(dt_class, val) + except: + return val + + def repr(self): + return asdict(self) + + +@dataclass +class Object7(Struct): + name: str + + +FORWARD_REFS['Object7'] = Object7 + +@dataclass +class Student(Struct): + id: int + name: str + peers: Union[List['Student'], None] + + +FORWARD_REFS['Student'] = Student + +@dataclass +class TwoInput(Struct): + name: str + + +FORWARD_REFS['TwoInput'] = TwoInput + +Type8Student = List['Student'] +TypeString6 = str + + +def __repr(value: any): + if isinstance(value, Struct): + return value.repr() + return value + + +def typed_fnOne(user_fn: Callable[[Object7], Type8Student]): + def exported_wrapper(raw_inp): + inp: Object7 = Struct.new(Object7, raw_inp) + out: Type8Student = user_fn(inp) + if type(out) is list: + return [__repr(v) for v in out] + return __repr(out) + return exported_wrapper + +def typed_fnTwo(user_fn: Callable[[TwoInput], TypeString6]): + def exported_wrapper(raw_inp): + inp: TwoInput = Struct.new(TwoInput, raw_inp) + out: TypeString6 = user_fn(inp) + if type(out) is list: + return [__repr(v) for v in out] + return __repr(out) + return exported_wrapper + +\`, + overwrite: true, + path: "./workspace/some/base/path/python/./scripts/same_hit_types.py", + }, + { + content: " +mod mdk; +pub use mdk::*; + +/* +init_mat! { + hook: || { + // initialize global stuff here if you need it + MatBuilder::new() + // register function handlers here + .register_handler(stubs::MyFunc::erased(MyMat)) + } +} + +struct MyMat; + +// FIXME: use actual types from your mdk here +impl stubs::MyFunc for MyMat { + fn handle(&self, input: types::MyFuncIn, _cx: Ctx) -> anyhow::Result { + unimplemented!() + } +} +*/ +", + overwrite: false, + path: "./workspace/some/base/path/rust/lib.rs", + }, + { + content: \`// This file was @generated by metagen and is intended +// to be generated again on subsequent metagen runs. +#![cfg_attr(rustfmt, rustfmt_skip)] + +// gen-static-start +#![allow(unused)] + +pub mod wit { + wit_bindgen::generate!({ + pub_export_macro: true, + + inline: "package metatype:wit-wire; + +interface typegate-wire { + hostcall: func(req: tuple) -> result; +} + +interface mat-wire { + type json-str = string; + + record mat-info { + op-name: string, + mat-title: string, + mat-hash: string, + mat-data-json: string, + } + + record init-args { + metatype-version: string, + expected-ops: list + } + + record init-response { + ok: bool + } + + variant init-error { + version-mismatch(string), + unexpected-mat(mat-info), + other(string) + } + + init: func(args: init-args) -> result; + + record handle-req { + op-name: string, + in-json: json-str, + } + + variant handle-err { + no-handler, + in-json-err(string), + handler-err(string), + } + + handle: func(req: handle-req) -> result; +} + +world wit-wire { + import typegate-wire; + export mat-wire; +} +" + }); +} + +use std::cell::RefCell; +use std::collections::HashMap; + +use anyhow::Context; + +use wit::exports::metatype::wit_wire::mat_wire::*; + +pub type HandlerFn = Box Result>; + +pub struct ErasedHandler { + mat_id: String, + mat_trait: String, + mat_title: String, + handler_fn: HandlerFn, +} + +pub struct MatBuilder { + handlers: HashMap, +} + +impl MatBuilder { + pub fn new() -> Self { + Self { + handlers: Default::default(), + } + } + + pub fn register_handler(mut self, handler: ErasedHandler) -> Self { + self.handlers.insert(handler.mat_trait.clone(), handler); + self + } +} + +pub struct Router { + handlers: HashMap, +} + +impl Router { + pub fn from_builder(builder: MatBuilder) -> Self { + Self { + handlers: builder.handlers, + } + } + + pub fn init(&self, args: InitArgs) -> Result { + static MT_VERSION: &str = "0.4.1-0"; + if args.metatype_version != MT_VERSION { + return Err(InitError::VersionMismatch(MT_VERSION.into())); + } + for info in args.expected_ops { + let mat_trait = stubs::op_to_trait_name(&info.op_name); + if !self.handlers.contains_key(mat_trait) { + return Err(InitError::UnexpectedMat(info)); + } + } + Ok(InitResponse { ok: true }) + } + + pub fn handle(&self, req: HandleReq) -> Result { + let mat_trait = stubs::op_to_trait_name(&req.op_name); + let Some(handler) = self.handlers.get(mat_trait) else { + return Err(HandleErr::NoHandler); + }; + let cx = Ctx { + gql: GraphqlClient {}, + }; + (handler.handler_fn)(&req.in_json, cx) + } +} + +pub type InitCallback = fn() -> anyhow::Result; + +thread_local! { + pub static MAT_STATE: RefCell = panic!("MDK_STATE has not been initialized"); +} + +pub struct Ctx { + gql: GraphqlClient, +} + +pub struct GraphqlClient {} + +#[macro_export] +macro_rules! init_mat { + (hook: \$init_hook:expr) => { + struct MatWireGuest; + use wit::exports::metatype::wit_wire::mat_wire::*; + wit::export!(MatWireGuest with_types_in wit); + + #[allow(unused)] + impl Guest for MatWireGuest { + fn handle(req: HandleReq) -> Result { + MAT_STATE.with(|router| { + let router = router.borrow(); + router.handle(req) + }) + } + + fn init(args: InitArgs) -> Result { + let hook = \$init_hook; + let router = Router::from_builder(hook()); + let resp = router.init(args)?; + MAT_STATE.set(router); + Ok(resp) + } + } + }; +} +// gen-static-end +use types::*; +pub mod types { + use super::*; + #[derive(Debug, serde::Serialize, serde::Deserialize)] + pub struct Object7 { + pub name: String, + } + pub type Student3 = Vec; + pub type Student43 = Option; + #[derive(Debug, serde::Serialize, serde::Deserialize)] + pub struct Student { + pub id: i64, + pub name: String, + pub peers: Student43, + } + pub type Student8 = Vec; + #[derive(Debug, serde::Serialize, serde::Deserialize)] + pub struct TwoInput { + pub name: String, + } +} +use stubs::*; +pub mod stubs { + use super::*; + pub fn op_to_trait_name(op_name: &str) -> &'static str { + match op_name { + _ => panic!("unrecognized op_name: {op_name}"), + } + } +} +\`, + overwrite: true, + path: "./workspace/some/base/path/rust/mdk.rs", + }, + { + content: \`from types import NoneType +from typing import Callable, List, Union, get_origin, ForwardRef +from dataclasses import dataclass, asdict, fields + +FORWARD_REFS = {} + +class Struct: + def try_new(dt_class, val: any): + # Object + ftypes = {f.name: f.type for f in fields(dt_class)} + attrs = {} + for f in val: + fval = val[f] + ftype = ftypes[f] + serialized = False + # Union + if get_origin(ftype) is Union: + try: + attrs[f] = Struct.try_union(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # List + elif get_origin(ftype) is list: + try: + attrs[f] = Struct.try_typed_list(ftype.__args__, fval) + serialized = True + except Exception as _e: + pass + # Any + if not serialized: + if type(ftype) is str and ftype in FORWARD_REFS: + klass = FORWARD_REFS[ftype] + attrs[f] = Struct.new(klass, fval) + else: + attrs[f] = Struct.new(ftype, fval) + return dt_class(**attrs) + + def try_typed_list(tpe: any, items: any): + hint = tpe.__args__[0] + klass = FORWARD_REFS[hint.__forward_arg__] if type(hint) is ForwardRef else hint + return [Struct.new(klass, v) for v in items] + + def try_union(variants: List[any], val: any): + errors = [] + for variant in variants: + try: + if variant is NoneType: + if val is None: + return None + else: + continue + if get_origin(variant) is list: + if type(val) is list: + return Struct.try_typed_list(variant, val) + else: + continue + klass = FORWARD_REFS[variant.__forward_arg__] + return Struct.try_new(klass, val) + except Exception as e: + errors.append(str(e)) + raise Exception("\\\\n".join(errors)) + + + def new(dt_class: any, val: any): + try: + return Struct.try_new(dt_class, val) + except: + return val + + def repr(self): + return asdict(self) + + +@dataclass +class Object7(Struct): + name: str + + +FORWARD_REFS['Object7'] = Object7 + +@dataclass +class Student(Struct): + id: int + name: str + peers: Union[List['Student'], None] + + +FORWARD_REFS['Student'] = Student + + + +def __repr(value: any): + if isinstance(value, Struct): + return value.repr() + return value + + +def typed_three(user_fn: Callable[[Object7], Student]): + def exported_wrapper(raw_inp): + inp: Object7 = Struct.new(Object7, raw_inp) + out: Student = user_fn(inp) + if type(out) is list: + return [__repr(v) for v in out] + return __repr(out) + return exported_wrapper + +\`, + overwrite: true, + path: "./workspace/some/base/path/python/other_types.py", + }, + { + content: 'from .same_hit_types import Object7, typed_fnOne, typed_fnTwo, Type8Student, TwoInput, TypeString6 + + +@typed_fnOne +def fnOne(inp: Object7) -> Type8Student: + # TODO: write your logic here + raise Exception("fnOne not implemented") + +@typed_fnTwo +def fnTwo(inp: TwoInput) -> TypeString6: + # TODO: write your logic here + raise Exception("fnTwo not implemented") + +', + overwrite: false, + path: "./workspace/some/base/path/python/./scripts/same_hit.py", + }, + { + content: 'from .other_types import typed_three, Object7, Student + + +@typed_three +def three(inp: Object7) -> Student: + # TODO: write your logic here + raise Exception("three not implemented") + +', + overwrite: false, + path: "./workspace/some/base/path/python/other.py", + }, +] +`; diff --git a/typegate/tests/metagen/metagen_test.ts b/typegate/tests/metagen/metagen_test.ts index 00fc7567bf..6b4311742b 100644 --- a/typegate/tests/metagen/metagen_test.ts +++ b/typegate/tests/metagen/metagen_test.ts @@ -26,7 +26,7 @@ typegates: password: password metagen: - targets: + targets: main: mdk_rust: path: ${genCratePath} @@ -59,3 +59,104 @@ members = ["mdk/"] 0, ); }); + +Meta.test("metagen python runs on cyclic types", async (t) => { + const tmpDir = await newTempDir(); + t.addCleanup(() => Deno.remove(tmpDir, { recursive: true })); + + const typegraphPath = join( + import.meta.dirname!, + "typegraphs/python.py", + ); + const basePath = join(tmpDir, "mdk"); + + Deno.writeTextFile( + join(tmpDir, "metatype.yml"), + ` +typegates: + dev: + url: "http://localhost:7890" + username: admin1 + password: password2 + +metagen: + targets: + my_target: + mdk_python: + path: ${basePath} + typegraph_path: ${typegraphPath} +`, + ); + + assertEquals( + (await Meta.cli({}, ...`-C ${tmpDir} gen mdk my_target`.split(" "))).code, + 0, + ); +}); + +Meta.test("Metagen within sdk", async (t) => { + const workspace = "./workspace"; + const targetName = "my_target"; + const genConfig = { + targets: { + my_target: { + mdk_rust: { + typegraph: "example-metagen", + path: "some/base/path/rust", + }, + mdk_python: { + typegraph: "example-metagen", + path: "some/base/path/python", + }, + }, + }, + }; + + const sdkResults = [] as Array; + + await t.should("Run metagen within typescript", async () => { + const { tg } = await import("./typegraphs/metagen.mjs"); + const { Metagen } = await import("@typegraph/sdk/metagen.js"); + const metagen = new Metagen(workspace, genConfig); + const generated = metagen.dryRun(tg, targetName); + await t.assertSnapshot(generated); + + sdkResults.push(JSON.stringify(generated, null, 2)); + }); + + await t.should("Run metagen within python", async () => { + const typegraphPath = join( + import.meta.dirname!, + "./typegraphs/metagen.py", + ); + const command = new Deno.Command("python3", { + args: [typegraphPath], + env: { + workspace_path: workspace, + gen_config: JSON.stringify(genConfig), + target_name: targetName, + }, + stderr: "piped", + stdout: "piped", + }); + + const child = command.spawn(); + const output = await child.output(); + if (output.success) { + const generated = JSON.parse(new TextDecoder().decode(output.stdout)); + await t.assertSnapshot(generated); + + sdkResults.push(JSON.stringify(generated, null, 2)); + } else { + const err = new TextDecoder().decode(output.stderr); + throw new Error(`metagen python: ${err}`); + } + }); + + if (sdkResults.length > 0) { + await t.should("SDKs should produce same metagen output", () => { + const [fromTs, fromPy] = sdkResults; + assertEquals(fromTs, fromPy); + }); + } +}); diff --git a/typegate/tests/metagen/typegraphs/metagen.mjs b/typegate/tests/metagen/typegraphs/metagen.mjs new file mode 100644 index 0000000000..cb226029f0 --- /dev/null +++ b/typegate/tests/metagen/typegraphs/metagen.mjs @@ -0,0 +1,44 @@ +import { Policy, t, typegraph } from "@typegraph/sdk/index.js"; +import { PythonRuntime } from "@typegraph/sdk/runtimes/python.js"; + +export const tg = await typegraph({ + name: "example-metagen", +}, (g) => { + const python = new PythonRuntime(); + const pub = Policy.public(); + const student = t.struct( + { + id: t.integer({}, { asId: true }), + name: t.string(), + peers: t.list(g.ref("Student")).optional(), + }, + { name: "Student" }, + ); + + g.expose({ + one: python.import( + t.struct({ name: t.string() }), + t.list(student), + { + module: "./scripts/same_hit.py", + name: "fnOne", + }, + ), + two: python.import( + t.struct({ name: t.string() }).rename("TwoInput"), + t.string(), + { + module: "./scripts/same_hit.py", + name: "fnTwo", + }, + ), + three: python.import( + t.struct({ name: t.string() }), + student, + { + module: "other.py", + name: "three", + }, + ), + }, pub); +}); diff --git a/typegate/tests/metagen/typegraphs/metagen.py b/typegate/tests/metagen/typegraphs/metagen.py new file mode 100644 index 0000000000..53192e52d2 --- /dev/null +++ b/typegate/tests/metagen/typegraphs/metagen.py @@ -0,0 +1,52 @@ +from dataclasses import asdict +from os import getenv +from typegraph import typegraph, Policy, t, Graph +from typegraph.graph.metagen import Metagen +from typegraph.runtimes.python import PythonRuntime +import json + + +@typegraph() +def example_metagen(g: Graph): + python = PythonRuntime() + pub = Policy.public() + student = t.struct( + { + "id": t.integer(as_id=True), + "name": t.string(), + "peers": t.list(g.ref("Student")).optional(), + }, + name="Student", + ) + g.expose( + pub, + one=python.import_( + t.struct({"name": t.string()}), + t.list(student), + module="./scripts/same_hit.py", + name="fnOne", + ), + two=python.import_( + t.struct({"name": t.string()}).rename("TwoInput"), + t.string(), + module="./scripts/same_hit.py", + name="fnTwo", + ), + three=python.import_( + t.struct({"name": t.string()}), + student, + module="other.py", + name="three", + ), + ) + + +tg = example_metagen() +workspace_path = getenv("workspace_path") +target_name = getenv("target_name") +gen_config = json.loads(getenv("gen_config")) + +metagen = Metagen(workspace_path, gen_config) +items = metagen.dry_run(tg, target_name, None) + +print(json.dumps([asdict(it) for it in items], indent=2)) diff --git a/typegate/tests/metagen/typegraphs/python.py b/typegate/tests/metagen/typegraphs/python.py new file mode 100644 index 0000000000..38534a504e --- /dev/null +++ b/typegate/tests/metagen/typegraphs/python.py @@ -0,0 +1,47 @@ +from typegraph import typegraph, Policy, t, Graph +from typegraph.runtimes.python import PythonRuntime + + +@typegraph() +def example(g: Graph): + references = t.struct( + {"string": t.string(), "example": g.ref("Example").optional()}, + name="References", + ) + example = t.struct( + { + "string": t.string(), + "integer": t.integer(), + "email": t.email().optional(), + "list_integer": t.list(t.integer()), + "opt_union_flat": t.union([t.integer(), t.integer(), t.float()]).optional(), + "reference": t.list(g.ref("Example")).optional(), + "nested_ref": t.struct( + {"either": t.either([g.ref("Example"), references])} + ), + }, + name="Example", + ) + python = PythonRuntime() + pub = Policy.public() + g.expose( + pub, + duplicate=python.import_( + example, + t.list(example).rename("Duplicates"), + name="duplicate_one", + module="scripts/example.py", + ), + duplicate_hit_same_file=python.import_( + example, + t.list(example).rename("DuplicatesTwo"), + name="duplicate_two", + module="scripts/example.py", + ), + another=python.import_( + example, + t.integer(), + name="another", + module="scripts/another.py", + ), + ) diff --git a/typegraph/core/Cargo.toml b/typegraph/core/Cargo.toml index f67b4b92d8..aad75a5138 100644 --- a/typegraph/core/Cargo.toml +++ b/typegraph/core/Cargo.toml @@ -15,7 +15,9 @@ wit-bindgen = "0.24.0" regex.workspace = true indexmap.workspace = true common = { path = "../../libs/common" } +metagen = { path = "../../libs/metagen", features = [] } indoc.workspace = true +anyhow.workspace = true graphql-parser = "0.4.0" sha2 = "0.10.8" paste = "1.0.14" diff --git a/typegraph/core/src/utils/fs_host.rs b/typegraph/core/src/utils/fs_host.rs index 7203273abc..7a13eda4ff 100644 --- a/typegraph/core/src/utils/fs_host.rs +++ b/typegraph/core/src/utils/fs_host.rs @@ -16,16 +16,15 @@ use common::archive::{ use indexmap::IndexMap; use sha2::{Digest, Sha256}; -pub fn read_text_file>(path: P) -> Result { - read_file(&path.into()).and_then(|bytes| { +pub fn read_text_file(path: &Path) -> Result { + read_file(&path.display().to_string()).and_then(|bytes| { let s = std::str::from_utf8(&bytes).map_err(|e| e.to_string())?; Ok(s.to_owned()) }) } -#[allow(unused)] -pub fn write_text_file>(path: P, text: P) -> Result<(), String> { - write_file(&path.into(), text.into().as_bytes()) +pub fn write_text_file(path: &Path, text: String) -> Result<(), String> { + write_file(&path.display().to_string(), text.as_bytes()) } pub fn common_prefix_paths(paths: &[PathBuf]) -> Option { @@ -162,7 +161,7 @@ pub fn load_tg_ignore_file() -> Result, String> { let file = cwd()?.join(".tgignore"); match path_exists(&file)? { - true => read_text_file(file.to_string_lossy()).map(|content| { + true => read_text_file(&file).map(|content| { content .lines() .filter_map(|line| { diff --git a/typegraph/core/src/utils/metagen_utils.rs b/typegraph/core/src/utils/metagen_utils.rs new file mode 100644 index 0000000000..46eda1ac6e --- /dev/null +++ b/typegraph/core/src/utils/metagen_utils.rs @@ -0,0 +1,26 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + +use common::typegraph::Typegraph; +use metagen::{GeneratorInputOrder, GeneratorInputResolved, InputResolverSync}; + +#[derive(Clone)] +pub struct RawTgResolver { + pub tg: Typegraph, +} + +impl InputResolverSync for RawTgResolver { + fn resolve( + &self, + order: metagen::GeneratorInputOrder, + ) -> anyhow::Result { + match order { + GeneratorInputOrder::TypegraphFromTypegate { .. } => { + Ok(GeneratorInputResolved::TypegraphFromTypegate { + raw: self.tg.clone(), + }) + } + GeneratorInputOrder::TypegraphFromPath { .. } => unimplemented!(), + } + } +} diff --git a/typegraph/core/src/utils/mod.rs b/typegraph/core/src/utils/mod.rs index eebe36dc83..652519eba7 100644 --- a/typegraph/core/src/utils/mod.rs +++ b/typegraph/core/src/utils/mod.rs @@ -2,22 +2,25 @@ // SPDX-License-Identifier: MPL-2.0 use std::collections::HashMap; +use std::path::PathBuf; +use crate::utils::metagen_utils::RawTgResolver; use common::typegraph::{Auth, AuthProtocol}; use indexmap::IndexMap; use serde_json::json; +use self::oauth2::std::{named_provider, Oauth2Builder}; use crate::errors::Result; use crate::global_store::{get_sdk_version, NameRegistration, Store}; use crate::types::subgraph::Subgraph; use crate::types::{TypeDefExt, TypeId}; use crate::wit::core::{Guest, TypeBase, TypeId as CoreTypeId, TypeStruct}; -use crate::wit::utils::{Auth as WitAuth, QueryDeployParams}; +use crate::wit::utils::{Auth as WitAuth, MdkConfig, MdkOutput, QueryDeployParams}; use crate::Lib; - -use self::oauth2::std::{named_provider, Oauth2Builder}; +use std::path::Path; pub mod fs_host; +pub mod metagen_utils; mod oauth2; pub mod postprocess; pub mod reduce; @@ -267,6 +270,43 @@ impl crate::wit::utils::Guest for crate::Lib { Err(e) => Err(e), } } + + fn metagen_exec(config: MdkConfig) -> Result, String> { + let gen_config: metagen::Config = serde_json::from_str(&config.config_json) + .map_err(|e| format!("Load metagen config: {}", e))?; + + let tg = serde_json::from_str(&config.tg_json).map_err(|e| e.to_string())?; + let resolver = RawTgResolver { tg }; + + metagen::generate_target_sync( + &gen_config, + &config.target_name, + PathBuf::from(config.workspace_path), + resolver, + ) + .map(|map| { + map.0 + .iter() + .map(|(k, v)| MdkOutput { + path: k.to_string_lossy().to_string(), + content: v.contents.clone(), + overwrite: v.overwrite, + }) + .collect::>() + }) + .map_err(|e| format!("Generate target: {}", e)) + } + + fn metagen_write_files(items: Vec) -> Result<(), String> { + for item in items { + let path = fs_host::make_absolute(Path::new(&item.path))?; + if fs_host::path_exists(&path)? && !item.overwrite { + continue; + } + fs_host::write_text_file(&path, item.content)?; + } + Ok(()) + } } pub fn remove_injection(type_id: TypeId) -> Result { diff --git a/typegraph/core/src/utils/postprocess/mod.rs b/typegraph/core/src/utils/postprocess/mod.rs index 72c69a451c..f591581e0f 100644 --- a/typegraph/core/src/utils/postprocess/mod.rs +++ b/typegraph/core/src/utils/postprocess/mod.rs @@ -58,7 +58,7 @@ impl PostProcessor for TypegraphPostProcessor { #[allow(dead_code)] pub fn compress_and_encode(main_path: &Path) -> Result { - if let Err(e) = fs_host::read_text_file(main_path.display().to_string()) { + if let Err(e) = fs_host::read_text_file(main_path) { return Err(format!("Unable to read {:?}: {}", main_path.display(), e)); } diff --git a/typegraph/core/wit/typegraph.wit b/typegraph/core/wit/typegraph.wit index fbf55057e5..7d16c53d04 100644 --- a/typegraph/core/wit/typegraph.wit +++ b/typegraph/core/wit/typegraph.wit @@ -579,6 +579,22 @@ interface utils { remove-injections: func(type-id: type-id) -> result; get-cwd: func() -> result; + + record mdk-config { + workspace-path: string, + target-name: string, + config-json: string, + tg-json: string, + } + + record mdk-output { + path: string, + content: string, + overwrite: bool, + } + + metagen-exec: func(config: mdk-config) -> result, string>; + metagen-write-files: func(items: list) -> result<_, string>; } interface host { diff --git a/typegraph/node/sdk/src/metagen.ts b/typegraph/node/sdk/src/metagen.ts new file mode 100644 index 0000000000..50afd6bb29 --- /dev/null +++ b/typegraph/node/sdk/src/metagen.ts @@ -0,0 +1,54 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + +import { ArtifactResolutionConfig } from "./gen/interfaces/metatype-typegraph-core.js"; +import { TypegraphOutput } from "./typegraph.js"; +import { wit_utils } from "./wit.js"; +import { freezeTgOutput } from "./utils/func_utils.js"; +import { + MdkConfig, + MdkOutput, +} from "./gen/interfaces/metatype-typegraph-utils.js"; + +const codegenArtefactConfig = { + prismaMigration: { + globalAction: { + create: false, + reset: false, + }, + migrationDir: ".", + }, + disableArtifactResolution: true, + codegen: true, +} as ArtifactResolutionConfig; + +export class Metagen { + constructor(private workspacePath: string, private genConfig: unknown) {} + + private getMdkConfig( + tgOutput: TypegraphOutput, + targetName: string, + ) { + const frozenOut = freezeTgOutput(codegenArtefactConfig, tgOutput); + return { + configJson: JSON.stringify(this.genConfig), + tgJson: frozenOut.serialize(codegenArtefactConfig).tgJson, + targetName, + workspacePath: this.workspacePath, + } as MdkConfig; + } + + dryRun(tgOutput: TypegraphOutput, targetName: string, overwrite?: false) { + const mdkConfig = this.getMdkConfig(tgOutput, targetName); + return wit_utils.metagenExec(mdkConfig) + .map((value) => ({ + ...value, + overwrite: overwrite ?? value.overwrite, + })) as Array; + } + + run(tgOutput: TypegraphOutput, targetName: string, overwrite?: false) { + const items = this.dryRun(tgOutput, targetName, overwrite); + wit_utils.metagenWriteFiles(items); + } +} diff --git a/typegraph/node/sdk/src/tg_manage.ts b/typegraph/node/sdk/src/tg_manage.ts index 903e9b742b..5746d09b8c 100644 --- a/typegraph/node/sdk/src/tg_manage.ts +++ b/typegraph/node/sdk/src/tg_manage.ts @@ -2,10 +2,10 @@ // SPDX-License-Identifier: MPL-2.0 import { ArtifactResolutionConfig } from "./gen/interfaces/metatype-typegraph-core.js"; -import { BasicAuth, tgDeploy, tgRemove } from "./tg_deploy.js"; -import { TgFinalizationResult, TypegraphOutput } from "./typegraph.js"; +import { BasicAuth, tgDeploy } from "./tg_deploy.js"; +import { TypegraphOutput } from "./typegraph.js"; import { getEnvVariable } from "./utils/func_utils.js"; -import { dirname } from "node:path"; +import { freezeTgOutput } from "./utils/func_utils.js"; const PORT = "META_CLI_SERVER_PORT"; // meta-cli instance that executes the current file const SELF_PATH = "META_CLI_TG_PATH"; // path to the current file to uniquely identify the run results diff --git a/typegraph/node/sdk/src/utils/func_utils.ts b/typegraph/node/sdk/src/utils/func_utils.ts index 79eda1fb60..3d0ae27cfd 100644 --- a/typegraph/node/sdk/src/utils/func_utils.ts +++ b/typegraph/node/sdk/src/utils/func_utils.ts @@ -1,9 +1,14 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -import { InheritDef } from "../typegraph.js"; +import { + InheritDef, + TgFinalizationResult, + TypegraphOutput, +} from "../typegraph.js"; import { ReducePath } from "../gen/interfaces/metatype-typegraph-utils.js"; import { serializeStaticInjection } from "./injection_utils.js"; +import { ArtifactResolutionConfig } from "../gen/interfaces/metatype-typegraph-core.js"; export function stringifySymbol(symbol: symbol) { const name = symbol.toString().match(/\((.+)\)/)?.[1]; @@ -79,3 +84,18 @@ export function getAllEnvVariables(): any { const glob = globalThis as any; return glob?.process ? glob?.process.env : glob?.Deno.env.toObject(); } + +const frozenMemo: Record = {}; + +/** Create a reusable version of a `TypegraphOutput` */ +export function freezeTgOutput( + config: ArtifactResolutionConfig, + tgOutput: TypegraphOutput, +): TypegraphOutput { + frozenMemo[tgOutput.name] = frozenMemo[tgOutput.name] ?? + tgOutput.serialize(config); + return { + ...tgOutput, + serialize: (_: ArtifactResolutionConfig) => frozenMemo[tgOutput.name], + }; +} diff --git a/typegraph/python/typegraph/graph/metagen.py b/typegraph/python/typegraph/graph/metagen.py new file mode 100644 index 0000000000..868c981890 --- /dev/null +++ b/typegraph/python/typegraph/graph/metagen.py @@ -0,0 +1,75 @@ +# Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +# SPDX-License-Identifier: MPL-2.0 + +import json +from typing import List, Union +from typegraph.gen.exports.core import ( + ArtifactResolutionConfig, + MigrationAction, + MigrationConfig, +) +from typegraph.gen.exports.utils import MdkConfig, MdkOutput +from typegraph.gen.types import Err +from typegraph.graph.shared_types import TypegraphOutput +from typegraph.utils import freeze_tg_output +from typegraph.wit import store, wit_utils + +codegen_artefact_config = ArtifactResolutionConfig( + prisma_migration=MigrationConfig( + global_action=MigrationAction(create=False, reset=False), + migration_dir=".", + runtime_actions=None, + ), + dir=None, + prefix=None, + disable_artifact_resolution=True, + codegen=True, +) + + +class Metagen: + workspace_path: str = "" + gen_config: any + + def __init__(self, workspace_path: str, gen_config: any) -> None: + self.gen_config = gen_config + self.workspace_path = workspace_path + + def _get_mdk_config( + self, + tg_output: TypegraphOutput, + target_name: str, + ) -> MdkConfig: + frozen_out = freeze_tg_output(codegen_artefact_config, tg_output) + return MdkConfig( + tg_json=frozen_out.serialize(codegen_artefact_config).tgJson, + config_json=json.dumps(self.gen_config), + workspace_path=self.workspace_path, + target_name=target_name, + ) + + def dry_run( + self, + tg_output: TypegraphOutput, + target_name: str, + overwrite: Union[bool, None] = None, + ) -> List[MdkOutput]: + mdk_config = self._get_mdk_config(tg_output, target_name) + res = wit_utils.metagen_exec(store, mdk_config) + if isinstance(res, Err): + raise Exception(res.value) + for item in res.value: + if overwrite is not None: + item.overwrite = overwrite + return res.value + + def run( + self, + tg_output: TypegraphOutput, + target_name: str, + overwrite: Union[bool, None] = None, + ): + items = self.dry_run(tg_output, target_name, overwrite) + res = wit_utils.metagen_write_files(store, items) + if isinstance(res, Err): + raise Exception(res.value) diff --git a/typegraph/python/typegraph/graph/tg_manage.py b/typegraph/python/typegraph/graph/tg_manage.py index a3e0b9b205..5491d746df 100644 --- a/typegraph/python/typegraph/graph/tg_manage.py +++ b/typegraph/python/typegraph/graph/tg_manage.py @@ -15,6 +15,7 @@ ) from typegraph.graph.shared_types import BasicAuth, TypegraphOutput from typegraph.graph.tg_deploy import TypegraphDeployParams, tg_deploy +from typegraph.utils import freeze_tg_output PORT = "META_CLI_SERVER_PORT" # meta-cli instance that executes the current file SELF_PATH = ( diff --git a/typegraph/python/typegraph/utils.py b/typegraph/python/typegraph/utils.py index a83831c6c7..c20bf5ef0c 100644 --- a/typegraph/python/typegraph/utils.py +++ b/typegraph/python/typegraph/utils.py @@ -5,7 +5,9 @@ from functools import reduce from typing import Any, Dict, List, Optional, Tuple, Union +from typegraph.gen.exports.core import ArtifactResolutionConfig from typegraph.gen.exports.utils import ReducePath, ReduceValue +from typegraph.graph.shared_types import FinalizationResult, TypegraphOutput from typegraph.injection import InheritDef, serialize_static_injection from typegraph.wit import store, wit_utils @@ -76,3 +78,16 @@ def build_reduce_data(node: Any, paths: List[ReducePath], curr_path: List[str]): def unpack_tarb64(tar_b64: str, dest: str): return wit_utils.unpack_tarb64(store, tar_b64, dest) + + +frozen_memo: Dict[str, FinalizationResult] = {} + + +def freeze_tg_output( + config: ArtifactResolutionConfig, tg_output: TypegraphOutput +) -> TypegraphOutput: + if tg_output.name not in frozen_memo: + frozen_memo[tg_output.name] = tg_output.serialize(config) + return TypegraphOutput( + name=tg_output.name, serialize=lambda _: frozen_memo[tg_output.name] + ) diff --git a/website/docs/reference/runtimes/wasm/index.mdx b/website/docs/reference/runtimes/wasm/index.mdx index 197a288c80..8340d235eb 100644 --- a/website/docs/reference/runtimes/wasm/index.mdx +++ b/website/docs/reference/runtimes/wasm/index.mdx @@ -32,14 +32,13 @@ from typegraph.runtimes.wasm import WasmRuntime @typegraph def example(g: Graph): pub = Policy.public() - wasm = WasmRuntime() + wasm = WasmRuntime.reflected("path/to/your-compiled-component.wasm"); g.expose( - add=wasm.from_wasm( + add=wasm.from_export( t.struct({"a": t.integer(), "b": t.integer()}), t.integer(), func="add", # exported function - wasm="path/to/your-compiled-component.wasm", ) ) @@ -52,15 +51,14 @@ import { WasmRuntime } from "@typegraph/sdk/runtimes/wasm.js"; typegraph("example", (g) => { const pub = Policy.public(); - const wasm = new WasmRuntime(); + const wasm = WasmRuntime.reflected("path/to/your-compiled-component.wasm"); g.expose({ - add: wasm.fromWasm( + add: wasm.fromExport( t.struct({ a: t.integer(), b: t.integer() }), t.integer(), { func: "add", // exported function - wasm: "path/to/your-compiled-component.wasm" }, ).withPolicy(pub), });