Skip to content

Commit

Permalink
feat: basic builder API for SDK (#518)
Browse files Browse the repository at this point in the history
  • Loading branch information
morgante authored Sep 23, 2024
1 parent 1470218 commit b3aedb7
Show file tree
Hide file tree
Showing 26 changed files with 896 additions and 357 deletions.
3 changes: 3 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion crates/cli/src/analyze.rs
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ where
if let ApplyInput::Virtual(my_input) = my_input {
compiled.execute_files_streaming(my_input.files, context, tx, cache_ref);
} else {
unreachable!();
panic!("apply_pattern should be called with paths or a virtual input");
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/cli/src/commands/apply_pattern.rs
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,7 @@ pub(crate) async fn run_apply_pattern(
let lang = PatternLanguage::get_language(&body);
(lang, None, body)
} else {
unreachable!()
bail!("pattern should end with .grit or .md");
}
}
Err(_) => {
Expand Down
12 changes: 12 additions & 0 deletions crates/core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,15 @@ getrandom = { version = "0.2.11", optional = true }
lazy_static = { version = "1.4.0", optional = true }
walkdir = { version = "2.3.3", optional = true }
fs-err = { version = "2.11.0" }
wasm-bindgen = { version = "0.2.89", features = [
"serde-serialize",
], optional = true }
napi = { version = "2.16.4", default-features = false, features = [
"napi8",
"async",
], optional = true }
napi-derive = { version = "2.12.2", optional = true }


[dev-dependencies]
similar = "2.2.1"
Expand Down Expand Up @@ -83,10 +92,13 @@ wasm_core = [
"external_functions_common",
"external_functions_ffi",
"marzano-util/external_functions_ffi",
"dep:wasm-bindgen",
]
grit_tracing = ["dep:tracing-opentelemetry"]
language-parsers = ["marzano-language/builtin-parser"]
grit-parser = ["marzano-language/grit-parser"]
absolute_filename = []
non_wasm = ["absolute_filename"]
test_utils = []

napi = ["dep:napi", "dep:napi-derive"]
11 changes: 10 additions & 1 deletion crates/core/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
#![deny(clippy::wildcard_enum_match_arm)]

#[cfg(feature = "napi")]
#[macro_use]
extern crate napi_derive;

pub mod analysis;
pub mod api;
pub mod ast_node;
Expand All @@ -10,9 +15,10 @@ mod equivalence;
mod foreign_function_definition;
pub mod fs;
mod inline_snippets;

#[cfg(any(feature = "napi", feature = "wasm_core"))]
pub mod sdk;

mod lazy;
mod limits;
pub mod marzano_binding;
pub mod marzano_code_snippet;
Expand All @@ -30,6 +36,9 @@ mod text_unparser;
pub mod tree_sitter_serde;
mod variables;

#[cfg(any(feature = "napi", feature = "wasm_core"))]
pub use sdk::UncompiledPatternBuilder;

// getrandom is a deeply nested dependency used by many things eg. uuid
// to get wasm working we needed to enable a feature for this crate, so
// while we don't have a direct usage of it, we had to add it as a dependency
Expand Down
2 changes: 1 addition & 1 deletion crates/core/src/marzano_context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ impl<'a> ExecContext<'a, MarzanoQueryContext> for MarzanoContext<'a> {
let the_new_files = state.bindings[GLOBAL_VARS_SCOPE_INDEX.into()]
.back_mut()
.unwrap()[NEW_FILES_INDEX]
.as_mut();
.as_mut();
the_new_files.value = Some(ResolvedPattern::from_list_parts([].into_iter()));
Ok(true)
}
Expand Down
12 changes: 9 additions & 3 deletions crates/core/src/marzano_resolved_pattern.rs
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,9 @@ impl<'a> ResolvedPattern<'a, MarzanoQueryContext> for MarzanoResolvedPattern<'a>
parts.push(ResolvedSnippet::Text(string.into()));
}
DynamicSnippetPart::Variable(var) => {
let content = &state.bindings[var.try_scope().unwrap().into()].last().unwrap()[var.try_index().unwrap().into()];
let content = &state.bindings[var.try_scope().unwrap().into()]
.last()
.unwrap()[var.try_index().unwrap().into()];
let name = &content.name;
// feels weird not sure if clone is correct
let value = if let Some(value) = &content.value {
Expand Down Expand Up @@ -424,7 +426,9 @@ impl<'a> ResolvedPattern<'a, MarzanoQueryContext> for MarzanoResolvedPattern<'a>
) -> GritResult<Self> {
match pattern {
DynamicPattern::Variable(var) => {
let content = &state.bindings[var.try_scope().unwrap().into()].last().unwrap()[var.try_index().unwrap().into()];
let content = &state.bindings[var.try_scope().unwrap().into()]
.last()
.unwrap()[var.try_index().unwrap().into()];
let name = &content.name;
// feels weird not sure if clone is correct
if let Some(value) = &content.value {
Expand Down Expand Up @@ -516,7 +520,9 @@ impl<'a> ResolvedPattern<'a, MarzanoQueryContext> for MarzanoResolvedPattern<'a>
Pattern::FloatConstant(double) => Ok(Self::Constant(Constant::Float(double.value))),
Pattern::BooleanConstant(bool) => Ok(Self::Constant(Constant::Boolean(bool.value))),
Pattern::Variable(var) => {
let content = &state.bindings[var.try_scope().unwrap().into()].last().unwrap()[var.try_index().unwrap().into()];
let content = &state.bindings[var.try_scope().unwrap().into()]
.last()
.unwrap()[var.try_index().unwrap().into()];
let name = &content.name;
// feels weird not sure if clone is correct
if let Some(value) = &content.value {
Expand Down
2 changes: 1 addition & 1 deletion crates/core/src/pattern_compiler/auto_wrap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use crate::{optimizer::hoist_files::extract_filename_pattern, problem::MarzanoQu
use super::compiler::{DefinitionInfo, SnippetCompilationContext};
use anyhow::Result;
use grit_pattern_matcher::{
constants::{GRIT_RANGE_VAR},
constants::GRIT_RANGE_VAR,
pattern::{
And, Bubble, Call, Container, Contains, FilePattern, Includes, Limit, Match, Maybe,
Pattern, PatternDefinition, PrAnd, PrOr, Predicate, Range as PRange, Rewrite, Step,
Expand Down
9 changes: 0 additions & 9 deletions crates/core/src/pattern_compiler/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -714,15 +714,6 @@ impl VariableLocations {
Self { locations }
}

pub(crate) fn from_globals(globals: BTreeMap<String, usize>) -> Self {
Self {
locations: vec![globals
.into_keys()
.map(VariableSource::new_global)
.collect()],
}
}

pub(crate) fn initial_bindings(
&self,
) -> Vector<Vector<Vector<Box<VariableContent<MarzanoQueryContext>>>>> {
Expand Down
4 changes: 1 addition & 3 deletions crates/core/src/pattern_compiler/pattern_compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,8 @@ use super::{
where_compiler::WhereCompiler,
within_compiler::WithinCompiler,
};
use crate::ast_node::{ASTNode, AstLeafNode};
use crate::problem::MarzanoQueryContext;
use crate::{
ast_node::{ASTNode, AstLeafNode},
};
use anyhow::{anyhow, bail, Result};
use grit_pattern_matcher::{
context::QueryContext,
Expand Down
8 changes: 2 additions & 6 deletions crates/core/src/pattern_compiler/snippet_compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,10 @@ use super::{
pattern_compiler::PatternCompiler,
NodeCompiler,
};
use crate::{
marzano_code_snippet::MarzanoCodeSnippet, problem::MarzanoQueryContext,
};
use crate::{marzano_code_snippet::MarzanoCodeSnippet, problem::MarzanoQueryContext};
use crate::{pattern_compiler::compiler::NodeCompilationContext, split_snippet::split_snippet};
use anyhow::{anyhow, bail, Result};
use grit_pattern_matcher::{
pattern::{DynamicPattern, DynamicSnippet, DynamicSnippetPart, Pattern},
};
use grit_pattern_matcher::pattern::{DynamicPattern, DynamicSnippet, DynamicSnippetPart, Pattern};
use grit_util::{AstNode, ByteRange, Language};
use marzano_language::{
language::{nodes_from_indices, MarzanoLanguage, SortId},
Expand Down
160 changes: 160 additions & 0 deletions crates/core/src/sdk/binding.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
use grit_pattern_matcher::context::ExecContext;
use grit_pattern_matcher::effects::insert_effect;
use grit_pattern_matcher::pattern::PatternOrResolved;
use grit_pattern_matcher::pattern::PatternOrResolvedMut;
use grit_pattern_matcher::pattern::ResolvedPattern;
use grit_pattern_matcher::pattern::State;

use grit_pattern_matcher::pattern::Variable;
use napi::{Env, Result};

use crate::marzano_context::MarzanoContext;
use crate::marzano_resolved_pattern::MarzanoResolvedPattern;
use crate::problem::MarzanoQueryContext;

/// Resolved bindings provide hooks for JavaScript code to interact with a bit of code that has been matched by a pattern.
/// The binding corresponds to a specific piece of text, which can be further filtered against or modified inside a JavaScript callback.
#[napi]
pub struct ResultBinding {
inner: &'static MarzanoResolvedPattern<'static>,
context: &'static MarzanoContext<'static>,
state: &'static mut State<'static, MarzanoQueryContext>,
}

/// Internal implementation details, which we do not expose to host runtimes.
impl ResultBinding {
pub fn new(
inner: &'static MarzanoResolvedPattern<'static>,
context: &'static MarzanoContext<'static>,
state: &'static mut State<'static, MarzanoQueryContext>,
) -> Self {
Self {
inner,
context,
state,
}
}

/// Create a new binding from pointer references
///
/// SAFETY: This operation is inherently unsafe, as we cannot guarantee that the references live for the lifetime of the foreign object.
/// If you call this, you are responsible for warning users that the underlying data cannot be accessed outside the original lifetimes.
pub fn new_unsafe(
binding: &MarzanoResolvedPattern,
context: &MarzanoContext,
state: &mut State<MarzanoQueryContext>,
) -> Self {
let inner_context: &'static MarzanoContext = unsafe { std::mem::transmute(context) };
let inner_state: &'static mut State<MarzanoQueryContext> =
unsafe { std::mem::transmute(state) };

let inner_binding: &'static MarzanoResolvedPattern =
unsafe { std::mem::transmute(binding) };

let js_binding = ResultBinding {
inner: inner_binding,
context: inner_context,
state: inner_state,
};

js_binding
}
}

/// Implement the core binding methods, that should be called by host methods.
///
/// For example:
/// ```javascript
/// function callback(binding, context) {
/// console.log(binding.text());
/// }
/// ```
#[napi]
impl ResultBinding {
/// Retrieves the stringified representation of the binding (ie. the actual source code)
#[napi]
pub fn text(&self, _env: Env) -> Result<String> {
let stringified_result = self
.inner
.text(&self.state.files, self.context.language())
.map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?;
let string = stringified_result.to_string();
Ok(string)
}

/// If the binding was found in a source file, return the position of the binding.
#[napi]
pub fn range(&self, _env: Env) -> Result<Option<grit_util::RangeWithoutByte>> {
let range = self.inner.position(self.context.language());
Ok(range.map(|e| e.into()))
}

/// Retrieves the absolute file name of the file containing the current binding.
#[napi]
pub fn filename(&self, _env: Env) -> Result<String> {
let var = Variable::file_name();
let resolved = var
.get_pattern_or_resolved(self.state)
.map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?;
let Some(candidate) = resolved else {
return Err(napi::Error::from_reason("No resolved pattern found"));
};
let PatternOrResolved::Resolved(resolved) = candidate else {
return Err(napi::Error::from_reason("No resolved pattern found"));
};
let stringified_result = resolved
.text(&self.state.files, self.context.language())
.map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?;
let string = stringified_result.to_string();
Ok(string)
}

/// Inserts the provided text after the binding.
/// The GritQL engine handles the insertion of the text in transformed output code.
#[napi]
pub fn append(&mut self, _env: Env, text: String) -> Result<()> {
let left = PatternOrResolved::Resolved::<MarzanoQueryContext>(self.inner);
let replacement = ResolvedPattern::from_string(text);

insert_effect(&left, replacement, self.state, self.context)
.map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?;

Ok(())
}
}

/// Implement the core binding methods, that are technically attached to the binding but really belong to the surrounding scope.
///
/// For example:
/// ```javascript
/// function callback(binding, context) {
/// const var = context.var("$foo");
/// console.log(var.text());
/// }
#[napi]
impl ResultBinding {
/// Retrieve a variable's text from the current scope.
/// @hidden This API is not stable yet.
#[napi]
pub fn find_var_text(&self, _env: Env, name: String) -> Result<Option<String>> {
let var = self.state.find_var(&name);
let Some(var) = var else {
return Ok(None);
};
let resolved = var
.get_pattern_or_resolved(self.state)
.map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?;
let Some(candidate) = resolved else {
return Ok(None);
};
let PatternOrResolved::Resolved(resolved) = candidate else {
return Err(napi::Error::from_reason("No resolved pattern found"));
};
let stringified_result = resolved
.text(&self.state.files, self.context.language())
.map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?;
let string = stringified_result.to_string();

Ok(Some(string))
}
}
12 changes: 7 additions & 5 deletions crates/core/src/sdk/compiler.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
use anyhow::{bail, Result};
use grit_pattern_matcher::{
pattern::{DynamicSnippetPart, Pattern, PatternDefinition, Variable},
};
use grit_pattern_matcher::pattern::{DynamicSnippetPart, Pattern, PatternDefinition, Variable};
use grit_util::ByteRange;
use marzano_language::target_language::TargetLanguage;

use crate::{
built_in_functions::BuiltIns,
pattern_compiler::{
compiler::{DefinitionInfo, SnippetCompilationContext},
snippet_compiler::parse_snippet_content,
Expand All @@ -15,14 +14,17 @@ use crate::{

/// As opposed to our standard StatelessCompiler,
/// the StatelessCompiler can handle snippets without needing to maintain scopes
#[derive(Clone, Copy)]
pub struct StatelessCompilerContext {
lang: TargetLanguage,
pub built_ins: BuiltIns,
}

impl StatelessCompilerContext {
pub fn new(lang: TargetLanguage) -> Self {
Self { lang }
Self {
lang,
built_ins: BuiltIns::get_built_in_functions(),
}
}

/// Parse a snippet of code and returns a pattern
Expand Down
Loading

0 comments on commit b3aedb7

Please sign in to comment.