From dd24e08dd963f07f56f8a0d0acb485ff135905f0 Mon Sep 17 00:00:00 2001 From: clem Date: Fri, 18 Feb 2022 17:40:59 +0100 Subject: [PATCH 01/19] implemented new proxy id system --- .src/array.inl | 6 +- .src/array_iterator.inl | 7 +- .src/module.cpp | 2 +- .src/proxy.cpp | 171 +++++++++++----------------------------- .src/proxy.inl | 25 +++++- .src/state.cpp | 2 +- .src/symbol.cpp | 2 +- .src/type.cpp | 2 +- .test/main.cpp | 11 +++ include/array.hpp | 4 +- include/jluna.jl | 13 ++- include/module.hpp | 2 +- include/proxy.hpp | 82 +++++++++---------- include/symbol.hpp | 2 +- include/type.hpp | 2 +- 15 files changed, 146 insertions(+), 187 deletions(-) diff --git a/.src/array.inl b/.src/array.inl index 798835a..8cd0be8 100644 --- a/.src/array.inl +++ b/.src/array.inl @@ -17,14 +17,14 @@ namespace jluna } template - Array::Array(jl_value_t* value, std::shared_ptr& owner, jl_sym_t* symbol) + Array::Array(Any* value, std::shared_ptr& owner, Any* symbol) : Proxy(value, owner, symbol) { jl_assert_type(value, "Array"); } template - Array::Array(jl_value_t* value, jl_sym_t* symbol) + Array::Array(Any* value, jl_sym_t* symbol) : Proxy(value, symbol) { jl_assert_type(value, "Array"); @@ -287,7 +287,7 @@ namespace jluna {} template - Vector::Vector(jl_value_t* value, std::shared_ptr& owner, jl_sym_t* symbol) + Vector::Vector(jl_value_t* value, std::shared_ptr& owner, Any* symbol) : Array(value, owner, symbol) { jl_assert_type(value, "Vector"); diff --git a/.src/array_iterator.inl b/.src/array_iterator.inl index bdc6c99..7022477 100644 --- a/.src/array_iterator.inl +++ b/.src/array_iterator.inl @@ -67,11 +67,14 @@ namespace jluna template Array::ConstIterator::operator Proxy() { - return Proxy( + jl_gc_pause; + auto res = Proxy( jl_arrayref((jl_array_t*) _owner->_content->value(), _index), _owner->_content, - jl_symbol(("[" + std::to_string(_index+1) + "]").c_str()) + jl_box_uint64(_index+1) ); + jl_gc_unpause; + return res; } template diff --git a/.src/module.cpp b/.src/module.cpp index b77341b..f16beed 100644 --- a/.src/module.cpp +++ b/.src/module.cpp @@ -14,7 +14,7 @@ namespace jluna jl_assert_type((Any*) value, "Module"); } - Module::Module(jl_value_t* value, std::shared_ptr& owner, jl_sym_t* name) + Module::Module(jl_value_t* value, std::shared_ptr& owner, Any* name) : Proxy(value, owner, name) { jl_assert_type(value, "Module"); diff --git a/.src/proxy.cpp b/.src/proxy.cpp index be893f2..8578356 100644 --- a/.src/proxy.cpp +++ b/.src/proxy.cpp @@ -14,98 +14,78 @@ namespace jluna { - Proxy::ProxyValue::ProxyValue(Any* value, std::shared_ptr& owner, jl_sym_t* symbol) - : _is_mutating(symbol != nullptr) + // no owner + Proxy::ProxyValue::ProxyValue(Any* value, jl_sym_t* id) + : _is_mutating(id != nullptr) { if (value == nullptr) return; + static jl_function_t* make_unnamed_proxy_id = jl_find_function("jluna.memory_handler", "make_unnamed_proxy_id"); + static jl_function_t* make_named_proxy_id = jl_find_function("jluna.memory_handler", "make_named_proxy_id"); + jl_gc_pause; - _owner = owner; + _value_key = State::detail::create_reference(value); _value_ref = State::detail::get_reference(_value_key); - if (symbol == nullptr) - { - std::stringstream str; - str << State::detail::_id_marker << _value_key << "[]"; - symbol = jl_symbol(str.str().c_str()); - } + if (id == nullptr) + _id_key = State::detail::create_reference(jl_call1(make_unnamed_proxy_id, jl_box_uint64(_value_key))); + else + _id_key = State::detail::create_reference(jl_call2(make_named_proxy_id, (Any*) id, jl_nothing)); + + _id_ref = State::detail::get_reference(_id_key); - _symbol_key = State::detail::create_reference((Any*) symbol); - _symbol_ref = State::detail::get_reference(_symbol_key); jl_gc_unpause; } - Proxy::ProxyValue::ProxyValue(Any* value, jl_sym_t* symbol) - : _owner(nullptr), _is_mutating(symbol != nullptr) + // with owner + Proxy::ProxyValue::ProxyValue(Any* value, std::shared_ptr& owner, Any* id) + : _owner(nullptr), _is_mutating(id != nullptr) { if (value == nullptr) return; + static jl_function_t* make_unnamed_proxy_id = jl_find_function("jluna.memory_handler", "make_unnamed_proxy_id"); + static jl_function_t* make_named_proxy_id = jl_find_function("jluna.memory_handler", "make_named_proxy_id"); + jl_gc_pause; + _value_key = State::detail::create_reference(value); _value_ref = State::detail::get_reference(_value_key); - if (symbol == nullptr) - { - std::stringstream str; - str << State::detail::_id_marker << _value_key; - symbol = jl_symbol(str.str().c_str()); - } + if (id == nullptr) + _id_key = State::detail::create_reference(jl_call1(make_unnamed_proxy_id, jl_box_uint64(_value_key))); + else + _id_key = State::detail::create_reference(jl_call2(make_named_proxy_id, id, owner->id())); + + _id_ref = State::detail::get_reference(_id_key); - _symbol_key = State::detail::create_reference((Any*) symbol); - _symbol_ref = State::detail::get_reference(_symbol_key); jl_gc_unpause; } Proxy::ProxyValue::~ProxyValue() { State::detail::free_reference(_value_key); - State::detail::free_reference(_symbol_key); - } - - Any * Proxy::ProxyValue::value() - { - return jl_ref_value(_value_ref); - } - - Any * Proxy::ProxyValue::symbol() - { - return jl_ref_value(_symbol_ref); - } - - size_t Proxy::ProxyValue::value_key() - { - return _value_key; + State::detail::free_reference(_id_key); } - size_t Proxy::ProxyValue::symbol_key() - { - return _symbol_key; - } - - const Any * Proxy::ProxyValue::value() const + Any * Proxy::ProxyValue::value() const { return jl_ref_value(_value_ref); } - const Any * Proxy::ProxyValue::symbol() const + Any * Proxy::ProxyValue::id() const { - return jl_ref_value(_symbol_ref); + return jl_ref_value(_id_ref); } Any * Proxy::ProxyValue::get_field(jl_sym_t* symbol) { - static jl_module_t* jluna_module = (jl_module_t*) jl_eval_string("return Main.jluna"); - static jl_module_t* exception_module = (jl_module_t*) jl_eval_string("return Main.jluna.exception_handler"); - static jl_function_t* safe_call = jl_get_function(exception_module, "safe_call"); - static jl_function_t* dot = jl_get_function(jluna_module, "dot"); + static jl_function_t* dot = jl_find_function("jluna", "dot"); jl_gc_pause; - Any* args[3] = {(Any*) dot, value(), (Any*) symbol}; - auto* res = jl_call(safe_call, args, 3); - forward_last_exception(); + auto* res = jluna::safe_call(dot, value(), (Any*) symbol); jl_gc_unpause; return res; @@ -117,7 +97,7 @@ namespace jluna : Proxy(jl_nothing, nullptr) {} - Proxy::Proxy(Any* value, std::shared_ptr& owner, jl_sym_t* symbol) + Proxy::Proxy(Any* value, std::shared_ptr& owner, Any* symbol) : _content(new ProxyValue(value, owner, symbol)) {} @@ -133,7 +113,11 @@ namespace jluna Proxy Proxy::operator[](const std::string& field) { jl_sym_t* symbol = jl_symbol(field.c_str()); - return Proxy(_content.get()->get_field(symbol), _content, _content->symbol() == nullptr ? nullptr : symbol); + return Proxy( + _content.get()->get_field(symbol), + _content, + (Any*) (_content->id() == nullptr ? nullptr : symbol) + ); } Proxy Proxy::operator[](size_t i) @@ -142,7 +126,7 @@ namespace jluna return Proxy( jluna::safe_call(getindex, _content->value(), box(i + 1)), _content, - jl_symbol(("[" + std::to_string(i + 1) + "]").c_str()) + jl_box_uint64(i+1) ); } @@ -170,46 +154,9 @@ namespace jluna return std::string(jl_string_data(jl_call1(to_string, _content->value()))); } - std::deque Proxy::assemble_name() const - { - jl_gc_pause; - const ProxyValue* ptr = _content.get(); - std::deque name; - - while (ptr != nullptr and ptr->symbol() != nullptr) - { - name.push_front((jl_sym_t*) ptr->symbol()); - ptr = ptr->_owner.get(); - } - jl_gc_unpause; - - return name; - } - std::string Proxy::get_name() const { - std::deque name = assemble_name(); - std::stringstream str; - - jl_gc_pause; - for (size_t i = 0; i < name.size(); ++i) - { - std::string sname = jl_symbol_name(name.at(i)); - - if (i != 0 and sname.at(0) != '[') - str << "."; - - if (sname.at(0) == State::detail::_id_marker) - if (sname.at(1) == '1' and sname.size() == 2) - str << "Main"; - else - str << ""; - else - str << jl_symbol_name(name.at(i)); - } - jl_gc_unpause; - - return str.str(); + return "TODO"; } std::vector Proxy::get_field_names() const @@ -236,27 +183,15 @@ namespace jluna Proxy & Proxy::operator=(Any* new_value) { - static jl_function_t* safe_call = jl_get_function((jl_module_t*) jl_eval_string("return Main.jluna.exception_handler"), "safe_call"); - static jl_function_t* set_reference = jl_get_function((jl_module_t*) jl_eval_string("return Main.jluna.memory_handler"), "set_reference"); + static jl_function_t* assign = jl_find_function("jluna.memory_handler", "assign"); + static jl_function_t* set_reference = jl_find_function("jluna.memory_handler", "set_reference"); jl_gc_pause; - _content->_value_ref = jl_call3(safe_call, (Any*) set_reference, - jl_box_uint64(_content->value_key()), new_value); - forward_last_exception(); + _content->_value_ref = jluna::safe_call(set_reference, jl_box_uint64(_content->_value_key), new_value); if (_content->_is_mutating) - { - static jl_function_t* assign = jl_find_function("Main.jluna.memory_handler", "assign"); - auto name_q = assemble_name(); - std::vector params = {assign, new_value}; - for (auto* s : name_q) - params.push_back((Any*) s); - - static jl_function_t* safe_call = jl_get_function((jl_module_t*) jl_eval_string("return jluna.exception_handler"), "safe_call"); - jl_call(safe_call, params.data(), params.size()); - forward_last_exception(); - } + jluna::safe_call(assign, new_value, _content->id()); jl_gc_unpause; return *this; @@ -269,22 +204,12 @@ namespace jluna void Proxy::update() { - static jl_function_t* safe_call = jl_get_function((jl_module_t*) jl_eval_string("return Main.jluna.exception_handler"), "safe_call"); - static jl_function_t* assemble_eval = jl_get_function((jl_module_t*) jl_eval_string("return Main.jluna.memory_handler"), "evaluate"); - static jl_function_t* set_reference = jl_get_function((jl_module_t*) jl_eval_string("return Main.jluna.memory_handler"), "set_reference"); + static jl_function_t* evaluate = jl_find_function("jluna.memory_handler", "evaluate"); + static jl_function_t* set_reference = jl_find_function("jluna.memory_handler", "set_reference"); jl_gc_pause; - auto name = assemble_name(); - std::vector args = {(Any*) assemble_eval}; - - for (auto* n : name) - args.push_back((Any*) n); - - Any* new_value = jl_call(safe_call, args.data(), args.size()); - forward_last_exception(); - - _content->_value_ref = jl_call3(safe_call, (Any*) set_reference, jl_box_uint64(_content->value_key()), new_value); - forward_last_exception(); + auto* new_value = jluna::safe_call(evaluate, _content->id()); + _content->_value_ref = jluna::safe_call(set_reference, jl_box_uint64(_content->_value_key), new_value); jl_gc_unpause; } diff --git a/.src/proxy.inl b/.src/proxy.inl index fd3c288..1a43fd0 100644 --- a/.src/proxy.inl +++ b/.src/proxy.inl @@ -5,6 +5,27 @@ namespace jluna { + /// @brief unbox to proxy + template T> + inline T unbox(Any* value) + { + return Proxy(value, nullptr); + } + + /// @brief box jluna::Proxy to Base.Any + template T> + inline Any* box(T value) + { + return value.operator Any*(); + } + + /// @brief type deduction + template<> + struct detail::to_julia_type_aux + { + static inline const std::string type_name = "Any"; + }; + template T Proxy::operator[](size_t i) { @@ -21,13 +42,13 @@ namespace jluna template, bool>> Proxy::operator T() { - return T(_content->value(), _content->_owner, (jl_sym_t*) _content->symbol()); + return T(_content->value(), _content->_owner, _content->id()); } template, bool>> T Proxy::as() { - return T(_content->value(), _content->_owner, (jl_sym_t*) _content->symbol()); + return T(_content->value(), _content->_owner, _content->id()); } template diff --git a/.src/state.cpp b/.src/state.cpp index 1e776ac..1f1f86b 100644 --- a/.src/state.cpp +++ b/.src/state.cpp @@ -267,7 +267,7 @@ namespace jluna::State::detail void initialize_modules() { - Main = Proxy((Any*) jl_main_module, nullptr).as(); + Main = Module(jl_main_module); Core = Module(jl_core_module); Base = Module(jl_base_module); } diff --git a/.src/symbol.cpp b/.src/symbol.cpp index 7701bc6..a14cfd4 100644 --- a/.src/symbol.cpp +++ b/.src/symbol.cpp @@ -21,7 +21,7 @@ namespace jluna jl_assert_type((Any*) value, "Symbol"); } - Symbol::Symbol(jl_value_t* value, std::shared_ptr& owner, jl_sym_t* symbol) + Symbol::Symbol(jl_value_t* value, std::shared_ptr& owner, Any* symbol) : Proxy(value, owner, symbol) { jl_assert_type(value, "Symbol"); diff --git a/.src/type.cpp b/.src/type.cpp index 75a772f..9dd4326 100644 --- a/.src/type.cpp +++ b/.src/type.cpp @@ -14,7 +14,7 @@ namespace jluna : Proxy((jl_value_t*) value, value->name->name) {} - Type::Type(Any* value, std::shared_ptr& owner, jl_sym_t* symbol) + Type::Type(Any* value, std::shared_ptr& owner, Any* symbol) : Proxy(value, owner, symbol) {} diff --git a/.test/main.cpp b/.test/main.cpp index 6acc95d..44b6dff 100644 --- a/.test/main.cpp +++ b/.test/main.cpp @@ -12,6 +12,17 @@ using namespace jluna::detail; int main() { State::initialize(); + + jl_eval_string(R"( + module M + test = [1, 2, 3, 4] + end + )"); + + auto proxy = Main["M"]; + std::cout << proxy.operator std::string() << std::endl; + + return 0; Test::initialize(); Test::test("catch c exception", [](){ diff --git a/include/array.hpp b/include/array.hpp index aaf9864..125a218 100644 --- a/include/array.hpp +++ b/include/array.hpp @@ -32,7 +32,7 @@ namespace jluna /// @param value /// @param owner /// @param symbol - Array(Any* value, std::shared_ptr&, jl_sym_t*); + Array(Any* value, std::shared_ptr&, Any*); /// @brief ctor unowned proxy /// @param value @@ -240,7 +240,7 @@ namespace jluna /// @param value /// @param owner /// @param symbol - Vector(Any* value, std::shared_ptr&, jl_sym_t*); + Vector(Any* value, std::shared_ptr&, Any*); /// @brief ctor /// @param value diff --git a/include/jluna.jl b/include/jluna.jl index 75fbec2..3d6d52f 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -625,19 +625,26 @@ module jluna const _ref_id_marker = '#' const _refs_expression = Meta.parse("jluna.memory_handler._refs[]") + # proxy id that is actually an expression, the ID of topmodule Main is ProxyID = Union{Expr, Nothing} # make as unnamed make_unnamed_proxy_id(id::UInt64) = return Expr(:ref, _refs_expression, id) # make as named with owner and symbol name - make_named_proxy_id(id::Symbol, owner_id::ProxyID) = return Expr(Symbol("."), owner_id, QuoteNode(id)) + make_named_proxy_id(id::Symbol, owner_id::ProxyID) ::ProxyID = return Expr(:(.), owner_id, QuoteNode(id)) # make as named with main as owner and symbol name - make_named_proxy_id(id::Symbol, owner_id::ProxyID) = return Expr(Symbol("."), :Main, QuoteNode(id)) + make_named_proxy_id(id::Symbol, owner_id::Nothing) ::ProxyID = return Expr(:(.), :Main, QuoteNode(id)) # make as named with owner and array index name - make_named_proxy_id(id::Number, owner_id::ProxyID) = return Expr(:ref, owner_id, id) + make_named_proxy_id(id::Number, owner_id::ProxyID) ::ProxyID = return Expr(:ref, owner_id, id) + + # assign to proxy id + assign(new_value::T, name::ProxyID) where {T} = return Main.eval(Expr(:(=), name, new_value)) + + # eval proxy id + evaluate(name::ProxyID) ::Any = return Main.eval(name) """ `print_refs() -> Nothing` diff --git a/include/module.hpp b/include/module.hpp index f7cafaf..d46eee5 100644 --- a/include/module.hpp +++ b/include/module.hpp @@ -26,7 +26,7 @@ namespace jluna /// @param value /// @param owner: internal proxy value owner /// @param name: symbol - Module(jl_value_t* value, std::shared_ptr& owner, jl_sym_t* name); + Module(jl_value_t* value, std::shared_ptr& owner, Any* name); /// @brief decay to C-type explicit operator jl_module_t*(); diff --git a/include/proxy.hpp b/include/proxy.hpp index 86c6ab3..17ba3a2 100644 --- a/include/proxy.hpp +++ b/include/proxy.hpp @@ -37,7 +37,7 @@ namespace jluna /// @param value /// @param owner: shared pointer to owner /// @param symbol - Proxy(Any* value, std::shared_ptr& owner, jl_sym_t* symbol); + Proxy(Any* value, std::shared_ptr& owner, Any* name_or_index); /// @brief dtor ~Proxy(); @@ -143,61 +143,53 @@ namespace jluna bool isa(const Type& type); protected: - class ProxyValue - { - friend class Proxy; - - public: - ProxyValue(Any*, jl_sym_t*); - ProxyValue(Any*, std::shared_ptr& owner, jl_sym_t*); - ~ProxyValue(); - - Any* get_field(jl_sym_t*); + std::shared_ptr _content; + }; - std::shared_ptr _owner; + /// @brief internal proxy value, not intended to be interfaced with by end-users + class Proxy::ProxyValue + { + friend class Proxy; - Any* value(); - Any* symbol(); + public: + /// @brief dtor + ~ProxyValue(); - size_t value_key(); - size_t symbol_key(); + /// @brief get value + /// @returns pointer to value + Any* value() const; - const Any* value() const; - const Any* symbol() const; + /// @brief get id + /// @returns pointer to jluna.memory_handler.ProxyID + Any* id() const; - const bool _is_mutating = true; + protected: + /// @brief ctor without owner + /// @param value: pointer to value + /// @param id: jluna.memory_handler.ProxyID object + ProxyValue(Any* value, jl_sym_t* id); - private: - size_t _symbol_key; - size_t _value_key; + /// @brief ctor with owner and proper name + /// @param value: pointer to value + /// @param id: jluna.memory_handler.ProxyID object + ProxyValue(Any* value, std::shared_ptr& owner, Any* symbol_or_index); - Any* _symbol_ref; - Any* _value_ref; - }; + /// @brief access field + /// @param symbol: name of field + /// @returns pointer to field data + Any* get_field(jl_sym_t*); - std::shared_ptr _content; - std::deque assemble_name() const; - }; + /// @brief owner + std::shared_ptr _owner; - /// @brief unbox to proxy - template T> - inline T unbox(Any* value) - { - return Proxy(value, nullptr); - } + /// @brief points to julia-side variable + const bool _is_mutating = true; - /// @brief box jluna::Proxy to Base.Any - template T> - inline Any* box(T value) - { - return value.operator Any*(); - } + size_t _id_key; + size_t _value_key; - /// @brief type deduction - template<> - struct detail::to_julia_type_aux - { - static inline const std::string type_name = "Any"; + mutable Any* _id_ref; + mutable Any* _value_ref; }; } diff --git a/include/symbol.hpp b/include/symbol.hpp index c227c7e..ec64f20 100644 --- a/include/symbol.hpp +++ b/include/symbol.hpp @@ -29,7 +29,7 @@ namespace jluna /// @param value /// @param owner: internal proxy value owner /// @param name: symbol - Symbol(jl_value_t* value, std::shared_ptr& owner, jl_sym_t* symbol); + Symbol(jl_value_t* value, std::shared_ptr& owner, Any* symbol); /// @brief decay to C-type operator jl_sym_t*() const; diff --git a/include/type.hpp b/include/type.hpp index a18545e..441cef4 100644 --- a/include/type.hpp +++ b/include/type.hpp @@ -32,7 +32,7 @@ namespace jluna /// @param value /// @param owner /// @param symbol - Type(Any* value, std::shared_ptr& owner, jl_sym_t* symbol); + Type(Any* value, std::shared_ptr& owner, Any* symbol); /// @brief decay to C-type operator jl_datatype_t*(); From 3ca77d196f9cdb59f1e4e060ddbb12401079b70a Mon Sep 17 00:00:00 2001 From: clem Date: Fri, 18 Feb 2022 18:30:56 +0100 Subject: [PATCH 02/19] stashing --- .benchmark/main.cpp | 4 ++-- .src/proxy.cpp | 6 ++++-- .test/main.cpp | 11 ----------- include/jluna.jl | 29 +++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/.benchmark/main.cpp b/.benchmark/main.cpp index f0cab68..9032af8 100644 --- a/.benchmark/main.cpp +++ b/.benchmark/main.cpp @@ -70,7 +70,7 @@ int main() jl_eval_string("module M1; module M2; module M3; end end end"); - Benchmark::run("Old Proxy Eval", count, [](){ + Benchmark::run("New Proxy Eval", count, [](){ auto name = generate_string(8); jl_eval_string(("M1.M2.M3.eval(:(" + name + " = \"" + generate_string(16) + "\"))").c_str()); @@ -78,7 +78,7 @@ int main() volatile auto value = Main["M1"]["M2"]["M3"][name].operator std::string(); }); - Benchmark::run("Old Proxy Assign", count, [](){ + Benchmark::run("New Proxy Assign", count, [](){ auto name = generate_string(8); jl_eval_string(("M1.M2.M3.eval(:(" + name + " = undef))").c_str()); diff --git a/.src/proxy.cpp b/.src/proxy.cpp index 8578356..fd36df4 100644 --- a/.src/proxy.cpp +++ b/.src/proxy.cpp @@ -98,7 +98,7 @@ namespace jluna {} Proxy::Proxy(Any* value, std::shared_ptr& owner, Any* symbol) - : _content(new ProxyValue(value, owner, symbol)) + : _content((owner.get() == nullptr ? new ProxyValue(value, (jl_sym_t*) symbol) : new ProxyValue(value, owner, symbol))) {} Proxy::Proxy(Any* value, jl_sym_t* symbol) @@ -156,7 +156,9 @@ namespace jluna std::string Proxy::get_name() const { - return "TODO"; + static jl_function_t* get_name = jl_find_function("jluna.memory_handler", "get_name"); + + return unbox(jluna::safe_call(get_name, _content->id())); } std::vector Proxy::get_field_names() const diff --git a/.test/main.cpp b/.test/main.cpp index 44b6dff..6acc95d 100644 --- a/.test/main.cpp +++ b/.test/main.cpp @@ -12,17 +12,6 @@ using namespace jluna::detail; int main() { State::initialize(); - - jl_eval_string(R"( - module M - test = [1, 2, 3, 4] - end - )"); - - auto proxy = Main["M"]; - std::cout << proxy.operator std::string() << std::endl; - - return 0; Test::initialize(); Test::test("catch c exception", [](){ diff --git a/include/jluna.jl b/include/jluna.jl index 3d6d52f..37997bd 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -646,6 +646,35 @@ module jluna # eval proxy id evaluate(name::ProxyID) ::Any = return Main.eval(name) + """ + `get_name(::ProxyID) -> String` + + parse name from proxy id + """ + function get_name(id::ProxyID) ::String + + current = id + while current.args[1] isa Expr && length(current.args) >= 2 + + if current.args[2] isa UInt64 + current.args[2] = convert(Int64, current.args[2]) + end + + current = current.args[1] + end + + # modifying Base.string() result instead of parsing the expression is super slow but Proxy::get_name is a debug feature anyway + out = string(id) + if out[1:5] == "Main." + chop(string(id), head = length("Main."), tail = 0) + end + + out = replace(out, "(jluna.memory_handler._refs[])[" => "[unnamed proxy #") + return out; + end + + get_name(::Nothing) ::String = return "Main" + """ `print_refs() -> Nothing` From 004093f34e8a726ab62352097593b86153ebde9e Mon Sep 17 00:00:00 2001 From: clem Date: Fri, 18 Feb 2022 18:53:12 +0100 Subject: [PATCH 03/19] stashing --- .src/array_iterator.inl | 8 +++++--- .test/main.cpp | 13 ++++++++++++- include/jluna.jl | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.src/array_iterator.inl b/.src/array_iterator.inl index 7022477..e2392c1 100644 --- a/.src/array_iterator.inl +++ b/.src/array_iterator.inl @@ -68,11 +68,13 @@ namespace jluna Array::ConstIterator::operator Proxy() { jl_gc_pause; + auto res = Proxy( - jl_arrayref((jl_array_t*) _owner->_content->value(), _index), - _owner->_content, - jl_box_uint64(_index+1) + jl_arrayref((jl_array_t*) _owner->_content->value(), _index), + _owner->_content, + jl_box_uint64(_index+1) ); + jl_gc_unpause; return res; } diff --git a/.test/main.cpp b/.test/main.cpp index 6acc95d..50e4274 100644 --- a/.test/main.cpp +++ b/.test/main.cpp @@ -12,6 +12,18 @@ using namespace jluna::detail; int main() { State::initialize(); + + + + Array arr = State::safe_eval("array = reshape(collect(1:27), 3, 3, 3)"); + std::cout << arr.get_name() << std::endl; + + auto it = arr.at(0, 0, 0); + Proxy as_proxy = it; + + Test::assert_that(as_proxy.get_name() == "Main.array[1]"); + + return 0; Test::initialize(); Test::test("catch c exception", [](){ @@ -675,7 +687,6 @@ int main() } Test::assert_that(thrown); - Test::assert_that(arr.begin().operator Int64() == 1); }); diff --git a/include/jluna.jl b/include/jluna.jl index 37997bd..0086f23 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -629,7 +629,7 @@ module jluna ProxyID = Union{Expr, Nothing} # make as unnamed - make_unnamed_proxy_id(id::UInt64) = return Expr(:ref, _refs_expression, id) + make_unnamed_proxy_id(id::UInt64) = return Expr(:ref, Expr(:ref, _refs_expression, id)) # make as named with owner and symbol name make_named_proxy_id(id::Symbol, owner_id::ProxyID) ::ProxyID = return Expr(:(.), owner_id, QuoteNode(id)) From 1c717f3c65e2533805931c07f6d799bab8707c95 Mon Sep 17 00:00:00 2001 From: clem Date: Sat, 19 Feb 2022 18:35:39 +0100 Subject: [PATCH 04/19] fixed type deduction specificity --- .src/box.inl | 23 ++++++++++++++--------- .src/generator_expression.cpp | 1 - README.md | 15 ++++++++++----- include/jluna.jl | 18 ++++++++++++++++++ 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.src/box.inl b/.src/box.inl index f00cb32..ffaaa03 100644 --- a/.src/box.inl +++ b/.src/box.inl @@ -131,10 +131,14 @@ namespace jluna template>, bool>> Any* box(const T& value) { - static jl_function_t* vector = jl_get_function(jl_base_module, "Vector"); + static jl_function_t* new_vector = jl_find_function("jluna", "new_vector"); jl_gc_pause; - auto* res = (jl_array_t*) jl_call2(vector, jl_undef_initializer(), jl_box_uint64(value.size())); + auto* res = (jl_array_t*) jl_call2( + new_vector, + jl_box_uint64(value.size()), + value.empty() ? jl_eval_string(to_julia_type::type_name.c_str()) : box(value.front()) + ); for (size_t i = 0; i < value.size(); ++i) jl_arrayset(res, box(value.at(i)), i); @@ -189,18 +193,19 @@ namespace jluna template>, bool>> Any* box(const T& value) { - static jl_function_t* vector = jl_get_function(jl_base_module, "Vector"); + static jl_function_t* new_vector = jl_find_function("jluna", "new_vector"); static jl_function_t* set = jl_get_function(jl_base_module, "Set"); jl_gc_pause; - auto* res = (jl_array_t*) jl_call2(vector, jl_undef_initializer(), jl_box_uint64(value.size())); + auto* res = (jl_array_t*) jl_call2( + new_vector, + jl_box_uint64(value.size()), + value.empty() ? jl_eval_string(to_julia_type::type_name.c_str()) : box(*value.begin()) + ); size_t i = 0; - for (const auto& s : value) - { - jl_arrayset(res, box(s), i); - i += 1; - } + for (auto s : value) + jl_arrayset(res, box(s), i++); auto* out = jl_call1(set, (Any*) res); jl_gc_unpause; diff --git a/.src/generator_expression.cpp b/.src/generator_expression.cpp index b549fa2..280372f 100644 --- a/.src/generator_expression.cpp +++ b/.src/generator_expression.cpp @@ -44,7 +44,6 @@ namespace jluna jl_gc_pause; _length = jl_unbox_int64(jl_call1(length, get())); - std::cout << _length << std::endl; forward_last_exception(); jl_gc_unpause; } diff --git a/README.md b/README.md index c7dce93..e12bac1 100644 --- a/README.md +++ b/README.md @@ -60,19 +60,24 @@ instance._field is now: 456 State::script("array = collect(1:9)"); Array cpp_array = Main["array"]; -// julia style list indexing -auto sub_array = cpp_array[{6, 5, 4, 3, 2}]; -Base["println"]((Any*) sub_array); - // iterable and assignable for (auto e : cpp_array) e = e.operator size_t() + 10; State::script("println(array)"); + +// julia style list indexing +auto sub_array = cpp_array[{6, 5, 4, 3, 2}]; +Base["println"]((Any*) sub_array); + +// even supports comprehension +auto comprehended_vec = Vector("(i for i in 1:10 if i % 2 == 0)"_gen); + Base["println"](comprehended_vec); ``` ``` -[7, 6, 5, 4, 3] [11, 12, 13, 14, 15, 16, 17, 18, 19] +[17, 16, 15, 14, 13] +[2, 4, 6, 8, 10] ``` --- #### Call C++ Functions from julia diff --git a/include/jluna.jl b/include/jluna.jl index 0086f23..99584bc 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -135,6 +135,24 @@ module jluna return convert.(T, [xs...]) end + """ + `new_vector(::Integer, ::T) -> Vector{T}` + + create vector by deducing argument type + """ + function new_vector(size::Integer, _::T) where T + return Vector{T}(undef, size) + end + + """ + `new_vector(::Integer, ::T) -> Vector{T}` + + create vector by deducing argument type + """ + function new_vector(size::Integer, type::Type) + return Vector{type}(undef, size) + end + """ `make_set(::T...) -> Set{T}` From 4112e9536c1f8edd016efd068cedb39d47dc0de6 Mon Sep 17 00:00:00 2001 From: clem Date: Sat, 19 Feb 2022 21:29:23 +0100 Subject: [PATCH 05/19] owner proxy id now correctly handed down to child --- .src/array.inl | 25 +++++++------ .src/module.cpp | 8 ++--- .src/proxy.cpp | 15 +++----- .src/proxy.inl | 4 +-- .src/symbol.cpp | 8 ++--- .src/symbol.inl | 3 +- .src/type.cpp | 8 +++-- .src/type.inl | 3 +- .test/main.cpp | 11 +++--- include/array.hpp | 17 ++++----- include/jluna.jl | 72 ++++--------------------------------- include/julia_extension.hpp | 4 +-- include/module.hpp | 11 +++--- include/symbol.hpp | 8 ++--- include/type.hpp | 6 ++-- 15 files changed, 69 insertions(+), 134 deletions(-) diff --git a/.src/array.inl b/.src/array.inl index 8cd0be8..a6b7429 100644 --- a/.src/array.inl +++ b/.src/array.inl @@ -17,17 +17,19 @@ namespace jluna } template - Array::Array(Any* value, std::shared_ptr& owner, Any* symbol) - : Proxy(value, owner, symbol) + Array::Array(Any* value, jl_sym_t* symbol) + : Proxy(value, symbol) { - jl_assert_type(value, "Array"); + static Any* array_t = jl_eval_string("return Base.Array"); + jl_assert_type(value, array_t); } template - Array::Array(Any* value, jl_sym_t* symbol) - : Proxy(value, symbol) + Array::Array(Proxy* proxy) + : Proxy(*proxy) { - jl_assert_type(value, "Array"); + static Any* array_t = jl_eval_string("return Base.Array"); + jl_assert_type(proxy->value(), array_t); } template @@ -287,17 +289,19 @@ namespace jluna {} template - Vector::Vector(jl_value_t* value, std::shared_ptr& owner, Any* symbol) - : Array(value, owner, symbol) + Vector::Vector(Proxy* owner) + : Array(owner) { - jl_assert_type(value, "Vector"); + static Any* vector_t = jl_eval_string("return Vector"); + jl_assert_type(owner->value(), vector_t); } template Vector::Vector(jl_value_t* value, jl_sym_t* symbol) : Array(value, symbol) { - jl_assert_type(value, "Vector"); + static Any* vector_t = jl_eval_string("return Vector"); + jl_assert_type(value, vector_t); } template @@ -308,6 +312,7 @@ namespace jluna vec.reserve(gen.get().size()); for (auto it : gen.get()) vec.push_back(unbox(it)); + return vec; }()) {} diff --git a/.src/module.cpp b/.src/module.cpp index f16beed..e32cba1 100644 --- a/.src/module.cpp +++ b/.src/module.cpp @@ -11,13 +11,13 @@ namespace jluna Module::Module(jl_module_t* value) : Proxy((jl_value_t*) value, value->name) { - jl_assert_type((Any*) value, "Module"); + jl_assert_type((Any*) value, (Any*) jl_module_type); } - Module::Module(jl_value_t* value, std::shared_ptr& owner, Any* name) - : Proxy(value, owner, name) + Module::Module(Proxy* owner) + : Proxy(*owner) { - jl_assert_type(value, "Module"); + jl_assert_type(owner->value(), (Any*) jl_module_type); } jl_module_t * Module::get() const diff --git a/.src/proxy.cpp b/.src/proxy.cpp index fd36df4..a009ad7 100644 --- a/.src/proxy.cpp +++ b/.src/proxy.cpp @@ -41,24 +41,18 @@ namespace jluna // with owner Proxy::ProxyValue::ProxyValue(Any* value, std::shared_ptr& owner, Any* id) - : _owner(nullptr), _is_mutating(id != nullptr) { - if (value == nullptr) - return; + jl_gc_pause; static jl_function_t* make_unnamed_proxy_id = jl_find_function("jluna.memory_handler", "make_unnamed_proxy_id"); static jl_function_t* make_named_proxy_id = jl_find_function("jluna.memory_handler", "make_named_proxy_id"); - jl_gc_pause; + _owner = owner; _value_key = State::detail::create_reference(value); _value_ref = State::detail::get_reference(_value_key); - if (id == nullptr) - _id_key = State::detail::create_reference(jl_call1(make_unnamed_proxy_id, jl_box_uint64(_value_key))); - else - _id_key = State::detail::create_reference(jl_call2(make_named_proxy_id, id, owner->id())); - + _id_key = State::detail::create_reference(jl_call2(make_named_proxy_id, id, owner->id())); _id_ref = State::detail::get_reference(_id_key); jl_gc_unpause; @@ -98,7 +92,7 @@ namespace jluna {} Proxy::Proxy(Any* value, std::shared_ptr& owner, Any* symbol) - : _content((owner.get() == nullptr ? new ProxyValue(value, (jl_sym_t*) symbol) : new ProxyValue(value, owner, symbol))) + : _content(new ProxyValue(value, owner, symbol)) {} Proxy::Proxy(Any* value, jl_sym_t* symbol) @@ -157,7 +151,6 @@ namespace jluna std::string Proxy::get_name() const { static jl_function_t* get_name = jl_find_function("jluna.memory_handler", "get_name"); - return unbox(jluna::safe_call(get_name, _content->id())); } diff --git a/.src/proxy.inl b/.src/proxy.inl index 1a43fd0..b6f0ed3 100644 --- a/.src/proxy.inl +++ b/.src/proxy.inl @@ -42,13 +42,13 @@ namespace jluna template, bool>> Proxy::operator T() { - return T(_content->value(), _content->_owner, _content->id()); + return as(); } template, bool>> T Proxy::as() { - return T(_content->value(), _content->_owner, _content->id()); + return T(this); } template diff --git a/.src/symbol.cpp b/.src/symbol.cpp index a14cfd4..57f271a 100644 --- a/.src/symbol.cpp +++ b/.src/symbol.cpp @@ -18,13 +18,13 @@ namespace jluna Symbol::Symbol(jl_sym_t* value, jl_sym_t* symbol) : Proxy((jl_value_t*) value, symbol) { - jl_assert_type((Any*) value, "Symbol"); + jl_assert_type((Any*) value, (Any*) jl_symbol_type); } - Symbol::Symbol(jl_value_t* value, std::shared_ptr& owner, Any* symbol) - : Proxy(value, owner, symbol) + Symbol::Symbol(Proxy* owner) + : Proxy(*owner) { - jl_assert_type(value, "Symbol"); + jl_assert_type(owner->value(), (Any*) jl_symbol_type); } Symbol::operator jl_sym_t*() const diff --git a/.src/symbol.inl b/.src/symbol.inl index 9a76033..f332e06 100644 --- a/.src/symbol.inl +++ b/.src/symbol.inl @@ -9,7 +9,8 @@ namespace jluna template T> inline T unbox(Any* value) { - jl_assert_type(value, "Symbol"); + static Any* symbol_t = jl_eval_string("return Symbol"); + jl_assert_type(value, symbol_t); return Symbol((jl_sym_t*) value); } diff --git a/.src/type.cpp b/.src/type.cpp index 9dd4326..eff4c59 100644 --- a/.src/type.cpp +++ b/.src/type.cpp @@ -14,9 +14,11 @@ namespace jluna : Proxy((jl_value_t*) value, value->name->name) {} - Type::Type(Any* value, std::shared_ptr& owner, Any* symbol) - : Proxy(value, owner, symbol) - {} + Type::Type(Proxy* owner) + : Proxy(*owner) + { + jl_assert_type(owner->value(), (Any*) jl_type_type); + } Type Type::unroll() const { diff --git a/.src/type.inl b/.src/type.inl index 91c1f00..6f54fdb 100644 --- a/.src/type.inl +++ b/.src/type.inl @@ -9,7 +9,8 @@ namespace jluna template T> inline T unbox(Any* value) { - jl_assert_type(value, "Type"); + static Any* type_t = jl_eval_string("return Type"); + jl_assert_type(value, type_t); return Type((jl_datatype_t*) value); } diff --git a/.test/main.cpp b/.test/main.cpp index 50e4274..afa6e1f 100644 --- a/.test/main.cpp +++ b/.test/main.cpp @@ -13,17 +13,17 @@ int main() { State::initialize(); - - - Array arr = State::safe_eval("array = reshape(collect(1:27), 3, 3, 3)"); - std::cout << arr.get_name() << std::endl; + State::safe_eval("array = reshape(collect(1:27), 3, 3, 3)"); + Array3d arr = Main["array"]; auto it = arr.at(0, 0, 0); Proxy as_proxy = it; + std::cout << as_proxy.get_name() << std::endl; + Test::assert_that(as_proxy.get_name() == "Main.array[1]"); - return 0; + return 0; Test::initialize(); Test::test("catch c exception", [](){ @@ -34,6 +34,7 @@ int main() }); }); + Test::test("jl_find_function", [](){ auto* expected = jl_get_function(jl_base_module, "println"); diff --git a/include/array.hpp b/include/array.hpp index 125a218..37e2164 100644 --- a/include/array.hpp +++ b/include/array.hpp @@ -32,12 +32,11 @@ namespace jluna /// @param value /// @param owner /// @param symbol - Array(Any* value, std::shared_ptr&, Any*); + Array(Proxy*); - /// @brief ctor unowned proxy - /// @param value - /// @param name or nulltpr - Array(Any*, jl_sym_t* = nullptr); + /// @brief ctor from proxy + /// @param proxy + Array(Any* value, jl_sym_t* = nullptr); /// @brief linear indexing, no bounds checking /// @param index, 0-based @@ -236,11 +235,9 @@ namespace jluna /// @param generator_expression Vector(const GeneratorExpression&); - /// @brief ctor - /// @param value - /// @param owner - /// @param symbol - Vector(Any* value, std::shared_ptr&, Any*); + /// @brief ctor from proxy + /// @param proxy + Vector(Proxy*); /// @brief ctor /// @param value diff --git a/include/jluna.jl b/include/jluna.jl index 99584bc..e54f7db 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -197,10 +197,7 @@ module jluna throw assertion if x is not of named type """ - function assert_isa(x::Any, type_name::Symbol) ::Nothing - - type = Main.eval(type_name); - @assert type isa Type + function assert_isa(x::Any, type::Type) ::Nothing if !(x isa type) throw(AssertionError("expected " * string(type) * " but got an object of type " * string(typeof(x)))); @@ -663,6 +660,7 @@ module jluna # eval proxy id evaluate(name::ProxyID) ::Any = return Main.eval(name) + evaluate(name::Symbol) ::Any = return Main.eval(:($name)) """ `get_name(::ProxyID) -> String` @@ -683,8 +681,9 @@ module jluna # modifying Base.string() result instead of parsing the expression is super slow but Proxy::get_name is a debug feature anyway out = string(id) + if out[1:5] == "Main." - chop(string(id), head = length("Main."), tail = 0) + out = chop(string(id), head = length("Main."), tail = 0) end out = replace(out, "(jluna.memory_handler._refs[])[" => "[unnamed proxy #") @@ -692,6 +691,8 @@ module jluna end get_name(::Nothing) ::String = return "Main" + get_name(s::Symbol) ::String = return string(s) + get_name(i::Integer) ::String = return "[" * string(i) * "]" """ `print_refs() -> Nothing` @@ -739,67 +740,6 @@ module jluna return _refs[][key] end - function assign(new_value::T, names::Symbol...) ::T where T - - unnamed_to_index(s::Symbol) = tryparse(UInt64, chop(string(s), head = 1, tail = 0)) - - name = ""; - for n in names - - as_string = string(n); - if as_string[1] == _ref_id_marker - if as_string[2] == '1' && length(as_string) == 2 # main - continue - else - name *= "jluna.memory_handler._refs[][" * chop(string(n), head = 1, tail = 0) * "][]" - end - elseif as_string[1] == '[' - name *= string(n) - else - name *= "." * string(n) - end - end - - if name[1] == '.' - name = chop(name, head = 1, tail = 0) # remove first . - end - - - expr = :($(Meta.parse(name)) = $new_value); - Main.eval(expr); - return new_value; - end - - """ - `evaluate(::Symbol) -> Any` - - evaluate variable value using proxy name-chain - """ - function evaluate(names::Symbol...) ::Any - - name = ""; - - for n in names - - as_string = string(n); - if as_string[1] == _ref_id_marker - name *= "jluna.memory_handler._refs[][" * chop(string(n), head = 1, tail = 0) * "][]" - elseif as_string[1] == '[' - name *= string(n) - else - name *= "." * string(n) - end - end - - if name[1] == '.' - name = chop(name, head = 1, tail = 0) # remove first . - end - - print_refs(); - println(name) - return Main.eval(:($(Meta.parse(name)))) - end - """ `get_reference(::Int64) -> Any` diff --git a/include/julia_extension.hpp b/include/julia_extension.hpp index 7d710c7..5357df9 100644 --- a/include/julia_extension.hpp +++ b/include/julia_extension.hpp @@ -77,10 +77,10 @@ extern "C" /// @brief throw error if value is not of type named /// @param value /// @param types_name - inline void jl_assert_type(jl_value_t* value, const std::string& type_str) + inline void jl_assert_type(jl_value_t* value, jl_value_t* type) { static jl_function_t* assert_isa = jl_find_function("jluna", "assert_isa"); - jluna::safe_call(assert_isa, value, jl_symbol(type_str.c_str())); + jluna::safe_call(assert_isa, value, type); } /// @brief get value of reference diff --git a/include/module.hpp b/include/module.hpp index d46eee5..1bcfc65 100644 --- a/include/module.hpp +++ b/include/module.hpp @@ -22,11 +22,9 @@ namespace jluna /// @param name: symbol Module(jl_module_t*); - /// @brief ctor as owned - /// @param value - /// @param owner: internal proxy value owner - /// @param name: symbol - Module(jl_value_t* value, std::shared_ptr& owner, Any* name); + /// @brief ctor from proxy + /// @param proxy + Module(Proxy*); /// @brief decay to C-type explicit operator jl_module_t*(); @@ -116,7 +114,8 @@ namespace jluna template T> inline T unbox(Any* value) { - jl_assert_type(value, "Module"); + static Any* module_t = jl_eval_string("return Module"); + jl_assert_type(value, module_t); return Module((jl_module_t*) value); } diff --git a/include/symbol.hpp b/include/symbol.hpp index ec64f20..a394ef1 100644 --- a/include/symbol.hpp +++ b/include/symbol.hpp @@ -25,11 +25,9 @@ namespace jluna /// @param name Symbol(jl_sym_t* value, jl_sym_t* symbol = nullptr); - /// @brief ctor as owned - /// @param value - /// @param owner: internal proxy value owner - /// @param name: symbol - Symbol(jl_value_t* value, std::shared_ptr& owner, Any* symbol); + /// @brief ctor from proxy + /// @param proxy + Symbol(Proxy*); /// @brief decay to C-type operator jl_sym_t*() const; diff --git a/include/type.hpp b/include/type.hpp index 441cef4..d5907f9 100644 --- a/include/type.hpp +++ b/include/type.hpp @@ -29,10 +29,8 @@ namespace jluna Type(jl_datatype_t* value); /// @brief ctor - /// @param value - /// @param owner - /// @param symbol - Type(Any* value, std::shared_ptr& owner, Any* symbol); + /// @param proxy + Type(Proxy*); /// @brief decay to C-type operator jl_datatype_t*(); From 57e702f77bb563eecb0e22f6853a5e4f5a4b3cb9 Mon Sep 17 00:00:00 2001 From: clem Date: Sat, 19 Feb 2022 21:50:30 +0100 Subject: [PATCH 06/19] polished get_name --- .test/main.cpp | 11 ++--------- include/jluna.jl | 4 +++- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.test/main.cpp b/.test/main.cpp index afa6e1f..1facd9e 100644 --- a/.test/main.cpp +++ b/.test/main.cpp @@ -13,15 +13,8 @@ int main() { State::initialize(); - State::safe_eval("array = reshape(collect(1:27), 3, 3, 3)"); - Array3d arr = Main["array"]; - - auto it = arr.at(0, 0, 0); - Proxy as_proxy = it; - - std::cout << as_proxy.get_name() << std::endl; - - Test::assert_that(as_proxy.get_name() == "Main.array[1]"); + auto arr = State::safe_eval("array = reshape(collect(1:27), 3, 3, 3)"); + std::cout << arr.get_name() << std::endl; return 0; Test::initialize(); diff --git a/include/jluna.jl b/include/jluna.jl index e54f7db..18cef13 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -686,7 +686,9 @@ module jluna out = chop(string(id), head = length("Main."), tail = 0) end - out = replace(out, "(jluna.memory_handler._refs[])[" => "[unnamed proxy #") + reg = r"\Q((jluna.memory_handler._refs[])[\E(.*)\Q])[]\E" + out = replace(out, reg => "") + return out; end From c444d372cd9fbc279e58713b40c50255da274a65 Mon Sep 17 00:00:00 2001 From: clem Date: Sat, 19 Feb 2022 22:16:38 +0100 Subject: [PATCH 07/19] stashing --- .src/array.inl | 18 +++++++++--------- .src/module.cpp | 4 ++-- .src/proxy.cpp | 6 +++++- .src/symbol.cpp | 4 ++-- .src/symbol.inl | 3 +-- .src/type.cpp | 3 ++- .src/type.inl | 2 +- .test/main.cpp | 14 +++++++++++--- include/julia_extension.hpp | 2 +- include/module.hpp | 3 +-- include/proxy.hpp | 2 +- 11 files changed, 36 insertions(+), 25 deletions(-) diff --git a/.src/array.inl b/.src/array.inl index a6b7429..6d22aac 100644 --- a/.src/array.inl +++ b/.src/array.inl @@ -20,7 +20,7 @@ namespace jluna Array::Array(Any* value, jl_sym_t* symbol) : Proxy(value, symbol) { - static Any* array_t = jl_eval_string("return Base.Array"); + static jl_datatype_t* array_t = (jl_datatype_t*) jl_eval_string("return Base.Array"); jl_assert_type(value, array_t); } @@ -28,8 +28,8 @@ namespace jluna Array::Array(Proxy* proxy) : Proxy(*proxy) { - static Any* array_t = jl_eval_string("return Base.Array"); - jl_assert_type(proxy->value(), array_t); + static jl_datatype_t* array_t = (jl_datatype_t*) jl_eval_string("return Base.Array"); + jl_assert_type(proxy->operator Any*(), array_t); } template @@ -261,19 +261,19 @@ namespace jluna template size_t Array::get_n_elements() const { - return reinterpret_cast(_content->value())->length; + return reinterpret_cast(this->operator const Any*())->length; } //° template bool Array::empty() const { - return reinterpret_cast(_content->value())->length == 0; + return reinterpret_cast(this->operator const Any*())->length == 0; } //° template void* Array::data() { - return reinterpret_cast(value())->data; + return reinterpret_cast(this->operator Any*())->data; } //° // ### @@ -292,15 +292,15 @@ namespace jluna Vector::Vector(Proxy* owner) : Array(owner) { - static Any* vector_t = jl_eval_string("return Vector"); - jl_assert_type(owner->value(), vector_t); + static jl_datatype_t* vector_t = (jl_datatype_t*) jl_eval_string("return Vector"); + jl_assert_type(owner->operator Any*(), vector_t); } template Vector::Vector(jl_value_t* value, jl_sym_t* symbol) : Array(value, symbol) { - static Any* vector_t = jl_eval_string("return Vector"); + static jl_datatype_t* vector_t = (jl_datatype_t*) jl_eval_string("return Vector"); jl_assert_type(value, vector_t); } diff --git a/.src/module.cpp b/.src/module.cpp index e32cba1..e362fcc 100644 --- a/.src/module.cpp +++ b/.src/module.cpp @@ -11,13 +11,13 @@ namespace jluna Module::Module(jl_module_t* value) : Proxy((jl_value_t*) value, value->name) { - jl_assert_type((Any*) value, (Any*) jl_module_type); + jl_assert_type((Any*) value, jl_module_type); } Module::Module(Proxy* owner) : Proxy(*owner) { - jl_assert_type(owner->value(), (Any*) jl_module_type); + jl_assert_type(owner->operator Any*(), jl_module_type); } jl_module_t * Module::get() const diff --git a/.src/proxy.cpp b/.src/proxy.cpp index a009ad7..3fb8419 100644 --- a/.src/proxy.cpp +++ b/.src/proxy.cpp @@ -126,7 +126,9 @@ namespace jluna Proxy::operator Any*() { + jl_gc_pause; auto* res = _content->value(); + jl_gc_unpause; if (res == nullptr) return jl_nothing; else @@ -135,7 +137,9 @@ namespace jluna Proxy::operator const Any*() const { + jl_gc_pause; auto* res = _content->value(); + jl_gc_unpause; if (res == nullptr) return jl_nothing; else @@ -192,7 +196,7 @@ namespace jluna return *this; } - Proxy Proxy::value() const + Proxy Proxy::as_unnamed() const { return Proxy(jl_deepcopy(_content->value()), nullptr); } diff --git a/.src/symbol.cpp b/.src/symbol.cpp index 57f271a..35fb952 100644 --- a/.src/symbol.cpp +++ b/.src/symbol.cpp @@ -18,13 +18,13 @@ namespace jluna Symbol::Symbol(jl_sym_t* value, jl_sym_t* symbol) : Proxy((jl_value_t*) value, symbol) { - jl_assert_type((Any*) value, (Any*) jl_symbol_type); + jl_assert_type((Any*) value, jl_symbol_type); } Symbol::Symbol(Proxy* owner) : Proxy(*owner) { - jl_assert_type(owner->value(), (Any*) jl_symbol_type); + jl_assert_type(owner->operator Any*(), jl_symbol_type); } Symbol::operator jl_sym_t*() const diff --git a/.src/symbol.inl b/.src/symbol.inl index f332e06..d9bc701 100644 --- a/.src/symbol.inl +++ b/.src/symbol.inl @@ -9,8 +9,7 @@ namespace jluna template T> inline T unbox(Any* value) { - static Any* symbol_t = jl_eval_string("return Symbol"); - jl_assert_type(value, symbol_t); + jl_assert_type(value, jl_symbol_type); return Symbol((jl_sym_t*) value); } diff --git a/.src/type.cpp b/.src/type.cpp index eff4c59..2f46274 100644 --- a/.src/type.cpp +++ b/.src/type.cpp @@ -17,7 +17,8 @@ namespace jluna Type::Type(Proxy* owner) : Proxy(*owner) { - jl_assert_type(owner->value(), (Any*) jl_type_type); + static jl_datatype_t* type_t = (jl_datatype_t*) jl_eval_string("return Type"); + jl_assert_type(owner->operator Any*(), type_t); } Type Type::unroll() const diff --git a/.src/type.inl b/.src/type.inl index 6f54fdb..8ea4cf4 100644 --- a/.src/type.inl +++ b/.src/type.inl @@ -9,7 +9,7 @@ namespace jluna template T> inline T unbox(Any* value) { - static Any* type_t = jl_eval_string("return Type"); + static jl_datatype_t* type_t = (jl_datatype_t*) jl_eval_string("return Type"); jl_assert_type(value, type_t); return Type((jl_datatype_t*) value); } diff --git a/.test/main.cpp b/.test/main.cpp index 1facd9e..fec9d13 100644 --- a/.test/main.cpp +++ b/.test/main.cpp @@ -13,10 +13,18 @@ int main() { State::initialize(); - auto arr = State::safe_eval("array = reshape(collect(1:27), 3, 3, 3)"); - std::cout << arr.get_name() << std::endl; + jl_eval_string("module M1; module M2; module M3; end end end"); + std::string name = "name"; + jl_eval_string(("M1.M2.M3.eval(:(" + name + " = undef))").c_str()); + auto m1 = Main["M1"]; + auto m2 = m1["M2"]; + auto m3 = m2["M3"]; + auto as_module = m3.as(); + + as_module.assign(name, "alszdblasi"); return 0; + Test::initialize(); Test::test("catch c exception", [](){ @@ -439,7 +447,7 @@ int main() named[0] = 9999; Test::assert_that(State::eval("return var[1]").operator int() == 9999); - named = named.value(); + named = named.as_unnamed(); named[0] = 0; Test::assert_that(State::eval("return var[1]").operator int() == 9999); Test::assert_that(named[0].operator int() == 0); diff --git a/include/julia_extension.hpp b/include/julia_extension.hpp index 5357df9..e07f5dd 100644 --- a/include/julia_extension.hpp +++ b/include/julia_extension.hpp @@ -77,7 +77,7 @@ extern "C" /// @brief throw error if value is not of type named /// @param value /// @param types_name - inline void jl_assert_type(jl_value_t* value, jl_value_t* type) + inline void jl_assert_type(jl_value_t* value, jl_datatype_t* type) { static jl_function_t* assert_isa = jl_find_function("jluna", "assert_isa"); jluna::safe_call(assert_isa, value, type); diff --git a/include/module.hpp b/include/module.hpp index 1bcfc65..e6045b2 100644 --- a/include/module.hpp +++ b/include/module.hpp @@ -114,8 +114,7 @@ namespace jluna template T> inline T unbox(Any* value) { - static Any* module_t = jl_eval_string("return Module"); - jl_assert_type(value, module_t); + jl_assert_type(value, jl_module_type); return Module((jl_module_t*) value); } diff --git a/include/proxy.hpp b/include/proxy.hpp index 17ba3a2..d7d8eff 100644 --- a/include/proxy.hpp +++ b/include/proxy.hpp @@ -132,7 +132,7 @@ namespace jluna /// @brief create a new unnamed proxy that holds the same value /// @returns new proxy by value - [[nodiscard]] Proxy value() const; + [[nodiscard]] Proxy as_unnamed() const; /// @brief update value if proxy symbol was reassigned outside of operator= void update(); From 0f183716c6fa3dc88ff96bf93e4669c381da82eb Mon Sep 17 00:00:00 2001 From: clem Date: Sat, 19 Feb 2022 23:42:48 +0100 Subject: [PATCH 08/19] stashing --- .benchmark/main.cpp | 21 +++++++++++---------- .src/proxy.cpp | 8 +++++++- include/jluna.jl | 18 +++++++++++++++--- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.benchmark/main.cpp b/.benchmark/main.cpp index 9032af8..da6416b 100644 --- a/.benchmark/main.cpp +++ b/.benchmark/main.cpp @@ -68,22 +68,23 @@ int main() using namespace jluna; State::initialize(); - jl_eval_string("module M1; module M2; module M3; end end end"); + std::string name = "name_var"; + jl_eval_string((name + " = undef").c_str()); - Benchmark::run("New Proxy Eval", count, [](){ + std::cout << Main[name].get_name() << std::endl; - auto name = generate_string(8); - jl_eval_string(("M1.M2.M3.eval(:(" + name + " = \"" + generate_string(16) + "\"))").c_str()); + Main[name] = generate_string(16); - volatile auto value = Main["M1"]["M2"]["M3"][name].operator std::string(); - }); + return 0; - Benchmark::run("New Proxy Assign", count, [](){ + Benchmark::run("Proxy assign baseline", count, [&](){ + + jl_eval_string((name + " = " + generate_string(16)).c_str()); + }); - auto name = generate_string(8); - jl_eval_string(("M1.M2.M3.eval(:(" + name + " = undef))").c_str()); + Benchmark::run("New Proxy Assign", count, [&](){ - Main["M1"]["M2"]["M3"].as().assign(name, generate_string(16)); + Main[name] = generate_string(16); }); Benchmark::conclude(); diff --git a/.src/proxy.cpp b/.src/proxy.cpp index 3fb8419..3ccb952 100644 --- a/.src/proxy.cpp +++ b/.src/proxy.cpp @@ -71,6 +71,12 @@ namespace jluna Any * Proxy::ProxyValue::id() const { + if (value() == (Any*) jl_main_module) + { + std::cout << "nothing" << std::endl; + return jl_nothing; + } + return jl_ref_value(_id_ref); } @@ -110,7 +116,7 @@ namespace jluna return Proxy( _content.get()->get_field(symbol), _content, - (Any*) (_content->id() == nullptr ? nullptr : symbol) + (Any*) symbol ); } diff --git a/include/jluna.jl b/include/jluna.jl index 18cef13..66d0f99 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -650,13 +650,17 @@ module jluna make_named_proxy_id(id::Symbol, owner_id::ProxyID) ::ProxyID = return Expr(:(.), owner_id, QuoteNode(id)) # make as named with main as owner and symbol name - make_named_proxy_id(id::Symbol, owner_id::Nothing) ::ProxyID = return Expr(:(.), :Main, QuoteNode(id)) + make_named_proxy_id(id::Symbol, owner_id::Nothing) ::ProxyID = return QuoteNode(id) # make as named with owner and array index name make_named_proxy_id(id::Number, owner_id::ProxyID) ::ProxyID = return Expr(:ref, owner_id, id) # assign to proxy id - assign(new_value::T, name::ProxyID) where {T} = return Main.eval(Expr(:(=), name, new_value)) + function assign(new_value::T, name::ProxyID) where {T} + + println(string(Expr(:(=), name, new_value))) + return Main.eval(Expr(:(=), name, new_value)); + end # eval proxy id evaluate(name::ProxyID) ::Any = return Main.eval(name) @@ -669,6 +673,10 @@ module jluna """ function get_name(id::ProxyID) ::String + if length(id.args) == 0 + return "Main" + end + current = id while current.args[1] isa Expr && length(current.args) >= 2 @@ -687,7 +695,11 @@ module jluna end reg = r"\Q((jluna.memory_handler._refs[])[\E(.*)\Q])[]\E" - out = replace(out, reg => "") + captures = match(reg, out) + + if captures != nothing + out = replace(out, reg => "") + end return out; end From 93e64477ec42c78abf9674cfec3d6ac53516b3af Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 00:10:41 +0100 Subject: [PATCH 09/19] tests passing --- .benchmark/main.cpp | 23 +++++++++++------------ .src/proxy.cpp | 3 --- .test/main.cpp | 17 ++--------------- include/jluna.jl | 16 ++++------------ include/proxy.hpp | 2 ++ 5 files changed, 19 insertions(+), 42 deletions(-) diff --git a/.benchmark/main.cpp b/.benchmark/main.cpp index da6416b..5f41237 100644 --- a/.benchmark/main.cpp +++ b/.benchmark/main.cpp @@ -37,7 +37,7 @@ Number_t generate_number( return dist(engine); } -size_t count = 10000; +size_t count = 1000; void benchmark_lambda_call() { @@ -68,23 +68,22 @@ int main() using namespace jluna; State::initialize(); - std::string name = "name_var"; - jl_eval_string((name + " = undef").c_str()); + jl_eval_string("module M1; module M2; module M3; end end end"); - std::cout << Main[name].get_name() << std::endl; + Benchmark::run("Old Proxy Eval", count, [](){ - Main[name] = generate_string(16); + auto name = generate_string(8); + jl_eval_string(("M1.M2.M3.eval(:(" + name + " = \"" + generate_string(16) + "\"))").c_str()); - return 0; - - Benchmark::run("Proxy assign baseline", count, [&](){ - - jl_eval_string((name + " = " + generate_string(16)).c_str()); + volatile auto value = Main["M1"]["M2"]["M3"][name].operator std::string(); }); - Benchmark::run("New Proxy Assign", count, [&](){ + Benchmark::run("Old Proxy Assign", count, [](){ + + auto name = generate_string(8); + jl_eval_string(("M1.M2.M3.eval(:(" + name + " = undef))").c_str()); - Main[name] = generate_string(16); + Main["M1"]["M2"]["M3"].as().assign(name, generate_string(16)); }); Benchmark::conclude(); diff --git a/.src/proxy.cpp b/.src/proxy.cpp index 3ccb952..622cf38 100644 --- a/.src/proxy.cpp +++ b/.src/proxy.cpp @@ -72,10 +72,7 @@ namespace jluna Any * Proxy::ProxyValue::id() const { if (value() == (Any*) jl_main_module) - { - std::cout << "nothing" << std::endl; return jl_nothing; - } return jl_ref_value(_id_ref); } diff --git a/.test/main.cpp b/.test/main.cpp index fec9d13..cbe706d 100644 --- a/.test/main.cpp +++ b/.test/main.cpp @@ -12,19 +12,6 @@ using namespace jluna::detail; int main() { State::initialize(); - - jl_eval_string("module M1; module M2; module M3; end end end"); - std::string name = "name"; - jl_eval_string(("M1.M2.M3.eval(:(" + name + " = undef))").c_str()); - - auto m1 = Main["M1"]; - auto m2 = m1["M2"]; - auto m3 = m2["M3"]; - auto as_module = m3.as(); - - as_module.assign(name, "alszdblasi"); - return 0; - Test::initialize(); Test::test("catch c exception", [](){ @@ -699,8 +686,8 @@ int main() auto it = arr.at(0, 0, 0); Proxy as_proxy = it; - - Test::assert_that(as_proxy.get_name() == "Main.array[1]"); + as_proxy = 999; + Test::assert_that(State::eval("return array")[0].operator int() == 999); }); Test::test("array: comprehension", [](){ diff --git a/include/jluna.jl b/include/jluna.jl index 66d0f99..dd9caf7 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -641,7 +641,7 @@ module jluna const _refs_expression = Meta.parse("jluna.memory_handler._refs[]") # proxy id that is actually an expression, the ID of topmodule Main is - ProxyID = Union{Expr, Nothing} + ProxyID = Union{Expr, Symbol, Nothing} # make as unnamed make_unnamed_proxy_id(id::UInt64) = return Expr(:ref, Expr(:ref, _refs_expression, id)) @@ -650,17 +650,13 @@ module jluna make_named_proxy_id(id::Symbol, owner_id::ProxyID) ::ProxyID = return Expr(:(.), owner_id, QuoteNode(id)) # make as named with main as owner and symbol name - make_named_proxy_id(id::Symbol, owner_id::Nothing) ::ProxyID = return QuoteNode(id) + make_named_proxy_id(id::Symbol, owner_id::Nothing) ::ProxyID = return id # make as named with owner and array index name - make_named_proxy_id(id::Number, owner_id::ProxyID) ::ProxyID = return Expr(:ref, owner_id, id) + make_named_proxy_id(id::Number, owner_id::ProxyID) ::ProxyID = return Expr(:ref, owner_id, convert(Int64, id)) # assign to proxy id - function assign(new_value::T, name::ProxyID) where {T} - - println(string(Expr(:(=), name, new_value))) - return Main.eval(Expr(:(=), name, new_value)); - end + assign(new_value::T, name::ProxyID) where T = return Main.eval(Expr(:(=), name, new_value)); # eval proxy id evaluate(name::ProxyID) ::Any = return Main.eval(name) @@ -690,10 +686,6 @@ module jluna # modifying Base.string() result instead of parsing the expression is super slow but Proxy::get_name is a debug feature anyway out = string(id) - if out[1:5] == "Main." - out = chop(string(id), head = length("Main."), tail = 0) - end - reg = r"\Q((jluna.memory_handler._refs[])[\E(.*)\Q])[]\E" captures = match(reg, out) diff --git a/include/proxy.hpp b/include/proxy.hpp index d7d8eff..aec325b 100644 --- a/include/proxy.hpp +++ b/include/proxy.hpp @@ -174,6 +174,8 @@ namespace jluna /// @param id: jluna.memory_handler.ProxyID object ProxyValue(Any* value, std::shared_ptr& owner, Any* symbol_or_index); + ProxyValue(const ProxyValue&); + /// @brief access field /// @param symbol: name of field /// @returns pointer to field data From 12f2f2a49d83d10761474001dc738ac4165659c6 Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 00:38:16 +0100 Subject: [PATCH 10/19] stashing manual --- .src/gc_sentinel.cpp | 35 ------ docs/manual.md | 207 +++++++++++------------------------- include/gc_sentinel.hpp | 33 ------ include/jluna.jl | 2 - include/julia_extension.hpp | 3 +- 5 files changed, 65 insertions(+), 215 deletions(-) delete mode 100644 .src/gc_sentinel.cpp delete mode 100644 include/gc_sentinel.hpp diff --git a/.src/gc_sentinel.cpp b/.src/gc_sentinel.cpp deleted file mode 100644 index 98c1ff7..0000000 --- a/.src/gc_sentinel.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// -// Copyright 2022 Clemens Cords -// Created on 16.02.22 by clem (mail@clemens-cords.com) -// - -#include -#include - -namespace jluna -{ - GCSentinel::GCSentinel() - : _state_before(jl_gc_is_enabled()) - {} - - GCSentinel::~GCSentinel() - { - jl_gc_enable(_state_before); - } - - void GCSentinel::enable() - { - jl_gc_enable(true); - } - - void GCSentinel::disable() - { - jl_gc_enable(false); - } - - bool GCSentinel::is_enabled() const - { - return jl_gc_is_enabled(); - } -} - diff --git a/docs/manual.md b/docs/manual.md index b3bc459..464f091 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -46,6 +46,7 @@ Please navigate to the appropriate section by clicking the links below: 9.3 [Iterating](#iterating)
9.4 [Vectors](#vectors)
9.5 [Matrices](#matrices)
+ 9.6 [Generator Expressions](#generator-expressions) 10. [**Introspection**](#introspection)
10.1 [Type Proxies](#type-proxies)
10.2 [Core Type Constants](#core-types)
@@ -243,7 +244,7 @@ std::vector => Vector{T} * std::array => Vector{T} * std::pair => Pair{T, U} * std::tuple => Tuple{Ts...} * -std::map => IdDict{T, U} * +std::map => Dict{T, U} * std::unordered_map => Dict{T, U} * std::set => Set{T, U} * @@ -1438,6 +1439,63 @@ Note that `Array::operator[](Range_t&&)` (linear indexing with a range) al (this feature is not yet implemented, simply use `Array` until then) +## Generator Expressions + +One of julias most convenient features are [**generator expressions**](https://docs.julialang.org/en/v1/manual/arrays/#man-comprehensions) (also called list/array comprehensions). These are is a special kind of syntax that creates an iterable range. For example, the following expressions are equivalent: + +```julia +# comprehension +[i*i for i in 1:10 if i % 2 == 0] + +# mostly equivalent to +out = Vector() +f = i -> i*i +for i in 1:10 + if i % 2 == 0 + push!(out, f(i)) + end +end +``` + +In julia, we use `[]` when we want the expression to be vectorized and `()` when we want the expression to stay an object of type `Base.Generator`. The latter has the significant advantage that iterating and accessing its values is lazy-eval, meaning that we only do the any computation only when actually accessing the value. + +In `jluna`, we can create a generator expression using the postfix literal operator `_gen` on a c-string: + +```cpp +// in julia: +(i for i in 1:10 if i % 2 == 0) + +// in cpp: +"(i for i in 1:10 if i % 2 == 0)"_gen +``` +Note that when using `_gen`, **only round brackets are allowed**. Every generator expression has to be in round brackets. + +We can iterate a generator expression like so: + +```cpp +for (auto i : "(i for i in 1:10 if i % 2 == 0)"_gen) + std::cout << unbox(i) << std::endl; +``` +``` +2 +4 +6 +8 +10 +``` + +Where `i` needs to be unboxed manually and behaves exactly like an `Any*`. While this is convenient, the true power of generator expressions comes in its usaged along with `jluna::Vector`. We can: + +```cpp +// ...initialize a vector from a generator expression +auto vec = Vector("i*i for i in 1:99"_gen); + +// ...use a generator expressions just like a list index +vec["i for i in 1:99 if i < 50"_gen]; +``` + +Using generator expressions like this, we can imitate julia syntax very closely, despite being in a language that does not have list comprehension. + ## Introspection The main advantage julia has over C++ is its introspection and meta-level features. @@ -1771,149 +1829,10 @@ Once fully unrolled, we have access to the properties necessary for introspect. (this feature is not yet implemented) -## C-API - -(this section is not fully formatted yet. Code examples are given, however) - -```cpp -#include -#include -``` - -### Meaning of C-Types - -+ `jl_value_t*`: address julia-side value of arbitrary type -+ `jl_function_t*`: address of julia-side function -+ `jl_sym_t*`: address of julia-side symbol -+ `jl_module_t*`: address of julia-side singleton module - - `jl_main_module`: Main - - `jl_base_module`: Base - - `jl_core_module`: Core -+ `jl_datatype_t*`: address of julia-side singleton type - - `jl_any_type`: Any - - `jl_type_type`: Type - - `jl_module_type`: Module - - `jl_integer_type`: Integer - - etc. - -### Executing C Code - -```cpp -jl_eval_string("your code goes here") -jl_eval_string(R"( - your - multi-line - code - goes - here -)"); -``` - -### Forwarding Exceptions in C -```cpp -jl_eval_string("sqrt(-1)"); -// nothing happens - -jluna::forward_last_exception(); -// exception thrown -``` -``` -terminate called after throwing an instance of 'jluna::juliaException' - what(): DomainError(-1.0, "sqrt will only return a complex result if called with a complex argument. Try sqrt(Complex(x)).") -Stacktrace: -``` - -### Accessing Values in C - -```cpp -// by address: -Any* address = jl_eval_string("return value_name"); - -// by value -T value = unbox(jl_eval_string("return value_name")); - -// there is no memory-safe, GC-safe way to modify variables from within C only -``` -### Functions in C - -#### Accessing Functions in C - -```cpp -jl_function_t* println = jl_get_function((jl_module_t*) jl_eval_string("return Main.Base")), "println"); -``` - -#### Calling Functions in C - -```cpp -std::vector cpp_arguments = /* ... */ // T is any boxable type - -auto* function = jl_get_function(jl_base_module, "println"); -std::vector args; - -for (auto a : cpp_arguments) - args.push_back(box(a)); - -Any* res = jl_call(function, args, args.size()); -``` - -### Arrays in C -#### Accessing & Indexing Arrays in C - -```cpp -jl_array_t* array = (jl_array_t*) jl_eval_string("return [[1, 2, 3]; [2, 3, 4]; [3, 4, 5]]"); - -for (size_t i = 0; i < jl_array_len(array); ++i) // 0-based - std::cout << unbox(jl_arrayref(array, i)); - -// only linear indexing is available through the C-API -``` -``` -1 2 3 2 3 4 3 4 5 -``` - -#### Mutating Arrays in C -```cpp -jl_array_t* array = (jl_array_t*) jl_eval_string("return [1, 2, 3, 4]"); - -jl_arrayset(array, box(999), 0); -// box with exact equivalent of T julia -// jluna does fuzzy conversion behind the scenes, C doesn't - -jl_function_t* println = jl_get_function(jl_base_module, "println"); -jl_call1(println, (Any*) array); -``` - -### Strings in C -#### Accessing Strings in C -```cpp -Any* jl_string = jl_eval_string("return \"abcdef\""); -std::string std_string = std::string(jl_string_data(jl_string)); -std::cout << std_string << std::endl; -``` -``` -abcdef -``` -To modify strings, we need to cast them `jl_array_t` and modify them like arrays - -### Initialization & Shutdown in C - -```cpp -#include - -int main() -{ - jl_init(); - // or - jl_init_with_image("path/to/your/image", NULL); - - // all your application - - jl_atexit_hook(0); -} -``` - - - +## Performance +As of version 0.7, `jluna` went through extensive optimization, meaning it is finally decently fast. This section will give some pointers on how to achieve the best performance using its methods. +When evaluating `jluna`s performance, we first need to decide what to compare it to. When compared to native julia, `jluna` is objectively slower in almost every way. The best `jluna` can do is match julias performance. because all `jluna` does is be an interface to the julia state. This means it is not a very fair comparison, much more useful is comparing `jluna` to the native C-API. +The main overhead when dealing with proxies is that of reference allocation. To keep C++-side values allocated through the C-API safe from being immediately garbage collected, `jluna` needs to create a reference to them and store them in a julia-side datastructure. Anytime a proxy is created, a reference is created, a name for that proxy is created and both are stored in the global `jluna` state. \ No newline at end of file diff --git a/include/gc_sentinel.hpp b/include/gc_sentinel.hpp deleted file mode 100644 index 69cd9a6..0000000 --- a/include/gc_sentinel.hpp +++ /dev/null @@ -1,33 +0,0 @@ -// -// Copyright 2022 Clemens Cords -// Created on 16.02.22 by clem (mail@clemens-cords.com) -// - -#pragma once - -namespace jluna -{ - /// @brief lock-like object that keeps any values allocated while it is in scope safe from the garbage collector - class GCSentinel - { - public: - /// @brief ctor - GCSentinel(); - - /// @brief dtor, once this is called the sentinel seizes its protective behavior - ~GCSentinel(); - - /// @brief enable, from this point no garbage will be collected. If already enabled, does nothing - void enable(); - - /// @brief disable, from this point the garbage collector is free to collect. If already disabled, does nothing - void disable(); - - /// @brief is enabled - /// @returns bool - bool is_enabled() const; - - private: - bool _state_before; - }; -} diff --git a/include/jluna.jl b/include/jluna.jl index dd9caf7..52cddcd 100644 --- a/include/jluna.jl +++ b/include/jluna.jl @@ -683,9 +683,7 @@ module jluna current = current.args[1] end - # modifying Base.string() result instead of parsing the expression is super slow but Proxy::get_name is a debug feature anyway out = string(id) - reg = r"\Q((jluna.memory_handler._refs[])[\E(.*)\Q])[]\E" captures = match(reg, out) diff --git a/include/julia_extension.hpp b/include/julia_extension.hpp index e07f5dd..f7c254a 100644 --- a/include/julia_extension.hpp +++ b/include/julia_extension.hpp @@ -146,7 +146,8 @@ extern "C" } /// @brief pause gc and save current state - #define jl_gc_pause bool before = jl_gc_is_enabled(); jl_gc_enable(false); + #define jl_gc_pause bool _b_e_f_o_r_e_ = jl_gc_is_enabled(); jl_gc_enable(false); + // weird naming to avoid potential name-collision when used in C++ /// @brief restore previously saved state #define jl_gc_unpause jl_gc_enable(before); From e862d7e08db7e20d1e04533e4d4ebed4b2e7a571 Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 02:03:31 +0100 Subject: [PATCH 11/19] stashing clang compatibility mode --- .benchmark/main.cpp | 8 +++++--- .src/array.inl | 2 -- .src/box.inl | 2 -- .src/cppcall.inl | 8 ++++---- .src/proxy.cpp | 7 +------ .src/proxy.inl | 19 ++++++++++++++++++- CMakeLists.txt | 6 ++++-- include/julia_extension.hpp | 2 +- include/proxy.hpp | 12 ++++++++++++ 9 files changed, 45 insertions(+), 21 deletions(-) diff --git a/.benchmark/main.cpp b/.benchmark/main.cpp index 5f41237..ccfad58 100644 --- a/.benchmark/main.cpp +++ b/.benchmark/main.cpp @@ -68,6 +68,8 @@ int main() using namespace jluna; State::initialize(); + auto test = Main["abc"]; + jl_eval_string("module M1; module M2; module M3; end end end"); Benchmark::run("Old Proxy Eval", count, [](){ @@ -75,7 +77,7 @@ int main() auto name = generate_string(8); jl_eval_string(("M1.M2.M3.eval(:(" + name + " = \"" + generate_string(16) + "\"))").c_str()); - volatile auto value = Main["M1"]["M2"]["M3"][name].operator std::string(); + //auto value = Main["M1"]["M2"]["M3"][name].operator std::string(); }); Benchmark::run("Old Proxy Assign", count, [](){ @@ -83,7 +85,7 @@ int main() auto name = generate_string(8); jl_eval_string(("M1.M2.M3.eval(:(" + name + " = undef))").c_str()); - Main["M1"]["M2"]["M3"].as().assign(name, generate_string(16)); + //Main["M1"]["M2"]["M3"].as().assign(name, generate_string(16)); }); Benchmark::conclude(); @@ -130,7 +132,7 @@ int main() Benchmark::run("Proxy: CTOR DTOR Named", count, [](){ - volatile auto* p = new Proxy(Main["test"]); + volatile auto* p = new Proxy(Main[std::string("test")]); delete p; }); diff --git a/.src/array.inl b/.src/array.inl index 6d22aac..8c4ed42 100644 --- a/.src/array.inl +++ b/.src/array.inl @@ -3,8 +3,6 @@ // Created on 12.01.22 by clem (mail@clemens-cords.com) // -#include - namespace jluna { namespace detail diff --git a/.src/box.inl b/.src/box.inl index ffaaa03..5b0f5f3 100644 --- a/.src/box.inl +++ b/.src/box.inl @@ -183,8 +183,6 @@ namespace jluna for (auto& pair : value) pairs.push_back(jl_call2(make_pair, box(pair.first), box(pair.second))); - jl_gc_enable(before); - auto* res = jl_call(dict, pairs.data(), pairs.size()); jl_gc_unpause; return res; diff --git a/.src/cppcall.inl b/.src/cppcall.inl index 6c0982c..d623a6c 100644 --- a/.src/cppcall.inl +++ b/.src/cppcall.inl @@ -11,15 +11,15 @@ namespace jluna { namespace detail { - template, Bool> = true> - jl_value_t* detail::invoke_lambda(const Lambda_t* func, Args_t... args) + template, Bool>> + jl_value_t* invoke_lambda(const Lambda_t* func, Args_t... args) { (*func)(args...); return jl_nothing; } - template, Bool> = true> - jl_value_t* detail::invoke_lambda(const Lambda_t* func, Args_t... args) + template, Bool>> + jl_value_t* invoke_lambda(const Lambda_t* func, Args_t... args) { jl_value_t* res = (*func)(args...); return res; diff --git a/.src/proxy.cpp b/.src/proxy.cpp index 622cf38..c235ffc 100644 --- a/.src/proxy.cpp +++ b/.src/proxy.cpp @@ -109,12 +109,7 @@ namespace jluna Proxy Proxy::operator[](const std::string& field) { - jl_sym_t* symbol = jl_symbol(field.c_str()); - return Proxy( - _content.get()->get_field(symbol), - _content, - (Any*) symbol - ); + return operator[](field.c_str()); } Proxy Proxy::operator[](size_t i) diff --git a/.src/proxy.inl b/.src/proxy.inl index b6f0ed3..81d855e 100644 --- a/.src/proxy.inl +++ b/.src/proxy.inl @@ -60,7 +60,24 @@ namespace jluna template T Proxy::operator[](const std::string& field) { - return unbox(_content.get()->get_field(jl_symbol(field.c_str()))); + return operator[](field.c_str()); + } + + template, Bool>> + Proxy Proxy::operator[](const T* field) + { + jl_sym_t* symbol = jl_symbol(field); + return Proxy( + _content.get()->get_field(symbol), + _content, + (Any*) symbol + ); + } + + template, Bool>> + T Proxy::operator[](const U* field) + { + return unbox(_content.get()->get_field(jl_symbol(field))); } template diff --git a/CMakeLists.txt b/CMakeLists.txt index 05bb936..21e3493 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,9 @@ project(jluna) set(CMAKE_CXX_STANDARD 20) set(CMAKE_BUILD_TYPE Debug) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fconcepts") + +#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++") #clang-11 +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fconcepts") #g++11 ### JULIA ### @@ -106,7 +108,7 @@ add_library(jluna SHARED include/generator_expression.hpp .src/generator_expression.cpp #buffer - .src/symbol.inl include/gc_sentinel.hpp .src/gc_sentinel.cpp) + .src/symbol.inl) set_target_properties(jluna PROPERTIES LINKER_LANGUAGE C diff --git a/include/julia_extension.hpp b/include/julia_extension.hpp index f7c254a..dc13cbb 100644 --- a/include/julia_extension.hpp +++ b/include/julia_extension.hpp @@ -150,5 +150,5 @@ extern "C" // weird naming to avoid potential name-collision when used in C++ /// @brief restore previously saved state - #define jl_gc_unpause jl_gc_enable(before); + #define jl_gc_unpause jl_gc_enable(_b_e_f_o_r_e_); } \ No newline at end of file diff --git a/include/proxy.hpp b/include/proxy.hpp index aec325b..d37e938 100644 --- a/include/proxy.hpp +++ b/include/proxy.hpp @@ -47,12 +47,24 @@ namespace jluna /// @returns field as proxy Proxy operator[](const std::string& field); + /// @brief access field + /// @param field_name + /// @returns field as proxy + template, Bool> = true> + Proxy operator[](const T* field); + /// @brief access field /// @param field_name /// @returns unboxed value template T operator[](const std::string& field); + /// @brief access field + /// @param field_name + /// @returns unboxed value + template, Bool> = true> + T operator[](const U* field); + /// @brief linear indexing, if array type, returns getindex result /// @param index /// @returns field as proxy From 503452f1f1556a07187cf9458e84ef8fbfa9502e Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 02:28:44 +0100 Subject: [PATCH 12/19] appended clang support to readme --- CMakeLists.txt | 10 ++++++++-- README.md | 11 +++++------ docs/installation.md | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21e3493..07a15d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,14 @@ project(jluna) set(CMAKE_CXX_STANDARD 20) set(CMAKE_BUILD_TYPE Debug) -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++") #clang-11 -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fconcepts") #g++11 +# compiler support +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION STRGREATER_EQUAL "12.0.0") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION STRGREATER_EQUAL "10.0.0") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fconcepts") +else() + message(FATAL_ERROR "The only currently supported compilers are G++10, G++11 (recommended) and Clang-12") +endif() ### JULIA ### diff --git a/README.md b/README.md index e12bac1..c611bb6 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Heavily inspired in design and syntax by (but in no way affiliated with) the exc 4.3 [🔗 Troubleshooting](./docs/installation.md#troubleshooting)
5. [Dependencies](#dependencies)
5.1 [julia 1.7.0+](#dependencies)
- 5.2 [g++10](#dependencies)
+ 5.2 [Supported Compilers: g++11, clang12](#dependencies)
5.3 [cmake 3.19+](#dependencies)
5.4 [Linux / Mac OS](#dependencies) 6. [License](#license) @@ -150,12 +150,11 @@ Advanced users are encouraged to check the headers (available in `jluna/include/ For `jluna` you'll need: + [**Julia 1.7.0**](https://julialang.org/downloads/#current_stable_release) (or higher) -+ [**g++10**](https://askubuntu.com/questions/1192955/how-to-install-g-10-on-ubuntu-18-04) (or higher) - - including `-fconcepts` + [**cmake 3.16**](https://cmake.org/download/) (or higher) ++ C++ Compiler (see below) + unix-based, 64-bit operating system -Currently, only g++10 and g++11 are supported, clang support is planned in the future. +Currently [**g++10**](https://askubuntu.com/questions/1192955/how-to-install-g-10-on-ubuntu-18-04), [**g++11**](https://lindevs.com/install-g-on-ubuntu/) and [**clang-12**](https://linux-packages.com/ubuntu-focal-fossa/package/clang-12) are supported. g++-11 is the primary compiler used for development of `jluna` and is thus recommended. --- @@ -169,7 +168,7 @@ Users familiar with C++ and cmake can go through the following steps: Install: -+ `g++-11` ++ `g++-11` (or `clang-12`) + `julia 1.7+` + `cmake 3.16+` @@ -182,7 +181,7 @@ export JULIA_PATH=$(julia -e "println(joinpath(Sys.BINDIR, \"..\"))") mkdir build cd build -cmake -D CMAKE_CXX_COMPILER=g++-11 .. +cmake -D CMAKE_CXX_COMPILER=g++-11 .. # or clang-12 make ./JLUNA_TEST diff --git a/docs/installation.md b/docs/installation.md index 4aede28..e95fb15 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -68,7 +68,7 @@ With `JULIA_PATH` set correctly, we navigate into `~/my_project/jluna/` and crea cd ~/my_project/jluna mkdir build cd build -cmake -D CMAKE_CXX_COMPILER=g++-11 .. +cmake -D CMAKE_CXX_COMPILER=g++-11 .. # or clang-12 make ``` From 9b25b284131e1fdbb350ae921ceeb159ac8e3a42 Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 02:40:59 +0100 Subject: [PATCH 13/19] polish --- CMakeLists.txt | 4 ++-- docs/installation.md | 25 ++++++++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 07a15d0..c5e3aaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,6 +103,7 @@ add_library(jluna SHARED include/symbol.hpp .src/symbol.cpp + .src/symbol.inl include/type.hpp .src/type.cpp @@ -113,8 +114,7 @@ add_library(jluna SHARED include/generator_expression.hpp .src/generator_expression.cpp - #buffer - .src/symbol.inl) +) set_target_properties(jluna PROPERTIES LINKER_LANGUAGE C diff --git a/docs/installation.md b/docs/installation.md index e95fb15..cd9eb86 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -72,7 +72,7 @@ cmake -D CMAKE_CXX_COMPILER=g++-11 .. # or clang-12 make ``` -The following warnings may appear: +Warnings of the following type may appear: ``` (...) @@ -81,7 +81,7 @@ The following warnings may appear: (...) ``` -This is because the official julia header `julia.h` is slightly out of date. The warning is unrelated to `jluna`s codebase. `jluna` itself should report no warnings or errors. If this is not the case, head to [troubleshooting](#troubleshooting). +This is because the official julia header `julia.h` is slightly out of date. The warning is unrelated to `jluna`s codebase, `jluna` itself should report no warnings or errors. If this is not the case, head to [troubleshooting](#troubleshooting). We verify everything works by running `JLUNA_TEST` which we just compiled: @@ -155,12 +155,18 @@ cmake_minimum_required(VERSION 3.16) project(MyProject) # cmake and cpp settings -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fconcepts") set(CMAKE_CXX_STANDARD 20) - -# build type set(CMAKE_BUILD_TYPE Debug) +# compiler +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fconcepts") +else() + message(FATAL_ERROR "only g++11 or clang-12 are supported") +endif() + # julia if (NOT DEFINED ENV{JULIA_PATH}) message(WARNING "JULIA_PATH was not set correctly. Consider re-reading the jluna installation tutorial at https://github.com/Clemapfel/jluna/blob/master/docs/installation.md#setting-julia_path to fix this issue") @@ -183,7 +189,7 @@ add_executable(MY_EXECUTABLE ${CMAKE_SOURCE_DIR}/main.cpp) target_link_libraries(MY_EXECUTABLE ${JLUNA_LIB} ${JLUNA_C_ADAPTER_LIB} ${JULIA_LIB}) ``` -We again safe and close the file, then create our own build folder and run cmake, just like we did with `jluna` before +We again save and close the file, then create our own build folder and run cmake, just like we did with `jluna` before ```bash # in ~/my_project @@ -193,7 +199,7 @@ cmake -D CMAKE_CXX_COMPILER=g++-11 .. make ``` -If errors appear, be sure `JULIA_PATH` is still set correctly as it is needed to find `julia.h`. Otherwise head to [troubleshooting](#troubleshooting). +If errors appear, be sure `JULIA_PATH` is still set correctly, as it is needed to find `julia.h`. Otherwise, head to [troubleshooting](#troubleshooting). After compilation succeeded, the directory should now have the following layout: @@ -383,6 +389,7 @@ If your problem still persists, it may be appropriate to open a github issue. Fi + when building with cmake, you specified `-D CMAKE_CXX_COMPILER=g++-11` correctly - cmake is able to find the executable `g++-11` in the default search paths - the same applies to `-D CMAKE_C_COMPILER=gcc-9` as well + - any given issue may be compiler specific, consider trying `clang-12` instead + `~/my_project/CMakeLists.txt` is identical to the code in [installation](#linking-jluna) + `State::initialize` and `JULIA_PATH` are modified as outlined [above](#stateinitialize-fails) + `jluna` was freshly pulled from the git repo and recompiled @@ -392,8 +399,8 @@ If your problem still persists, it may be appropriate to open a github issue. Fi If and only if all of the above apply, head to the [issues tab](https://github.com/Clemapfel/jluna/issues). There, please create an issue and + describe your problem -+ state your operating system and distro ++ state your operating system, distro and compiler version + copy-paste the output of `JLUNA_TEST` (even if all tests are `OK`) -+ provide a minimum working example that recreates the bug ++ provide a minimum working example of your project that recreates the bug We will try to resolve your problem as soon as possible and are thankful for your input and collaboration. From ee5a033c23e5e3fcb0b40e071aad752f8c0e9b29 Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 03:06:32 +0100 Subject: [PATCH 14/19] added file eval --- .src/state.cpp | 26 ++++++++++++++++++++++++++ docs/manual.md | 14 ++++---------- include/state.hpp | 11 +++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/.src/state.cpp b/.src/state.cpp index 1f1f86b..34a341c 100644 --- a/.src/state.cpp +++ b/.src/state.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -16,6 +17,7 @@ #include #include + namespace jluna::detail { static void on_exit() @@ -129,6 +131,30 @@ namespace jluna::State jl_gc_unpause; return Proxy(result, nullptr); } + + Proxy eval_file(const std::string& path) noexcept + { + std::fstream file; + file.open(path); + + std::stringstream str; + str << file.rdbuf(); + + file.close(); + return eval(str.str()); + } + + Proxy safe_eval_file(const std::string& path) noexcept + { + std::fstream file; + file.open(path); + + std::stringstream str; + str << file.rdbuf(); + + file.close(); + return safe_eval(str.str()); + } template T safe_return(const std::string& full_name) diff --git a/docs/manual.md b/docs/manual.md index 464f091..87b42d5 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -110,7 +110,9 @@ State::safe_eval(R"( )"); ``` -`safe_` overloads have marginal overhead from try-catching and sanity checking inputs. If you want maximum performance (and the corresponding debug experience),`State::eval` provides a 0-overhead solution instead. +`safe_` overloads have marginal overhead from try-catching and sanity checking inputs. If you want maximum performance (and the corresponding debug experience),`State::eval` provides a 0-overhead solution instead. + +To run an entire julia file, we can either trigger `Base.include` julia-side or we can call `State::(safe_)eval_file`. This function simply reads the file as a string and call `State::(safe_)eval`. ## Garbage Collector (GC) @@ -1827,12 +1829,4 @@ Once fully unrolled, we have access to the properties necessary for introspect. ## Usertypes -(this feature is not yet implemented) - -## Performance - -As of version 0.7, `jluna` went through extensive optimization, meaning it is finally decently fast. This section will give some pointers on how to achieve the best performance using its methods. - -When evaluating `jluna`s performance, we first need to decide what to compare it to. When compared to native julia, `jluna` is objectively slower in almost every way. The best `jluna` can do is match julias performance. because all `jluna` does is be an interface to the julia state. This means it is not a very fair comparison, much more useful is comparing `jluna` to the native C-API. - -The main overhead when dealing with proxies is that of reference allocation. To keep C++-side values allocated through the C-API safe from being immediately garbage collected, `jluna` needs to create a reference to them and store them in a julia-side datastructure. Anytime a proxy is created, a reference is created, a name for that proxy is created and both are stored in the global `jluna` state. \ No newline at end of file +(this feature is not yet implemented) \ No newline at end of file diff --git a/include/state.hpp b/include/state.hpp index 6f7dc5a..c279343 100644 --- a/include/state.hpp +++ b/include/state.hpp @@ -47,6 +47,17 @@ namespace jluna::State /// @exceptions if an error occurs julia-side, a JuliaException will be thrown Proxy safe_eval(const std::string&); + /// @brief execute file + /// @param path to file + /// @returns proxy to result, if any + Proxy eval_file(const std::string& path) noexcept; + + /// @brief execute file + /// @param path to file + /// @returns proxy to result, if any + Proxy safe_eval_file(const std::string& path) noexcept; + + /// @brief access a value, equivalent to unbox(jl_eval_string("return " + name)) /// @tparam T: type to be unboxed to /// @param full name of the value, e.g. Main.variable._field[0] From fddcdd8e16a40dd56260a0524bdb422cf86cfc6b Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 03:22:59 +0100 Subject: [PATCH 15/19] stashing peformance section --- docs/manual.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/manual.md b/docs/manual.md index 87b42d5..7cdb96d 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -58,7 +58,7 @@ Please navigate to the appropriate section by clicking the links below: 10.8 [Type Classification](#type-classification)
11. [~~Expressions~~](#expressions)
12. [~~Usertypes~~](#usertypes)
-13. [~~C-API~~](#c-api)
+13. [Performance](#performance)
## Initialization @@ -1827,6 +1827,57 @@ Once fully unrolled, we have access to the properties necessary for introspect. (this feature is not yet implemented) +--- + ## Usertypes -(this feature is not yet implemented) \ No newline at end of file +(this feature is not yet implemented) + +--- + +## Performance + +This section will give some tips on how to achieve the best performance in `jluna`. As of release 0.7, `jluna` went through extensive optimization to minimize the amount of overhead as much as possible. Still, when compared to pure julia, `jluna` will always loose. Comparing `jluna` to the C-library however, `jluna` does much better. + +### Stay Julia-Side + +The most important tip is to do as much of the taxing computation as possible julia-side. Julia is a race-car of a language and unless our C++-library is similarly optimized, it is usually a better idea to do things with julia. To do this, we write performance critical code in a `.jl` file, then call that file from C++. Recall that proxies do not actually own any C++-side memory, thuse calling julia functions on proxies actually triggers no significant computation C++-side. Taxing work is only being done once the proxies value changes from one languages state to the other. + +### Avoid Changing States + +By far the easiest way to completely tank a programs performance is to unnecessarily box/unbox values constantly. If our program requires operation A, B, C julia side and X, Y, Z C++ side, it's very important to execute everything like: `ABC XYZ ` rather than `A X B Y `, etc.. Furthermore it's important to be aware of what we're boxing. If we were to give each box/unbox call a grade from 1 - 5 where 1 is 0 overhead and 5 is the most amount of overhead, box/unbox calls would be graded like so: + +``` +// type // grade, 1 ist fastest, 5 is slowest +int, size_t, float, etc 1 +const char* 1 +Proxy 2 +vector 2 +Pair 3 +Set 3 +Tuple 4 +(unordered) map 4 +lambda 5 +``` + +It's easy to remember these grades by simply considering, how different an object C++-side and julia-side representations are. Obviously to wrap a C+-side lambda into a julia-side object, a lot of things have to happen behind the scenes while a vector pretty much transfers directly between languages. + +### Minimize Proxy Construction + +The main overhead incurred by `jluna`s safety is that of proxy allocation. Anytime a proxy is constructed, its value and name have to be added to an internal state that protects them from the garbage collector. Consider the following: + +```cpp +/// constructs 5 proxies +auto a = Main["Module1"]["Module2"][vector_var][1]["field"]; + +/// constructs 1 proxy +auto b = State::safe_script("return Main.Module1.Module2.vector_var[1].field"); +``` + +It is no exaggeration that the second call is about 5 times faster. What we loose for that speedup is the convenience of being able to mutate the julia variable, as proxy `b` is unnamed. + +### Use the C-Library + +When performance needs to be optimal, that is not *good*, not *extremely good* but the absolute best it could possible be, we might need to resort to the C-library.Values are hard to manage, the syntax is very clunky and the garbage collector will probably steal many of our values from under our nose when we're not looking and segfault the program, but one thing C has is optimal speed. Luckily, the C-library and `jluna` are freely mixable, though remember to `.update` any proxies whos julia-side value was modified by the C-library. + +--- \ No newline at end of file From 025a479dba91da3b7d4b8c76d8245e81cf289f99 Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 21:26:53 +0100 Subject: [PATCH 16/19] polish --- CMakeLists.txt | 2 +- README.md | 29 +++--- docs/manual.md | 265 +++++++++++++++++++++++++++---------------------- 3 files changed, 167 insertions(+), 129 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c5e3aaa..8b140db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION STRGRE elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION STRGREATER_EQUAL "10.0.0") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fconcepts") else() - message(FATAL_ERROR "The only currently supported compilers are G++10, G++11 (recommended) and Clang-12") + message(FATAL_ERROR "Currently, the only supported compilers are G++10, G++11 (recommended) and Clang-12") endif() ### JULIA ### diff --git a/README.md b/README.md index c611bb6..1b1180d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# jluna: A modern julia ⭤ C++ Wrapper (v0.5.4) +# jluna: A modern julia ⭤ C++ Wrapper (v0.7.0) ![](./header.png) Julia is a beautiful language, it is well-designed and well-documented. Julias C-API is also well-designed, less beautiful and much less... documented. -Heavily inspired in design and syntax by (but in no way affiliated with) the excellent Lua⭤C++ wrapper [**sol2**](https://github.com/ThePhD/sol2), `jluna` aims to fully wrap the official julia C-API and replace it in usage in C++ projects, by making accessing julias unique strengths through C++ safe, hassle-free and just as beautiful. +Heavily inspired in design and syntax by (but in no way affiliated with) the excellent Lua⭤C++ wrapper [**sol2**](https://github.com/ThePhD/sol2), `jluna` aims to fully wrap the official julia C-API, replacing it in usage in C++ projects by making accessing julias unique strengths through C++ safe, hassle-free, and just as beautiful. --- @@ -28,7 +28,7 @@ Heavily inspired in design and syntax by (but in no way affiliated with) the exc --- ### Showcase -#### Access julia-Side Values/Functions +#### Access Julia-Side Values/Functions ```cpp // execute arbitrary strings with exception forwarding State::safe_script(R"( @@ -114,13 +114,14 @@ Some of the many advantages `jluna` has over the C-API include: + expressive generic syntax + call C++ functions from julia using any julia-type + assigning C++-side proxies also mutates the corresponding variable with the same name julia-side -+ julia-side values, including temporaries, are kept safe from the garbage collector while they are in use C++-side -+ verbose exception forwarding from julia, compile-time assertions ++ julia-side values, including temporaries, are kept safe from the garbage collector ++ verbose exception forwarding, compile-time assertions + wraps [most](./docs/quick_and_dirty.md#list-of-unboxables) of the relevant C++ `std` objects and types + multidimensional, iterable array interface with julia-style indexing -+ Deep, C++-side introspection functionalities for julia objects -+ manual written by a human for beginners -+ inline documentation for IDEs for both C++ and julia code ++ deep, C++-side introspection functionalities for julia objects ++ fast! All code is considered performance-critical and was optimized for minimal overhead compared to the C-API ++ manual written by a human ++ inline documentation for IDEs for both C++ and Julia code + freely mix `jluna` and the C-API + And more! @@ -128,11 +129,11 @@ Some of the many advantages `jluna` has over the C-API include: (in order of priority, highest first) -+ 0-overhead performance versions of proxies and `cppcall` -+ linear algebra wrapper, matrices ++ windows / MSVC support ++ thread-safety, parallelization + usertypes, creating modules and struct completely C++-side ++ linear algebra wrapper, matrices + expression proxies -+ thread-safety, parallelization + multiple julia worlds, save-states: restoring a previous julia state --- @@ -156,11 +157,15 @@ For `jluna` you'll need: Currently [**g++10**](https://askubuntu.com/questions/1192955/how-to-install-g-10-on-ubuntu-18-04), [**g++11**](https://lindevs.com/install-g-on-ubuntu/) and [**clang-12**](https://linux-packages.com/ubuntu-focal-fossa/package/clang-12) are supported. g++-11 is the primary compiler used for development of `jluna` and is thus recommended. +Windows and MSVC support is planned in the near future. + + + --- ## [Installation & Troubleshooting](./docs/installation.md) -A step-by-step tutorial on how to create, compile and link a new C++ Project with `jluna` can be found [here](./docs/installation.md). It is recommended that you follow this guide closely instead of trying to resolve issues on your own. +A step-by-step tutorial on how to create, compile and link a new C++ Project with `jluna` can be found [here](./docs/installation.md). It is recommended that you follow this guide closely, instead of trying to resolve issues on your own. ### For Advanced Users Only diff --git a/docs/manual.md b/docs/manual.md index 7cdb96d..9428a46 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -58,7 +58,11 @@ Please navigate to the appropriate section by clicking the links below: 10.8 [Type Classification](#type-classification)
11. [~~Expressions~~](#expressions)
12. [~~Usertypes~~](#usertypes)
-13. [Performance](#performance)
+13. [**Performance**](#performance)
+ 13.1 [Staying Julia-Side](#stay-julia-side)
+ 13.2 [(Un)Boxing](#minimize-unboxing)
+ 13.3 [Proxy Construction](#minimize-proxy-construction)
+ 13.4 [Using the C-Library](#use-the-c-library)
## Initialization @@ -82,11 +86,11 @@ When a program exits regularly, all julia-side values allocated through `jluna` ## Executing Code -`jluna` has two ways of executing code (represented C++ side as string): +`jluna` has two ways of executing code (represented C++-side as a string): + *with* exception forwarding and + *without* exception forwarding. -Code called without exception forwarding will not only not report any errors, but simply appear to "do nothing". If a fatal error occurs, the entire application wil crash without warning.
+If code called without exception forwarding fails, it will not only not report any exceptions but simply seem to do nothing. If a fatal error occurs, the entire application wil crash without warning.
Because of this, it is highly recommended to always air on the side of safety by using the `safe_` overloads whenever possible: @@ -110,9 +114,9 @@ State::safe_eval(R"( )"); ``` -`safe_` overloads have marginal overhead from try-catching and sanity checking inputs. If you want maximum performance (and the corresponding debug experience),`State::eval` provides a 0-overhead solution instead. +`safe_` overloads have marginal overhead from try-catching and sanity checking inputs. If you optimal performance (and the corresponding debug experience) is needed,`State::eval` provides a 0-overhead version instead. -To run an entire julia file, we can either trigger `Base.include` julia-side or we can call `State::(safe_)eval_file`. This function simply reads the file as a string and call `State::(safe_)eval`. +To run an entire julia file, we can either trigger a `Base.include` call Julia-side, or we can call `State::(safe_)eval_file`. This function simply reads the file as a string and uses it as the argument for `State::(safe_)eval`. ## Garbage Collector (GC) @@ -207,7 +211,7 @@ Where `Is` is a concept that resolves to true if `U` and `T` are the same ### List of (Un)Boxables -The following types are both boxable and unboxable out-of-the-box. +The following types are both boxable and unboxable: ```cpp // cpp type // julia-side type after boxing @@ -268,14 +272,14 @@ std::function => function (::Any, ::Any, ::Any, :: std::function)> => function (::Vector{Any}) ::Any ``` -We will learn more on how to box/unbox functions specifically in the [section on calling C++ functions from julia](#functions). +We will learn more on how to box/unbox functions, specifically, in the [section on calling C++ functions from julia](#functions). The template meta function `to_julia_type` is provided to convert a C++ type into a julia-type. `to_julia_type::type_name` is the name of the type as a string. -``` +```cpp std::cout << to_julia_type>::type_name << std::endl; ``` -``` +```julia Array{UInt64, 4} ``` @@ -283,7 +287,7 @@ Array{UInt64, 4} ## Accessing Variables -Now that we know we can (un)box values to move them from C++ to julia, we come to actually using them. +Now that we know that we can move values between C++ and Julia, we will learn how to actually do this. Let's say we have a variable `var` julia-side: @@ -313,7 +317,7 @@ std::cout << as_int << std::endl; 1234 ``` -While both ways get us the desired value, neither is a good way to actually manage the variable itself. How do we reassign it? Can we dereference the C-pointer? Who has ownership of the memory? Is it safe from the garbage collector?
All these questions are hard to manage using the C-API, however `jluna` offers a one-stop-shop solution: `jluna::Proxy`. +While both ways get us the desired value, neither is a good way to actually manage the variable itself. How do we reassign it? Can we dereference the C-pointer? Who has ownership of the memory? Is it safe from the garbage collector?
All these questions are hard to manage using the C-API, however `jluna` offers a simultaneous: `jluna::Proxy`. ### Proxies @@ -344,13 +348,13 @@ auto as_int = (int) proxy; auto as_int = unbox(proxy) // discouraged ``` -Where the first version is encouraged for style reasons. `jluna` handles implicit conversion behind the scenes. This makes it, so we don't have to worry what the actual type of the julia-value is. `jluna` will try it's hardest to make our declaration work: +Where the first version is encouraged for style reasons. `jluna` handles implicit conversion behind the scenes. This makes it, so we don't have to worry what the actual type of the julia-value is. `jluna` will try to make our declaration work: ```cpp State::eval("var = 1234") auto proxy = State::eval("return var") -// all of the following works by triggering julia-side conversion: +// all of the following works by triggering julia-side conversion implicitly: size_t as_size_t = proxy; std::string as_string = proxy; std::complex as_complex = proxy; @@ -380,14 +384,14 @@ This is already much more convenient than manually unboxing c-pointers, however ## Mutating Variables -To mutate a variable means to change its value, its type or both.
+To mutate a variable means to change its value, its type, or both.
-As stated before, a proxy holds exactly one pointer to julia-side memory and exactly one symbol. There are two types of symbols it can hold: +As stated before, a proxy holds exactly one pointer to julia-side memory and exactly one symbol. There are two types of symbols a proxy can hold: + a symbol starting with the character `#` is called an **internal id** + any other symbol is called a **name** -The behavior of proxies changes, depending on wether their symbol is a name or not. A proxies whose symbol is a name is called a **named proxy**, a proxy whose symbol is an internal id is called an **unnamed proxy**. +The behavior of proxies changes, depending on whether their symbol is a name, or not. A proxies whose symbol is a name, is called a **named proxy**, a proxy whose symbol is an internal id, is called an **unnamed proxy**. To generate an unnamed proxy, we use `State::(safe_)eval`, `State::safe_return` or `State::new_unnamed_`.
To generate a named proxy we use `Proxy::operator[]` or `State::new_named_`: @@ -398,10 +402,10 @@ auto unnamed_proxy = State::safe_eval("return var"); auto named_proxy = Main["var"]; // or, in one line: -auto named_proxy = State::new_named_int64() = 1234; +auto named_proxy = State::new_named_int64("var", 1234); ``` -where `Main`, `Base`, `Core` are global, pre-initialized proxies holding the corresponding julia-side modules singletons. +where `Main`, `Base`, `Core` are global, pre-initialized proxies holding the corresponding julia-side modules. We can check a proxies name using `.get_name()`: @@ -411,17 +415,17 @@ std::cout << named_proxy.get_name() << std::endl; ``` ``` -Main.var +var ``` -We see that `unnamed_proxy`s symbol is `#9`, while `named_proxy`s symbol is `Main.var`, the same name as the julia-side variable `var` that we used to create it. +We see that `unnamed_proxy`s symbol is `#9`, while `named_proxy`s symbol is the same name as the julia-side variable `var` that we used to create it. -It may seem hard to remember wether a function returns a named or unnamed proxy, however there is a simple logic behind it. In julia: +It may seem hard to remember whether a function returns a named or unnamed proxy, however there is a simple logic behind it. In julia: ```julia var = 1234 new_variable = begin return var end ``` -Here the statement `return var` returns **by value**, `new_variable` has no way to know where that value came from. It's the same with proxies, calling `State::eval("return var")` works exactly like in julia, the function returns **by value**. `Proxy::operator[]` on the other hand *does* have access to the exact name, we explicitly used it to call the operator, hence why the proxy can store it for later use. +The statement `return var` returns **by value**, `new_variable` has no way to know where that value came from, thus mutating `new_variable` leaves `var` unaffected. It's the same with proxies, calling `State::eval("return var")` works exactly like in julia, the function returns **by value**. `Proxy::operator[]` on the other hand *does* have access to the exact name, we explicitly used it to call the operator, hence why the proxy can store it for later use. ### Named Proxies @@ -450,7 +454,7 @@ cpp : 5678 julia : 5678 ``` -After assignment, both the value `named_proxy` is pointing to, and the variable of the same name `Main.var` were affected by the assignment. This is somewhat atypical for julia but familiar to C++-users. A named proxy acts like a reference to the julia-side variable.

While admittedly somewhat unusual, because of this behavior we are able to do things like: +After assignment, both the value `named_proxy` is pointing to, and the variable of the same name `Main.var` were affected by the assignment. This is somewhat atypical for julia but familiar to C++-users. A named proxy acts like a reference to the julia-side variable.

While initially somewhat hard to wread ones head around, because of this behavior, we are able to do things like: ```cpp State::safe_eval("vector_var = [1, 2, 3, 4]") @@ -498,7 +502,7 @@ std::cout << unnamed_proxy.get_name() << std::endl; ``` -We see that it does not have a name, just an internal id. Therefore, it has no way to know where its value came from and thus has no way to mutate anything but C++-side memory address. An unnamed proxy thus behaves like a deepcopy of the value, **not** like a reference. +We see that it does not have a name, just an internal id. Therefore, it has no way to know where its value came from and thus has no way to mutate anything but its C++-side memory address. An unnamed proxy thus behaves like a deepcopy of the value, **not** like a reference. #### In Summary @@ -538,7 +542,7 @@ While still retaining its name: std::cout << named_proxy.get_name() << std::endl; ``` ``` -Main.var +var ``` This proxy is in what's called a *detached state*. Even though it is a named proxy, its current value does not correspond to the value of it's julia-side variable. This may have unforeseen consequences: @@ -582,7 +586,7 @@ after : 5678 Because `named_proxy` still has its name `Main.var`, it reassigns the variable to `5678` which was specified through C++. -While this behavior is internally consistent and does not lead to undefined behavior (a proxy will mutate its variable, regardless of the proxies value or the current value of the variable), it may cause unintentional bugs when reassigning the same variable frequently in both C++ and julia. To alleviate this, `jluna` offers a member function `Proxy::update()`, which evaluates the proxies name and replaces its value with value of the correspondingly named variable: +While this is internally consistent and does not lead to undefined behavior (a proxy will mutate its variable, regardless of the proxies value or the current value of the variable), it may cause unintentional bugs when reassigning the same variable frequently in both C++ and julia. To alleviate this, `jluna` offers a member function, `Proxy::update()`, which evaluates the proxies name and replaces its value with the value of the correspondingly named variable: ```cpp auto named_proxy = State::new_int64("var", 1234); @@ -599,13 +603,13 @@ While not necessary to do everytime an assignment happens, it is a convenient wa ### Making a Named Proxy Unnamed -Sometimes it is desirable to stop a proxy from mutating the corresponding variable, even though it is a named proxy. While we cannot change a proxies name, we can generate a new unnamed proxy pointing to the same value using the member function `Proxy::value()`. This functions returns an unnamed proxy pointing to a newly allocated deepcopy of the value of the original proxy. +Sometimes it is desirable to stop a proxy from mutating the corresponding variable, even though it is a named proxy. While we cannot change a proxies name, we can generate a new unnamed proxy pointing to the same value using the member function `Proxy::as_unnamed()`. This functions returns an unnamed proxy pointing to a newly allocated deepcopy of the value of the original proxy. ```cpp State::eval("var = 1234"); auto named_proxy = Main["var"]; -auto value = named_proxy.value(); // create nameless deepcopy +auto value = named_proxy.as_unnamed(); // create nameless deepcopy // assigning either does not affect the other value = 9999; @@ -619,16 +623,16 @@ named: 4567 value: 9999 ``` -Therefore, to make a named proxy unnamed, we can simply assign its `value` to itself: +Therefore, to make a named proxy unnamed, we can simply assign the result of `as_unnamed` to itself: ```cpp auto proxy = /*...*/ proxy = proxy.value() ``` -This way, the actual julia-side memory stays safe, but the proxy can now be mutated without interfering with any variable of the former name. +This way, the actual julia-side memory is unaffected, but the proxy can now be mutated without interfering with any variable of the former name. -Calling `.value()` on a proxy that is already unnamed simply creates another unnamed deepcopy with a new internal id. +Calling `.as_unnamed()` on a proxy that is already unnamed simply creates another unnamed deepcopy with a new internal id. ### Accessing Fields @@ -691,9 +695,9 @@ For proxies who are indexable (that is `getindex(::T, ::Integer)` is defined), ` ## Module Proxies -We've seen how proxies handle their value. While the general-purpose `jluna::Proxy` handles primitives and structtypes well, some julia classes provide additional functionality beyond what we've seen so far. One of these more specialized proxies is `jluna::Module`. For the sake of brevity, henceforth, `jluna::Proxy` will be referred to as "general proxy" while `jluna::Module` will be called "module proxy", `jluna::Symbol` as "symbol proxy", `jluna::Type` as "type proxy", etc.. +We've seen how proxies handle their value. While the general-purpose `jluna::Proxy` deals with primitives and structtypes well, some julia classes provide additional functionality beyond what we've seen so far. One of these more specialized proxies is `jluna::Module`. For the sake of brevity, henceforth, `jluna::Proxy` will be referred to as "general proxy" while `jluna::Module` will be called "module proxy", `jluna::Symbol` as "symbol proxy", `jluna::Type` as "type proxy", etc.. -A general proxy can hold any value, including that of a module, however, `jluna::Module`, the specialized module proxy, can only hold a julia-side module. Furthermore, `Module` inherits from `Proxy` and thus retains all public member functions. Being more specialized just means that we have additional member functions to work with. +A general proxy can hold any value, including that of a module, however, `jluna::Module`, the specialized module proxy, can only hold a julia-side value of type `Base.Module`. Furthermore, `Module` inherits from `Proxy` and thus retains all public member functions. Being more specialized just means that we have additional member functions to work with on top of what was already available. ### Constructing Module Proxies @@ -712,9 +716,9 @@ auto as_module = as_general.operator Module(); auto as_module = as_general.as(); ``` -If the original proxy `as_general` was unnamed then the resulting module proxy will also be unnamed, if the original was named, the newly generated module proxy will be named. +If the original proxy `as_general` was unnamed, then the resulting module proxy will also be unnamed. If the original was named, the newly generated module proxy will be named. -`jluna` offers 3 already initialized module proxies that we've seen used in the previous section: +`jluna` offers 3 already initialized module proxies, we've seen them used in the previous section: ```cpp // in module.hpp @@ -774,7 +778,7 @@ Stacktrace: signal (6): Aborted ``` -We get an exception. This is expected as the same expression `OurModule.variable = 456` in the julia REPL would throw the same exception. The reason for this is that both `State::eval` and `Proxy::operator[]` evaluate the expression containing assignment in `Main` scope. Thus, any variable that is not in `Main` cannot be assigned. `jluna::Module` allows for this by providing `eval` and `safe_eval`, both of which evaluate in its own scope: +We get an exception. This is expected, as the same expression `OurModule.variable = 456` in the julia REPL would throw the same exception. The reason for this is that both `State::eval` and `Proxy::operator[]` evaluate the expression containing assignment in `Main` scope. Thus, any variable that is not in `Main` cannot be assigned. `jluna::Module`, however, does allow for this by providing the functions `eval` and `safe_eval`, both of which evaluate in its own scope: ```cpp State::safe_eval(R"( @@ -792,7 +796,7 @@ Base["println"](our_module["variable"]); ``` Where `eval` does not forward exceptions, while `safe_eval` does, just like using the global `State::(safe_)eval`.
-Equivalently, module proxies offer two functions `assign` and `create_or_assign` that assign a variable of the given name a value. If the variable does not exist, `assign` will throw an `UndefVarError`, while `create_or_assign` will create a new variable of that name in module-scope, then assign it the value: +Equivalently, module proxies offer two functions, `assign` and `create_or_assign`, that assign a variable of the given name a given value. If the variable does not exist, `assign` will throw an `UndefVarError`, while `create_or_assign` will create a new variable of that name in module-scope, then assign it the value: ```cpp our_module.create_or_assign("new_variable", std::vector{1, 2, 3, 4}); @@ -861,7 +865,7 @@ Base #### Bindings -Bindings are a little more relevant. `Module::get_bindings` returns a map (or `IdDict` in julia parlance). For each pair in the map, `.first` is the *name* (of type `jluna::Symbol`) of a variable, `.second` is the *value* of the variable, as an unnamed general proxy. +`Module::get_bindings` returns a map (or `IdDict` in julia parlance). For each pair in the map, `.first` is the *name* (of type `jluna::Symbol`) of a variable, `.second` is the *value* of the variable, as an unnamed general proxy. We will learn more about `jluna::Symbol` soon. For now, simply think of them as their julia-side equivalent `Base.Symbol`. ```cpp @@ -890,7 +894,7 @@ var2 => 0 where `jl_to_string` is a C-function that takes an `Any*` and calls `Base.string`, returning the resulting string. -Because the proxies hold ownership of the bound values and are unnamed, the result of `get_bindings` is a stable snapshot of a module, even if the module continues to be modified, the map returned by `get_bindings` stays the same. This means `get_bindings` gives us a way to save the current state of the module. +Because the proxies hold ownership of the bound values and are unnamed, the result of `get_bindings` is a stable snapshot of a module. Even if the module continues to be modified, the map returned by `get_bindings` stays the same. This means `get_bindings` gives us a way to save the current state of the module. --- @@ -922,7 +926,7 @@ Where, unlike with general proxy, the value the proxy is pointing to is asserted The main additional functionality `jluna::Symbol` brings, is that of *constant time hashing*. -A hash is essentially a UInt64 we assign to things as a label. In julia, hashes are unique and there a no hash collisions. This means if `A != B` then `hash(A) != hash(B)` and, furthermore, if `hash(A) == hash(B)` then `A === B` (sic). Unlike with many other classes, `Base.Symbol` can be hashed in constant time. This is because the hash is precomputed at the time of construction. We access the hash of a symbol proxy using `.hash()` +A hash is essentially a UInt64 we assign to things as a label. In julia, hashes are unique and there a no hash collisions. This means if `A != B` then `hash(A) != hash(B)` and, furthermore, if `hash(A) == hash(B)` then `A == B`. Unlike with many other classes, `Base.Symbol` can be hashed in constant time. This is because the hash is precomputed at the time of construction. We access the hash of a symbol proxy using `.hash()` ``` auto symbol = Main["symbol_var"]; @@ -934,7 +938,7 @@ name: abc hash: 16076289990349425027 ``` -In most cases, it is impossible to predict which number will be assigned to which symbol. This also means for Symbols `s1` and `s2` if `string(s1) < string(s2)` this **does not** mean `hash(s1) < hash(s2)`. This is very important to realize because `jluna::Symbol`s other main functionality is being constant-time *comparable*. This comparison is not lexicographical, rather the hashes of both symbols are compared as numbers. +In most cases, it is impossible to predict which number will be assigned to which symbol. This also means for Symbols `s1` and `s2` if `string(s1) < string(s2)` this **does not** mean `hash(s1) < hash(s2)`. This is very important to realize because `jluna::Symbol`s other main functionality is being constant-time *comparable*. This comparison is not lexicographical, rather, comparing two symbols means to compare their corresponding hashes. Symbol proxies provide the following comparison operators: @@ -976,13 +980,13 @@ def (14299692412389864439) abc (16076289990349425027) ``` -We see that, lexicographically, the symbol are out of order. They are, however, ordered properly according to their hashes. +We see that, lexicographically, the symbols are out of order. They are, however, ordered properly according to their hashes. --- ## Functions -Functions in julia are movable, reassignable objects just like any other type. In C++, raw functions are usually handled separately and while object wrappers exist, they are very different from julia functions. `jluna` aims to bridge this gap through its own function interface. +Functions in julia are movable, reassignable objects just like any other type. In C++, raw functions are usually handled separately, and while object wrappers exist, they are very different from julia functions. `jluna` aims to bridge this gap through its own function interface. ### Calling julia Functions from C++ @@ -1014,7 +1018,7 @@ println.call(/* ... */); Base["println"](Base["typeof"](std::set>>>({{{1, {1, {1, 1, 1}}}}}))); ``` ``` -Set{IdDict{UInt64, Pair{UInt64, Vector{Int32}}}} +Set{Dict{UInt64, Pair{UInt64, Vector{Int32}}}} ``` As mentioned before, any boxable type, including proxies themself, can be used as arguments directly without manually having to call `box` or `unbox`. If an argument is not (un)boxable, a compile-time error is thrown. This makes `jluna`s way of calling functions much, much safer than the C-APIs `jl_call`. @@ -1086,7 +1090,7 @@ State::eval(R"( julia prints: [3, 4, 5, 6] ``` -It may seem very limiting to only be able to call two kinds of function signatures and having to register them first everytime, however through clever programming, this system actually allows us to call any C++ function - with no restrictions. +It may seem very limiting to only be able to call two kinds of function signatures and having to register them first everytime, however, through clever design, this system actually allows us to call any C++ function - without exception. #### Allowed Function Names @@ -1123,13 +1127,13 @@ Only the following signatures for lambdas are allowed (this is enforced at compi (Any*, Any*) -> Any* (Any*, Any*, Any*) -> Any* (Any*, Any*, Any*, Any*) -> Any* -``` +``` -Templated lambdas will be supported in a future version but are currently disallowed as of `jluna v0.5`. The correct signature is enforced at compile time. +Templated lambdas will be supported in a future version but are currently disallowed as of `jluna v0.7`. The correct signature is enforced at compile time. #### Using Non-julia Objects in Functions -While this may seem limiting at first, as stated, it is not. We are restricted to `Any*` for **arguments**, however we can instead access arbitrary C++ objects by reference or by value through **captures** which are unrestricted: +While this may seem limiting at first, as stated, it is not. We are restricted to `Any*` for **arguments**, however we can instead access arbitrary C++ objects by reference or by value through **captures**, which are unrestricted: ```cpp // a C++-object, incompatible with julia @@ -1158,7 +1162,7 @@ State::safe_eval("cppcall(:call_object, 456)"); object called 456 ``` -This makes it possible to call C++-only static and non-static member function and modify C++-side objects through their member functions. Through this capturing by reference, we can for example hand a julia-side object to a registered function, which then sets a member of a purely C++-side object. We could also capture other functions and other lambdas or even proxies, the possibilities are endless, hence why this system is able to call arbitrary C++ code. +This makes it possible to call C++-only, static and non-static member function and modify C++-side objects using them. We could capture more objects, other functions and other lambdas or even proxies, the possibilities are endless, hence why this system is able to call arbitrary C++ code. #### Calling Templated Lambda Functions @@ -1228,7 +1232,7 @@ Main.jluna._cppcall.UnnamedFunctionProxy( ) ``` -We see that it is an unnamed function proxy with internal id `#1`. This type of object is callable, exactly like the lambda is C++ side, so we can use it just like any other function: +We see that it is an unnamed function proxy with internal id `#1`. This type of object is callable, exactly like the lambda is C++-side, so we can use it just like any other function: ```cpp State::eval("Main.lambda()"); @@ -1237,17 +1241,17 @@ State::eval("Main.lambda()"); cpp called ``` -For lambdas with different signature, they would of course expect 1, 2, etc. many arguments. The correct number of arguments is enforced at runtime. +For lambdas with different signature, they would of course expect 1, 2, etc. many arguments. This exact same process of assigning lambdas to julia-side variables also works in module scope using `jluna::Module.assign` which makes it possible to construct entire modules consisting of only c++-side functionality. ## Arrays -julia-side objects of type `T <: AbstractArray` that are rectangular (or more generally, n-dimensional orthotopic) are represented C++-side by their own proxy type: `jluna::Array`. Just like in julia, `T` is the value_type and `R` the rank (or dimensionality) of the array.
+Julia-side objects of type `T <: AbstractArray` that are rectangular (or, more generally, n-dimensional orthotopic) are represented C++-side by their own proxy type: `jluna::Array`. Just like in julia, `T` is the value-type and `R` the rank (or dimensionality) of the array.
julia array types and `jluna` array types correspond to each other like so: -| julia | jluna v0.5 | jluna v0.6+ | +| julia | jluna v0.7 | jluna v0.8+ | |----------------|---------------|---------------| | `Vector{T}` | `Vector` | `Vector` | | `Matrix{T}` | `Array` |`Matrix` | @@ -1269,16 +1273,13 @@ jluna::Array unnamed = State::eval("return array"); // named array proxy jluna::Array named = Main["array"]; ``` -Note that instead of `auto`, we declared `unnamed` and `named` to be explicitly of type `jluna::Array`. This declaration induces a conversion from `jluna::Proxy` (the super class) to `jluna::Array` (the child class). A functonally equivalent way to do this would be: +Note that instead of `auto`, we declared `unnamed` and `named` to be explicitly of type `jluna::Array`. This declaration triggers a conversion from `jluna::Proxy` to `jluna::Array`. Just like with `Module` and `Symbol`, we can also explicitely cast a proxy to an array: ```cpp auto array = State::eval("return array").as(); ``` -However, the latter is discouraged for style reasons. - - -We can use the generic value type `Any` to make it possible for the array proxy to attach any julia-side array, regardless of value type. `jluna` provides 3 convenient typedefs for this: +We can use the generic value type `Any*` to make it possible for the array proxy to attach any julia-side array, regardless of value type. `jluna` provides 3 convenient typedefs for this: ```cpp using Array1d = Array; @@ -1320,7 +1321,7 @@ before [1 4 7; 2 5 8; 3 6 9;;; 10 13 16; 11 14 17; 12 15 18;;; 19 22 25; 20 23 2 after [1 4 7; 2 5 8; 3 6 9;;; 10 9999 16; 11 14 17; 12 15 18;;; 19 9999 25; 20 23 26; 21 24 27] ``` -While it may be easy to remember to use 0 for `jluna` objects, make sure to be aware of wether you are calling a C++-side function or a julia-side function: +While it may be easy to remember to use 0 for `jluna` objects, make sure to be aware of whether you are calling a C++-side function, or a julia-side function: ```cpp State::eval("array = collect(1:9)"); @@ -1339,6 +1340,18 @@ julia: 3 Function proxies that call julia-side functions still expect 1-based indices. +#### Julia-Style List Indexing + +While `jluna` doesn't yet offer a list-comprehension engine as nice as that of julia, `jluna::Array` does allow for julia-style indexing using a collection: + +```cpp +auto sub_array = array.at({2, 13, 1}); // any iterable collection can be used +Base["println"](sub_array) +``` +``` +[3, 14, 2] +``` + #### 0-based vs 1-based To further illustrate the relationship between indexing in `jluna` and indexing in julia, consider the following table (where `M` is a N-dimensional array) @@ -1349,21 +1362,11 @@ To further illustrate the relationship between indexing in `jluna` and indexing | 2 | `M[1, 2]` | `M.at(0, 1)`| | 3 | `M[1, 2, 3]` | `M.at(0, 1)`| | Any | `M[ [1, 13, 7] ]`| `M[ {0, 12, 6} ]` | +| Any | `M[i for i in 1:10]` | `M["i for i in 1:10"_gen]` | | | | | * | `M[:]` | not available | - -#### Julia-Style List Indexing - -While `jluna` doesn't yet offer a list-comprehension engine as nice as that of julia, `jluna::Array` does allow for julia-style indexing using a collection: - -```cpp -auto sub_array = array.at({2, 13, 1}); // any iterable collection can be used -Base["println"](sub_array) -``` -``` -[3, 14, 2] -``` +We will learn more C++-side generator expressions in the [section on vector](#vectors). ### Iterating @@ -1387,7 +1390,7 @@ for (size_t i : array) 6 ``` -We can create an assignable iterator by doing the following (note the use of `auto` instead of `auto&`) +Iterators are assignable, doing the following (note the use of `auto` instead of `auto&`) ```cpp Array array = State::eval("return [1:2 3:4 5:6]"); @@ -1403,11 +1406,11 @@ before: [1 3 5; 2 4 6] after : [2 4 6; 3 5 7] ``` -Here, `auto` is deduced to a special iterator type that basically acts like a regular `jluna::Proxy` (for example, we need to manually cast it to `size_t` in the above example) but with faster, no-overhead read/write-access to the array data compared to an actual proxies accessed via `Proxy::operator[]`. +Here, `auto` is deduced to a special iterator type that basically acts like a regular `jluna::Proxy` (for example, we need to manually unbox it to `size_t`), but with faster, no-overhead read/write-access to the array data. This enhanced speed is the main difference between using `Array::operator[]` and `Proxy::operator[]`. ### Vectors -Just like in julia, `jluna` vectors are array, however their 1-dimensionality gives them access to additional functions: +Just like in julia, `jluna` vectors 1-dimensional arrays that offer a few functions in addition to what N-dimensional arrays have available: ```cpp // construct from std::vector @@ -1435,15 +1438,13 @@ Base["println"](vector); [10000, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 10000] ``` -Note that `Array::operator[](Range_t&&)` (linear indexing with a range) always returns a 1-rank vector of the corresponding value type, regardless of the original arrays rank. - ## Matrices (this feature is not yet implemented, simply use `Array` until then) ## Generator Expressions -One of julias most convenient features are [**generator expressions**](https://docs.julialang.org/en/v1/manual/arrays/#man-comprehensions) (also called list/array comprehensions). These are is a special kind of syntax that creates an iterable range. For example, the following expressions are equivalent: +One of julias most convenient features are [**generator expressions**](https://docs.julialang.org/en/v1/manual/arrays/#man-comprehensions) (also called list- or array-comprehensions). These are is a special kind of syntax that creates an iterable, in-line, lazy-eval range. For example: ```julia # comprehension @@ -1459,9 +1460,9 @@ for i in 1:10 end ``` -In julia, we use `[]` when we want the expression to be vectorized and `()` when we want the expression to stay an object of type `Base.Generator`. The latter has the significant advantage that iterating and accessing its values is lazy-eval, meaning that we only do the any computation only when actually accessing the value. +In julia, we use `[]` when we want the expression to be vectorized, and `()` when we want the expression to stay an object of type `Base.Generator`. The latter has the significant advantage that iterating and accessing its values is *lazy-eval*, meaning that we only do the any computation only when actually accessing the value. -In `jluna`, we can create a generator expression using the postfix literal operator `_gen` on a c-string: +In `jluna`, we can create a generator expression using the postfix literal operator `_gen` on a C-string: ```cpp // in julia: @@ -1470,9 +1471,9 @@ In `jluna`, we can create a generator expression using the postfix literal opera // in cpp: "(i for i in 1:10 if i % 2 == 0)"_gen ``` -Note that when using `_gen`, **only round brackets are allowed**. Every generator expression has to be in round brackets. +Note that when using `_gen`, **only round brackets are allowed**. Every generator expression has to be in round brackets, they cannot be ommitted or a syntax error will be triggered. -We can iterate a generator expression like so: +We can iterate through generator expression like so: ```cpp for (auto i : "(i for i in 1:10 if i % 2 == 0)"_gen) @@ -1486,7 +1487,7 @@ for (auto i : "(i for i in 1:10 if i % 2 == 0)"_gen) 10 ``` -Where `i` needs to be unboxed manually and behaves exactly like an `Any*`. While this is convenient, the true power of generator expressions comes in its usaged along with `jluna::Vector`. We can: +Where `i` needs to be unboxed manually and behaves exactly like an `Any*`. While this is convenient, the true power of generator expressions lies in its interfacing with `jluna::Vector`. We can: ```cpp // ...initialize a vector from a generator expression @@ -1496,12 +1497,14 @@ auto vec = Vector("i*i for i in 1:99"_gen); vec["i for i in 1:99 if i < 50"_gen]; ``` -Using generator expressions like this, we can imitate julia syntax very closely, despite being in a language that does not have list comprehension. +This imitates julia syntax very closely, despite being in a language that does not have list comprehension. + +--- ## Introspection -The main advantage julia has over C++ is its introspection and meta-level features. -While introspection is technically possible in C++, it can be quite cumbersome and complicated. In julia, things like deducing the signature of a function, or classifying types, is fairly straight-forward and accessible, even to a novice programmers. `jluna` aims to take advantage of this by giving a direct interface to that part of julias toolset: `jluna::Type`. +The main advantage julia has over C++ are its introspection and meta-level features. +While introspection is technically possible in C++, it can be quite cumbersome and complicated. In julia, things like deducing the signature of a function or classifying type, is fairly straight-forward and accessible, even to novice programmers. `jluna` aims to take advantage of this by giving a direct interface to that part of julias toolset: `jluna::Type`. ### Type Proxies @@ -1510,7 +1513,7 @@ We've seen specialized module-, symbol- and array-proxies. `jluna` currently has There are multiple ways to construct a type proxy: ```cpp -// get type of proxy +// get type of other proxy auto general_proxy = State::eval("return " + /* ... */); auto type = general_proxy.get_type(); @@ -1530,7 +1533,7 @@ Where the latter uses `to_julia_type` to deduce which julia-side type to cons ### Core Types -For convenience, `jluna` offers most of the types in `Core` and `Base` as pre-initialized global constants, similar to how the modules `Main`, `Base` and `Core` are available as module-proxies after initialization.

+For convenience, `jluna` offers most of the types in `Core` and `Base` as pre-initialized global constants, similar to how the modules `Main`, `Base`, and `Core` are available after initialization.

The following types can be accessed this way: | `jluna` constant name | julia-side name| @@ -1625,7 +1628,7 @@ bool operator==(const Type& other) const; bool operator!=(const Type& other) const; ``` -This ordering becomes relevant when talking about `TypeVar`s. TypeVars can be thought of as a sub-graph of the type-graph that has an upper and lower bound. An unrestricted types upper bound is `Any` while it's lower bound is `Union{}`. A declaration like `T where {T <: Integer}` restricts the upper bound to `Integer`, though any type that is "lower" along the sub-graph originating at `Integer` can still bind to `T`. This is important to keep in mind. +This ordering becomes relevant when talking about `TypeVar`s. TypeVars can be thought of as a sub-graph of the type-graph that has an upper and lower bound. An unrestricted types upper bound is `Any`, while its lower bound is `Union{}`. A declaration like `T where {T <: Integer}` restricts the upper bound to `Integer`, though any type that is "lower" along the sub-graph originating at `Integer` can still bind to `T`. This is important to keep in mind. ### Type Info @@ -1651,7 +1654,7 @@ This type is a parametric type, it has two **parameters** called `T` and `U`. `T + `_field2` which is declared as type `T`, thus it's upper bound is also `Integer` + `_field3` which is declared as type `U`, however because `U` is unrestricted, `_field3` is too -The type has 1 **method**: `MyType(::T, ::U)` which is its constructor. +The type has 1 **method**, `MyType(::T, ::U)`, which is its constructor. This type furthermore has the following properties: @@ -1694,7 +1697,7 @@ U => Any + `.first` is a symbol that is the name of the corresponding parameter + `.second` is the parameter upper type bound. -In case of `T`, the upper bound `Base.Integer` because we restricted it as such in the declaration. For `U`, there is no restriction which is why its upper bound is the default: `Any`. +In case of `T`, the upper bound is `Base.Integer`, because we restricted it as such in the declaration. For `U`, there is no restriction, which is why its upper bound is the default: `Any`. We can retrieve the number of parameters directly using `get_n_parameters()`. This saves allocating the vector of pairs. @@ -1738,7 +1741,7 @@ We, again, get a vector of pairs where `.first` is the name, `.second` is the up ### Type Classification -To classify a type means to evaluate a condition based on a types attributes in order to get information about how similiar or different clusters of types are. `jluna` offers a number of convenient classifications, some of which are available as part of the julia core library, some of which are not. This section will list all, along with their meaning: +To classify a type means to evaluate a condition based on a types attributes, in order to get information about how similiar or different clusters of types are. `jluna` offers a number of convenient classifications, some of which are available as part of the julia core library, some of which are not. This section will list all, along with their meaning: + `is_primitive`: was the type declared using the keyword `primitive` ```cpp @@ -1795,7 +1798,7 @@ end println(is_array_type(Base.Array)) ``` -Some may expect this to print `true`, however this is not the case. `Base.Array` is a parametric type, because of that `typeof(Array)` is actually a `UnionAll` which does not have a property `:name`: +Some may expect this to print `true`, however this is not the case. `Base.Array` is a parametric type, because of that, `typeof(Array)` is actually a `UnionAll` which does not have a property `:name`: ```julia println(propertynames(Array)) @@ -1819,7 +1822,7 @@ Array{T, N} (:name, :super, :parameters, :types, :instance, :layout, :size, :hash, :flags) ``` -Once fully unrolled, we have access to the properties necessary for introspect. `jluna` does this unrolling automatically, meaning all parametric types held by a `jluna::Type` proxy are fully specialized. We can fully specialize a type using the member function `.unroll()`. +Once fully unrolled, we have access to the properties necessary for introspect. `jluna` does this unrolling automatically, meaning all parametric types held by a `jluna::Type` proxy are fully specialized. We can fully specialize a type manually using the member function `.unroll()`. --- @@ -1837,47 +1840,77 @@ Once fully unrolled, we have access to the properties necessary for introspect. ## Performance -This section will give some tips on how to achieve the best performance in `jluna`. As of release 0.7, `jluna` went through extensive optimization to minimize the amount of overhead as much as possible. Still, when compared to pure julia, `jluna` will always loose. Comparing `jluna` to the C-library however, `jluna` does much better. +This section will give some tips on how to achieve the best performance using `jluna`. As of release 0.7, `jluna` went through extensive optimization, minimizing overhead as much as possible. Still, when compared to pure julia, `jluna` will always be slower. Comparing `jluna` to the C-library though is a much fairer comparison and in that aspect it can do quite well, though it is important to be aware of how to avoid overhead and when it is worth to stay with the C-library. + +The following suggestions are good to keep in mind when writing performance-critical code: ### Stay Julia-Side -The most important tip is to do as much of the taxing computation as possible julia-side. Julia is a race-car of a language and unless our C++-library is similarly optimized, it is usually a better idea to do things with julia. To do this, we write performance critical code in a `.jl` file, then call that file from C++. Recall that proxies do not actually own any C++-side memory, thuse calling julia functions on proxies actually triggers no significant computation C++-side. Taxing work is only being done once the proxies value changes from one languages state to the other. +All performance critical code should be inside a julia file. All C++ should do, is to manage data and dispatch the julia-side function. For example, let's say we are working on very big matrices. Any matrix operation should happen inside a julia function, and the actual data of the matrix should stay julia-side as much as possible. We can still control julia from C++, `State::eval` has almost no overhead. Similarly, `jluna::Proxy` is just a stand-in for julia-side data. Thus, using julia functions of `jluna::Proxy`s incurs very little overhead compared to doing both only in julia. + +### Minimize (Un)Boxing -### Avoid Changing States +By far the easiest way to completely tank a programs' performance is to unnecessarily box/unbox values constantly. If our program requires operation `A` , `B`, `C` julia-side and `X`, `Y`, `Z` C++-side, it's very important to execute everything in a way that minimizes the number of box/unbox calls. So: +`ABC XYZ ` rather than `A X B Y `, etc.. This may seem obvious when abstracted like this but in a complex application, it's easy to forget. Knowing what calls cause box/unboxing is essential to avoiding pitfalls. -By far the easiest way to completely tank a programs performance is to unnecessarily box/unbox values constantly. If our program requires operation A, B, C julia side and X, Y, Z C++ side, it's very important to execute everything like: `ABC XYZ ` rather than `A X B Y `, etc.. Furthermore it's important to be aware of what we're boxing. If we were to give each box/unbox call a grade from 1 - 5 where 1 is 0 overhead and 5 is the most amount of overhead, box/unbox calls would be graded like so: +Of course, we can't completely avoid it, either. To illustrate which box/unbox calls should be used as little as possible and which are perfectly fine, we will grade them in terms of how well they perform (where `A+` is "0 overhead" and `F` is "do not use this in performance critical code"): ``` -// type // grade, 1 ist fastest, 5 is slowest -int, size_t, float, etc 1 -const char* 1 -Proxy 2 -vector 2 -Pair 3 -Set 3 -Tuple 4 -(unordered) map 4 -lambda 5 +// type T of box/unbox // grade + +int, size_t, float, etc A+ +const char* A+ +Proxy A +vector A +std::string A +Pair B +Set B- +Tuple C +(unordered) map D +lambda D- ``` -It's easy to remember these grades by simply considering, how different an object C++-side and julia-side representations are. Obviously to wrap a C+-side lambda into a julia-side object, a lot of things have to happen behind the scenes while a vector pretty much transfers directly between languages. +Vectors and primitives can be directly swapped between language, which explains their excellent grade, while more complex objects need to be serialized or wrapped before being transferable. ### Minimize Proxy Construction -The main overhead incurred by `jluna`s safety is that of proxy allocation. Anytime a proxy is constructed, its value and name have to be added to an internal state that protects them from the garbage collector. Consider the following: +The main overhead incurred by `jluna` is that of safety. Anytime a proxy is constructed, its value and name have to be added to an internal state that protects them from the garbage collector. This takes some amount of time; internally, the value is wrapped and a reference to it and its name is stored in a dictionary. Neither of these is that expensive but when a function uses hundreds of proxies over its runtime, this can add up quickly. Consider the following: ```cpp -/// constructs 5 proxies -auto a = Main["Module1"]["Module2"][vector_var][1]["field"]; - -/// constructs 1 proxy -auto b = State::safe_script("return Main.Module1.Module2.vector_var[1].field"); +auto a = Main["Module1"]["Module2"]["vector_var"][1]["field"]; ``` -It is no exaggeration that the second call is about 5 times faster. What we loose for that speedup is the convenience of being able to mutate the julia variable, as proxy `b` is unnamed. +This statement constructs 5 proxies, it is exactly equivalent to: + +```cpp +Proxy a; +{ + auto _0 = Main; + auto _1 = _0["Module1"]; + auto _2 = _1["Module2"]; + auto _3 = _2["vector_var"]; + auto _4 = _3[1]; + a = _4["field"] +} +``` + +Where each declaration triggers a proxy to be created. At the end of the block, all no proxies are deallocated because the value of a depends on it's host, `vector_var`. + +If we instead access the value like so: + +```cpp +size_t a = State::safe_return("Main.Module1.Module2.vector_var[1].field") +``` + +We do not create any proxy at all, increasing performance by up to 6 times. Of course, doing this, we loose the convenience of being assignable, castable, and all other functionalities a named `jluna::Proxy` offers. Still, in performance-critical code, unnamed proxies are almost always faster than named proxies and should be preferred. ### Use the C-Library -When performance needs to be optimal, that is not *good*, not *extremely good* but the absolute best it could possible be, we might need to resort to the C-library.Values are hard to manage, the syntax is very clunky and the garbage collector will probably steal many of our values from under our nose when we're not looking and segfault the program, but one thing C has is optimal speed. Luckily, the C-library and `jluna` are freely mixable, though remember to `.update` any proxies whos julia-side value was modified by the C-library. +When performance needs to be optimal, not "good" or "excellent, but mathematically optimal, it is sometimes necessary to resort to the C-library. Values are hard to manage, the syntax is very clunky and the garbage collector will probably steal many of our values from under our nose and segfault the program, but that is the trade-off of performance vs convenience.
+Luckily the C-library and `jluna` are freely mixable, though we may need to `.update` any proxies whos julia-side value was modified outside `jluna`. + +### In Summary + +`jluna`s safety features incur an unavoidable overhead. Great care has been taken to minimize this overhead as much as possible, but it is still there. Knowing this, `jluna` is still perfectly fine for most applications. If a part of a library truly is performance critical, however, it may be necessary to avoid using `jluna` as much as possible in order for julia to do, what it's best at: being very fast. When mixing C++ and Julia, though, `jluna` does equally well at bridging that gap, and in many ways it does so better than the C-API. --- \ No newline at end of file From 7a38499920bd14dd3c807738994e4fe7e858678a Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 21:41:28 +0100 Subject: [PATCH 17/19] polish --- .benchmark/main.cpp | 3 +++ docs/installation.md | 22 +++++++++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.benchmark/main.cpp b/.benchmark/main.cpp index ccfad58..4cec0ae 100644 --- a/.benchmark/main.cpp +++ b/.benchmark/main.cpp @@ -65,6 +65,9 @@ void benchmark_lambda_call() int main() { + std::cerr << "This executable is not intended for end-users and should not be used. To verify jluna works, instead execute \"./JLUNA_TEST\"" << std::endl; + return 1; + using namespace jluna; State::initialize(); diff --git a/docs/installation.md b/docs/installation.md index cd9eb86..9113ccd 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,6 +1,6 @@ # Creating a Project with jluna -The following is a step-by-step guide to creating an application using `jluna` from scratch +The following is a step-by-step guide to creating an application using `jluna` from scratch. ### Table of Contents 1. [Creating the Project](#creating-the-project) @@ -58,7 +58,7 @@ And copy-paste the resulting output to assign `JULIA_PATH` in bash like so: export JULIA_PATH=/path/to/your/julia/bin/.. ``` -We can now continue to compiling `jluna` using cmake, being sure to stay in the same bash session where we set `JULIA_PATH`. To set `JULIA_PATH` globally, consider consulting [this guide](https://unix.stackexchange.com/questions/117467/how-to-permanently-set-environmental-variables). That way we don't have to re-set `JULIA_PATH` anytime we log out. +We can now continue to compiling `jluna` using cmake, being sure to stay in the same bash session where we set `JULIA_PATH`. To set `JULIA_PATH` globally, consider consulting [this guide](https://unix.stackexchange.com/questions/117467/how-to-permanently-set-environmental-variables). This way, it does not have to re-set anytime a bash session ends. ### Building `jluna` @@ -72,7 +72,9 @@ cmake -D CMAKE_CXX_COMPILER=g++-11 .. # or clang-12 make ``` -Warnings of the following type may appear: +You may need to specify the absolute path to `g++-11` / `clang-12` if their respective executables are not available in the default search paths. + +When compiling, warnings of the following type may appear: ``` (...) @@ -83,6 +85,8 @@ Warnings of the following type may appear: This is because the official julia header `julia.h` is slightly out of date. The warning is unrelated to `jluna`s codebase, `jluna` itself should report no warnings or errors. If this is not the case, head to [troubleshooting](#troubleshooting). +> `jluna` is being developed using g++-11. Because of this, any specific release may issue warnings when using a different compiler. Compilation should never fail with an error, regardless of which compiler is used by the end-user. + We verify everything works by running `JLUNA_TEST` which we just compiled: ```bash @@ -231,6 +235,8 @@ hello julia `State::initialize()` may fail. If this is the case, head to [troubleshooting](#troubleshooting). Otherwise, congratulations! We are done and can now start developing our own application with the aid of julia and `jluna`. +--- + ## Troubleshooting ### Cannot determine location of julia image @@ -258,10 +264,6 @@ This is because `JULIA_PATH` is not properly set. Repeat the section on [setting When building `jluna`, the following warnings may appear: -``` -CMake Warning at CMakeLists.txt:37 (message): - Cannot find library header julia.h in (...) -``` ``` CMake Warning at CMakeLists.txt:37 (message): Cannot find library header julia.h in (...) @@ -312,7 +314,7 @@ fatal error: jluna.hpp: No such file or directory compilation terminated. ``` -This means the the `include_directories` in cmake where set improperly. Make sure the following lines are present in your `CMakeLists.txt`: +This means the `include_directories` in cmake where set improperly. Make sure the following lines are present in your `CMakeLists.txt`: ``` include_directories("${CMAKE_SOURCE_DIR}/jluna") @@ -375,7 +377,7 @@ If C is true, replace `State::initialize()` with its overload: ```cpp State::initialize("/path/to/(...)/julia/bin") ``` -Where `/path/to/(...)` is replaced with the absolute path to your image of julia. This will tell the C-API to load julia from that image, rather from the default system image. +Where `/path/to/(...)` is replaced with the absolute path to your image of julia. This will tell the C-API to load julia from that image, rather than from the default system image. ## Creating an Issue @@ -404,3 +406,5 @@ If and only if all of the above apply, head to the [issues tab](https://github.c + provide a minimum working example of your project that recreates the bug We will try to resolve your problem as soon as possible and are thankful for your input and collaboration. + +--- \ No newline at end of file From 45558697e317035f6e2ad03e23a8b0ed65984baf Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 21:57:04 +0100 Subject: [PATCH 18/19] polish --- .src/array_iterator.inl | 7 +++++++ docs/installation.md | 2 +- include/array.hpp | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.src/array_iterator.inl b/.src/array_iterator.inl index e2392c1..a097fb4 100644 --- a/.src/array_iterator.inl +++ b/.src/array_iterator.inl @@ -106,4 +106,11 @@ namespace jluna return *this; } + + template + template, bool>> + Array::Iterator::operator T() const + { + return ConstIterator::operator T(); + } } \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index 9113ccd..ad90d93 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -85,7 +85,7 @@ When compiling, warnings of the following type may appear: This is because the official julia header `julia.h` is slightly out of date. The warning is unrelated to `jluna`s codebase, `jluna` itself should report no warnings or errors. If this is not the case, head to [troubleshooting](#troubleshooting). -> `jluna` is being developed using g++-11. Because of this, any specific release may issue warnings when using a different compiler. Compilation should never fail with an error, regardless of which compiler is used by the end-user. +> `jluna` is being developed using g++-11. Because of this, any specific release may issue warnings when using a different compiler. Compilation should never fail with an error, regardless which of the supported compiler is used by the end-user. We verify everything works by running `JLUNA_TEST` which we just compiled: diff --git a/include/array.hpp b/include/array.hpp index 37e2164..d6a89b4 100644 --- a/include/array.hpp +++ b/include/array.hpp @@ -208,6 +208,11 @@ namespace jluna template auto& operator=(T value); + /// @brief decay into unboxed value + /// @tparam value-type, not necessarily the same as declared in the array type + template, bool> = true> + operator T() const; + protected: using ConstIterator::_owner; using ConstIterator::_index; From 73e862d385c7bf05d25ca3828d8d181aa13c3b50 Mon Sep 17 00:00:00 2001 From: clem Date: Sun, 20 Feb 2022 22:42:50 +0100 Subject: [PATCH 19/19] changed to release --- CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b140db..96cb174 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.16) project(jluna) set(CMAKE_CXX_STANDARD 20) -set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_BUILD_TYPE Release) # compiler support if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION STRGREATER_EQUAL "12.0.0") @@ -142,8 +142,8 @@ add_executable(JLUNA_TEST ) target_link_libraries(JLUNA_TEST jluna ${JULIA_LIB}) -add_executable(JLUNA_BENCHMARK - .benchmark/main.cpp - .benchmark/benchmark.hpp -) -target_link_libraries(JLUNA_BENCHMARK jluna ${JULIA_LIB}) +#add_executable(JLUNA_BENCHMARK +# .benchmark/main.cpp +# .benchmark/benchmark.hpp +#) +#target_link_libraries(JLUNA_BENCHMARK jluna ${JULIA_LIB})