From c5deb005bafc3be4a9631cf27df3ccce574784a0 Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Mon, 28 Oct 2024 13:07:28 -0600 Subject: [PATCH] fix regression related to bundled libraries `componentize-py` tried to read the bundled libraries from the filesystem at runtime rather than actually bundling them. That worked fine as long as the files still existed at the same path when you ran the binary, but stopped working if you deleted them or tried to run the binary on a different machine. Signed-off-by: Joel Dice --- Cargo.lock | 2 +- Cargo.toml | 2 +- examples/cli/README.md | 4 +- examples/http/README.md | 4 +- examples/matrix-math/README.md | 4 +- examples/sandbox/README.md | 4 +- examples/tcp/README.md | 4 +- pyproject.toml | 2 +- src/prelink.rs | 111 +++++++++++++++++++++++++-------- 9 files changed, 97 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be2e25d..9f450e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -391,7 +391,7 @@ dependencies = [ [[package]] name = "componentize-py" -version = "0.15.1" +version = "0.15.2" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 64cea2a..8f1e12c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "componentize-py" -version = "0.15.1" +version = "0.15.2" edition = "2021" exclude = ["cpython"] diff --git a/examples/cli/README.md b/examples/cli/README.md index 7ae316c..ca32872 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -10,7 +10,7 @@ run a Python-based component targetting the [wasi-cli] `command` world. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.1 +* `componentize-py` 0.15.2 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -18,7 +18,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.1 +pip install componentize-py==0.15.2 ``` ## Running the demo diff --git a/examples/http/README.md b/examples/http/README.md index f7a9d91..85bdd94 100644 --- a/examples/http/README.md +++ b/examples/http/README.md @@ -10,7 +10,7 @@ run a Python-based component targetting the [wasi-http] `proxy` world. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.1 +* `componentize-py` 0.15.2 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -18,7 +18,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.1 +pip install componentize-py==0.15.2 ``` ## Running the demo diff --git a/examples/matrix-math/README.md b/examples/matrix-math/README.md index 50793db..c5c1d16 100644 --- a/examples/matrix-math/README.md +++ b/examples/matrix-math/README.md @@ -11,7 +11,7 @@ within a guest component. ## Prerequisites * `wasmtime` 26.0.0 or later -* `componentize-py` 0.15.1 +* `componentize-py` 0.15.2 * `NumPy`, built for WASI Note that we use an unofficial build of NumPy since the upstream project does @@ -23,7 +23,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.1 +pip install componentize-py==0.15.2 curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz tar xf numpy-wasi.tar.gz ``` diff --git a/examples/sandbox/README.md b/examples/sandbox/README.md index 9ab7ec8..30d9591 100644 --- a/examples/sandbox/README.md +++ b/examples/sandbox/README.md @@ -8,10 +8,10 @@ sandboxed Python code snippets from within a Python app. ## Prerequisites * `wasmtime-py` 25.0.0 or later -* `componentize-py` 0.15.1 +* `componentize-py` 0.15.2 ``` -pip install componentize-py==0.15.1 wasmtime==25.0.0 +pip install componentize-py==0.15.2 wasmtime==25.0.0 ``` ## Running the demo diff --git a/examples/tcp/README.md b/examples/tcp/README.md index 947c473..4849688 100644 --- a/examples/tcp/README.md +++ b/examples/tcp/README.md @@ -11,7 +11,7 @@ making an outbound TCP request using `wasi-sockets`. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.1 +* `componentize-py` 0.15.2 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -19,7 +19,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.1 +pip install componentize-py==0.15.2 ``` ## Running the demo diff --git a/pyproject.toml b/pyproject.toml index cf01f60..c7cc8b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ features = ["pyo3/extension-module"] [project] name = "componentize-py" -version = "0.15.1" +version = "0.15.2" description = "Tool to package Python applications as WebAssembly components" readme = "README.md" license = { file = "LICENSE" } diff --git a/src/prelink.rs b/src/prelink.rs index 4a7b8a2..345a307 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -51,17 +51,80 @@ pub fn embedded_helper_utils() -> Result { pub fn bundle_libraries( library_path: Vec<(&str, Vec)>, -) -> Result, io::Error> { +) -> Result, anyhow::Error> { let mut libraries = vec![ - library_from_so("libcomponentize_py_runtime.so")?, - library_from_so("libpython3.12.so")?, - library_from_so("libc.so")?, - library_from_so("libwasi-emulated-mman.so")?, - library_from_so("libwasi-emulated-process-clocks.so")?, - library_from_so("libwasi-emulated-getpid.so")?, - library_from_so("libwasi-emulated-signal.so")?, - library_from_so("libc++.so")?, - library_from_so("libc++abi.so")?, + Library { + name: "libcomponentize_py_runtime.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libcomponentize_py_runtime.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libpython3.12.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libpython3.12.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libc.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-mman.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libwasi-emulated-mman.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-process-clocks.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libwasi-emulated-process-clocks.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-getpid.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libwasi-emulated-getpid.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-signal.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libwasi-emulated-signal.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc++.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libc++.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc++abi.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/libc++abi.so.zst" + ))))?, + dl_openable: false, + }, ]; for (index, (path, libs)) in library_path.iter().enumerate() { @@ -76,7 +139,7 @@ pub fn bundle_libraries( libraries.push(Library { name: format!("/{index}/{path}"), - module: fs::read(library).unwrap(), + module: fs::read(library).with_context(|| library.display().to_string())?, dl_openable: true, }); } @@ -85,18 +148,6 @@ pub fn bundle_libraries( Ok(libraries) } -fn library_from_so(library_name: &str) -> Result { - let path = env!("OUT_DIR").to_owned(); - let filepath = path + "/" + library_name + ".zst"; - let bytes = fs::read(filepath)?; - - Ok(Library { - name: library_name.into(), - module: zstd::decode_all(Cursor::new(bytes))?, - dl_openable: false, - }) -} - pub fn search_for_libraries_and_configs<'a>( python_path: &'a Vec<&'a str>, module_worlds: &'a [(&'a str, &'a str)], @@ -171,15 +222,19 @@ fn search_directory( modules_seen: &mut HashSet, ) -> Result<(), anyhow::Error> { if path.is_dir() { - for entry in fs::read_dir(path)? { + for entry in fs::read_dir(path).with_context(|| path.display().to_string())? { search_directory(root, &entry?.path(), libraries, configs, modules_seen)?; } } else if let Some(name) = path.file_name().and_then(|name| name.to_str()) { if name.ends_with(NATIVE_EXTENSION_SUFFIX) { libraries.push(path.to_owned()); } else if name == "componentize-py.toml" { - let root = root.canonicalize()?; - let path = path.canonicalize()?; + let root = root + .canonicalize() + .with_context(|| root.display().to_string())?; + let path = path + .canonicalize() + .with_context(|| path.display().to_string())?; let module = module_name(&root, &path) .ok_or_else(|| anyhow!("unable to determine module name for {}", path.display()))?; @@ -224,7 +279,9 @@ fn search_directory( module, root: root.to_owned(), path: path.parent().unwrap().to_owned(), - config: toml::from_str::(&fs::read_to_string(path)?)?, + config: toml::from_str::( + &fs::read_to_string(&path).with_context(|| path.display().to_string())?, + )?, }); } }