benches: add OCI wasi-demo-app to the benchmark

this adds wasi-demo-app OCI image to the benches

Signed-off-by: Jiaxiao (mossaka) Zhou <[email protected]>

.github/workflows/benchmarks.yml

@@ -28,6 +28,8 @@ jobs:
           make install
           make test-image
           make load
+          make test-image/oci
+          make load/oci
       - name: Run Benchmarks
         run: |
           set -o pipefail

benches/containerd-shim-benchmarks/README.md

@@ -14,6 +14,8 @@ Then, build and load the wasi-demo-app:
 ```bash
 make test-image
 make load
+make test-image/oci
+make load/oci
 ```
 
 To run all benchmarks:

benches/containerd-shim-benchmarks/benches/wasi-demo-app-benchmarks.rs

@@ -2,18 +2,27 @@ use std::process::Command;
 use std::time::{Duration, Instant};
 use criterion::{criterion_group, criterion_main, Criterion};
 
-fn run_container(runtime: &str) -> Duration {
+fn run_container(runtime: &str, oci: bool) -> Duration {
     let start = Instant::now();
 
+    let image_name = if oci {
+        "ghcr.io/containerd/runwasi/wasi-demo-oci:latest" // OCI artifact
+    } else {
+        "ghcr.io/containerd/runwasi/wasi-demo-app:latest"
+    };
+
+    let container_name = "testwasm";
+    let wasm_file = if oci { "wasi-demo-oci.wasm" } else { "wasi-demo-app.wasm" };
+
     let output = Command::new("sudo")
         .args([
             "ctr",
             "run",
             "--rm",
             &format!("--runtime=io.containerd.{}.v1", runtime),
-            "ghcr.io/containerd/runwasi/wasi-demo-app:latest",
-            "testwasm",
-            "/wasi-demo-app.wasm",
+            image_name,
+            container_name,
+            wasm_file,
             "echo",
             "hello"
         ])
@@ -33,21 +42,19 @@ fn run_container(runtime: &str) -> Duration {
 fn benchmark_startup(c: &mut Criterion) {
     let mut group = c.benchmark_group("wasi-demo-app");
 
-    group.bench_function("wasmtime", |b| {
-        b.iter(|| run_container("wasmtime"));
-    });
-
-    group.bench_function("wasmedge", |b| {
-        b.iter(|| run_container("wasmedge"));
-    });
-
-    group.bench_function("wasmer", |b| {
-        b.iter(|| run_container("wasmer"));
-    });
-
-    group.bench_function("wamr", |b| {
-        b.iter(|| run_container("wamr"));
-    });
+    const RUNTIMES: &[&str] = &["wasmtime", "wasmedge", "wasmer", "wamr"];
+
+    for runtime in RUNTIMES {
+        group.bench_function(runtime, |b| {
+            b.iter(|| run_container(runtime, false));
+        });
+    }
+    for runtime in RUNTIMES {
+        let name = format!("{}-oci", runtime);
+        group.bench_function(&name, |b| {
+            b.iter(|| run_container(runtime, true));
+        });
+    }
 
     group.finish();
 }