diff --git a/Cargo.toml b/Cargo.toml
index f298946d6..834b38ab2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -138,7 +138,7 @@ wasmer = { version = "=5.0.0-rc.1", path = "lib/api", features = [
 	"sys",
 ] }
 anyhow = "1.0"
-criterion = { version = "0.5", default-features = false }
+criterion = { version = "0.5", features = ["csv_output"] }
 clap = { version = "=4.4.11" }
 clap_builder = { version = "=4.4.11" }
 clap_derive = { version = "=4.4.7" }
@@ -229,6 +229,10 @@ opt-level = 3
 [profile.dev.package.digest]
 opt-level = 3
 
+[[bench]]
+name = "compile_run"
+harness = false
+
 [[bench]]
 name = "deserialize_modules"
 harness = false
diff --git a/benches/compiler_run.rs b/benches/compiler_run.rs
new file mode 100644
index 000000000..52b6c1337
--- /dev/null
+++ b/benches/compiler_run.rs
@@ -0,0 +1,77 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use wasmer::*;
+
+static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-53a226d993e144159d6f8b993fe0cbf3.r2.dev";
+
+fn get_engine() -> Engine {
+    #[cfg(feature = "llvm")]
+    return LLVM::new().into();
+
+    #[cfg(feature = "singlepass")]
+    return Singlepass::new().into();
+
+    #[cfg(feature = "cranelift")]
+    return Cranelift::new().into();
+
+    #[cfg(not(any(feature = "cranelift", feature = "llvm", feature = "singlepass")))]
+    return Default::default();
+}
+
+pub fn run_fn(c: &mut Criterion, module: &[u8], name: &str, input: i64) {
+    c.bench_function(name, |b| {
+        let engine = get_engine();
+        let mut store = Store::new(engine);
+        let module = Module::new(&store, module).unwrap();
+        let import_object = imports! {};
+        let instance = Instance::new(&mut store, &module, &import_object).unwrap();
+        let func = instance
+            .exports
+            .get_typed_function::<i64, i64>(&store, "run")
+            .unwrap();
+
+        b.iter(|| {
+            func.call(&mut store, input);
+        })
+    });
+}
+
+pub fn download_and_run(c: &mut Criterion) {
+    let name = if cfg!(feature = "cranelift") {
+        "cranelift"
+    } else if cfg!(feature = "llvm") {
+        "llvm"
+    } else if cfg!(feature = "singlepass") {
+        "singlepass"
+    } else if cfg!(feature = "v8") {
+        "v8"
+    } else if cfg!(feature = "wamr") {
+        "wamr"
+    } else if cfg!(feature = "wasmi") {
+        "wasmi"
+    } else {
+        panic!("Unrecognized backend!")
+    };
+
+    let bytes = include_bytes!("./mods/counter.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/counter"), 5_000_000);
+    let bytes = include_bytes!("./mods/primes.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/primes"), 1_000);
+    let bytes = include_bytes!("./mods/fib_rec.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/fib_rec"), 40);
+    let bytes = include_bytes!("./mods/fib_iter.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/fib_iter"), 2_000_000);
+    let bytes = include_bytes!("./mods/bulk_ops.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/bulk_ops"), 5_000);
+    let bytes = include_bytes!("./mods/matmul.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/matmul"), 200);
+    let bytes = include_bytes!("./mods/argon2.wasm");
+    run_fn(c, bytes, &format!("exec/{name}/argon2"), 1);
+}
+
+criterion_group!(
+    name = run_benches;
+    config = Criterion::default().sample_size(60);
+    targets = download_and_run
+);
+
+criterion_main!(run_benches);
diff --git a/benches/mods/argon2.wasm b/benches/mods/argon2.wasm
new file mode 100644
index 000000000..704cd488c
Binary files /dev/null and b/benches/mods/argon2.wasm differ
diff --git a/benches/mods/bulk_ops.wasm b/benches/mods/bulk_ops.wasm
new file mode 100644
index 000000000..fe27e5600
Binary files /dev/null and b/benches/mods/bulk_ops.wasm differ
diff --git a/benches/mods/counter.wasm b/benches/mods/counter.wasm
new file mode 100644
index 000000000..6551b6de6
Binary files /dev/null and b/benches/mods/counter.wasm differ
diff --git a/benches/mods/fib_iter.wasm b/benches/mods/fib_iter.wasm
new file mode 100644
index 000000000..9526bb1c2
Binary files /dev/null and b/benches/mods/fib_iter.wasm differ
diff --git a/benches/mods/fib_rec.wasm b/benches/mods/fib_rec.wasm
new file mode 100644
index 000000000..75d0af323
Binary files /dev/null and b/benches/mods/fib_rec.wasm differ
diff --git a/benches/mods/matmul.wasm b/benches/mods/matmul.wasm
new file mode 100644
index 000000000..3a51f2c9a
Binary files /dev/null and b/benches/mods/matmul.wasm differ
diff --git a/benches/mods/primes.wasm b/benches/mods/primes.wasm
new file mode 100644
index 000000000..6551b6de6
Binary files /dev/null and b/benches/mods/primes.wasm differ