Added Benchmarking, bin-lib split and refactorings (#1)

.github/workflows/bench.yml

@@ -0,0 +1,64 @@
+name: Benchmarks
+## Adapted from this: https://github.com/infinyon/fluvio/blob/master/.github/workflows/benchmarks.yml
+concurrency:
+  group: benchmark-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches:
+      - main
+      - dev
+  push:
+    branches:
+      - main
+      - dev
+
+permissions:
+  contents: write
+
+jobs:
+  markov_model:
+    name: Markov Model Benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Get Rust toolchain
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: nightly
+          profile: minimal
+
+      - name: Cache Rust Cargo files
+        uses: Swatinem/rust-cache@v2
+        with:
+          # Additional non workspace directories, separated by newlines
+          key: benches-${{ runner.os }}-unit_markov_model-rust
+
+      - name: Cache Benchmark data
+        uses: actions/cache@v3
+        if: github.ref == 'refs/heads/main'
+        with:
+          path: ./benches_cache
+          key: benches-${{ runner.os }}-unit_markov_model-rust
+
+      - name: Run Benchmarks
+        run: cd lib; cargo bench -- --output-format bencher | tee markov_model_bench.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        if: github.ref == 'refs/heads/main'
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'cargo'
+          # Where the output from the benchmark tool is stored
+          output-file-path: markov_model_bench.txt
+          # GitHub API token to make a commit comment
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Leave a job summary with benchmark result comparison
+          summary-always: true
+          # Where the previous data file is stored
+          external-data-json-path: ./benches_cache/benchmark-data.json
+          alert-comment-cc-users: '@Nereuxofficial'

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-
+# TODO: Add benchmarking in CI(see https://github.com/benchmark-action/github-action-benchmark/pull/138)
 jobs:
   build:
     strategy:
@@ -41,7 +41,7 @@ jobs:
         run: cargo test -- --test-threads=1
       - name: Run cargo doc
         if: ${{ runner.os == 'Linux' }}
-        run: cargo doc --no-deps --document-private-items --all-features
+        run: cargo doc --no-deps --document-private-items
 
       - name: Run build --release --all-targets
         run: cargo build --release --all-targets
@@ -62,7 +62,7 @@ jobs:
           components: rustfmt
 
       - name: Run cargo fmt --all -- --check
-        run: cargo fmt --all -- --check
+        run: cd lib && cargo fmt --all -- --check
 
   clippy:
     runs-on: ubuntu-20.04
@@ -79,5 +79,5 @@ jobs:
           override: true
           components: clippy
 
-      - name: Run cargo clippy --package rusty-FUME --all-targets
-        run: cargo clippy --package rusty-fume --all-targets
+      - name: Run cargo clippy --package lib --all-targets
+        run: cargo clippy --package lib --all-targets

.gitignore

@@ -4,3 +4,5 @@ Cargo.lock
 .vscode
 *.log
 .env
+crashes
+threads

Cargo.toml

@@ -1,30 +1,48 @@
 [package]
 name = "rusty-fume"
-version = "0.1.0"
+version = "0.8.0"
 authors = ["Nereuxofficial <[email protected]>"]
 license = "GPLv3"
 readme = "README.md"
 edition = "2021"
 
+[workspace]
+
 [dependencies]
+lib = {path = "lib"}
+# Convenient error handling
 color-eyre = "0.6.2"
+# Logging
 tracing = "0.1.37"
 tracing-subscriber = "0.3.17"
-dotenvy = "0.15.7"
+# Futures
 tokio = { version = "1.32.0", features = ["full"] }
-markov = "1.1.0"
-rand = "0.8.5"
+futures = "0.3.28"
+# Hex en/decoding
 hex = "0.4.3"
+# MQTT Packet generation
 mqtt-protocol = "0.11.2"
+# Random number generation with xoshiro for faster PRNG
+rand = "0.8.5"
 rand_xoshiro = "0.6.0"
+# Command line interface
 clap = { version = "4.3.24", features = ["derive"] }
+# For serialization
 serde = { version = "1.0.186", features = ["derive"] }
 toml = "0.7.6"
-futures = "0.3.28"
-tokio-uring = "0.4.0"
-console-subscriber = "0.1.10"
+# For serialization of raw bytes
 serde_with = {version="3.1.0", features = ["hex"]}
 
+# Tokio Console Support
+console-subscriber = "0.1.10"
 [profile.release]
 debug = true
 codegen-units = 1
+
+[features]
+default = ["tcp"]
+tcp = []
+# TODO: Add quic, ws support
+quic = []
+websocket = []
+tls = ["lib/tls"]

Readme.md

@@ -36,7 +36,7 @@ Pull requests are welcome. For major changes, please open an issue first to disc
 Currently, the Windows build is failing in the ci, however i've only tested this on Linux so far. Maybe it works on Windows, maybe it doesn't. I don't know. Pull Requests to fix this if necessary are welcome.
 
 ## Trophies
-All bugs found with this software. If you find a bug using rusty-FUME, please open an issue and i'll add it to the list.
+All bugs found with this software. If you find a bug using rusty-FUME, please open an issue and I'll add it to the list once it is patched.
 - [FlashMQ Null pointer dereference](https://github.com/halfgaar/FlashMQ/commit/eb3acf88771af3eeddf086e4c9dc51d703456eee)
 
 

lib/.gitignore

@@ -0,0 +1,6 @@
+/target
+Cargo.lock
+.idea
+.vscode
+*.log
+.env

lib/Cargo.toml

@@ -0,0 +1,60 @@
+[package]
+name = "lib"
+version = "0.8.0"
+edition = "2021"
+
+[lib]
+bench=false
+
+[dependencies]
+# Convenient error handling
+color-eyre = "0.6.2"
+# Logging
+tracing = "0.1.37"
+tracing-subscriber = "0.3.17"
+# Futures
+tokio = { version = "1.32.0", features = ["full"] }
+futures = "0.3.28"
+# Hex en/decoding
+hex = "0.4.3"
+# MQTT Packet generation
+mqtt-protocol = "0.11.2"
+# Random number generation with xoshiro for faster PRNG
+rand = "0.8.5"
+rand_xoshiro = "0.6.0"
+# Command line interface
+clap = { version = "4.3.24", features = ["derive"] }
+# For serialization
+serde = { version = "1.0.186", features = ["derive"] }
+toml = "0.7.6"
+# For serialization of raw bytes
+serde_with = {version="3.1.0", features = ["hex"]}
+
+# Tokio Console Support
+console-subscriber = "0.1.10"
+# For Websocket support
+tokio-tungstenite = "0.20.0"
+# For TLS support
+tokio-rustls = { version="0.24.1", features = ["dangerous_configuration"], optional = true }
+rustls = { version="0.21.6", features = ["dangerous_configuration"], optional = true }
+
+
+
+[dev-dependencies]
+criterion = { version = "0.5.1", features=["html_reports", "async_tokio"]}
+
+[[bench]]
+name = "markov_models"
+harness = false
+
+[profile.release]
+debug = true
+codegen-units = 1
+
+[features]
+default = ["tcp"]
+tcp = []
+# TODO: Add quic, ws support
+quic = []
+websocket = []
+tls = ["dep:tokio-rustls", "dep:rustls"]

lib/benches/markov_models.rs

@@ -0,0 +1,37 @@
+use criterion::BenchmarkId;
+use criterion::Criterion;
+use criterion::{criterion_group, criterion_main};
+use rand::SeedableRng;
+use rand_xoshiro::Xoshiro256PlusPlus;
+use std::sync::Arc;
+use tokio::io::duplex;
+// This is a struct that tells Criterion.rs to use the "futures" crate's current-thread executor
+use lib::markov::{Mode, StateMachine};
+use lib::packets::PacketQueue;
+use tokio::runtime::Runtime;
+use tokio::sync::RwLock;
+
+async fn exec_markov_fast(m: Mode) {
+    let stream = duplex(10 * 1024).0;
+    let mut machine = StateMachine::new(stream, 100);
+    let mut rng = Xoshiro256PlusPlus::from_seed([5; 32]);
+    machine
+        .execute(m, &mut rng, &Arc::new(RwLock::new(PacketQueue::default())))
+        .await;
+}
+
+fn markov_model_execution(c: &mut Criterion) {
+    for mode in [Mode::MutationGuided, Mode::GenerationGuided] {
+        c.bench_with_input(
+            BenchmarkId::new("execute_markov_model", mode),
+            &mode,
+            |b, &m| {
+                b.to_async(Runtime::new().unwrap())
+                    .iter(|| exec_markov_fast(m));
+            },
+        );
+    }
+}
+
+criterion_group!(benches, markov_model_execution);
+criterion_main!(benches);

lib/src/lib.rs

@@ -0,0 +1,58 @@
+//! # rusty-FUME
+//! rusty-FUME is a fuzzer for the MQTT protocol. It is based on [FUME-Fuzzing-MQTT Brokers](https://github.com/PBearson/FUME-Fuzzing-MQTT-Brokers)
+//! and uses markov chains to generate new packet chains. If it discovers a new response behaviour the chain is added to the fuzzing queue.
+//! We use [tokio](https://tokio.rs/) for async networking.
+//! ## The state machine
+//! We implement a State machine with a markov chain. All probabilities are configurable for this process(except the ones with only one option).
+//! The state machine is defined as follows for the Mutation Guided Fuzzing:
+//! - S0: Initial State: Either goto CONNECT state or select a packet from the queue and go to MUTATION state
+//! - CONNECT: Add connect to the current chain and go to ADDING State
+//! - ADDING: Either add a new packet(configurable probability for each one) to the chain or go to MUTATION state
+//! - MUTATION: Mutate, delete, inject or SEND the current chain
+//! - SEND: Send the current chain and either go to Sf or MUTATION state
+//! - Sf: Final State
+//! And this way for Generation Guided Fuzzing:
+//! - S0: Initial State: Goto ADD(CONNECT) state
+//! - CONNECT: Add connect to the current chain and go to S1
+//! - S1: Either add a new packet or go to S2
+//! - S2: Inject/Delete/Mutate the current chain or go to SEND
+//! - SEND: Send the current chain and either go to Sf or S2
+//! Once they get to S2 they behave the same way.
+use crate::markov::MAX_PACKETS;
+use crate::packets::PacketQueue;
+use clap::{Parser, Subcommand};
+use rand::{thread_rng, Rng};
+use serde::{Deserialize, Serialize};
+use std::str::FromStr;
+
+pub mod markov;
+pub mod mqtt;
+pub mod network;
+mod packet_pool;
+pub mod packets;
+pub mod process_monitor;
+pub mod runtime;
+// TODO: Clean up main
+// TODO: Try fuzzing a basic mongoose server?
+// TODO: Fuzz mosquitto compiled with sanitizers
+// TODO: Lib-split to allow benchmarking
+
+/// Struct to serialize threads once they are done(aka the broker has crashed).
+#[derive(Serialize, Deserialize, Debug)]
+pub struct SeedAndIterations {
+    pub seed: String,
+    pub iterations: String,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::packets::Packets;
+    #[test]
+    fn test_serialize_packet_queue() {
+        let mut packet_queue = PacketQueue::default();
+        packet_queue.inner.insert(vec![0x10], Packets::default());
+        let serialized = toml::to_string(&packet_queue).unwrap();
+        println!("{}", serialized);
+    }
+}