Benchmark Visualizer: Conjure Native vs Oxide

.gitignore

@@ -17,6 +17,9 @@ conjure_oxide/tests/*.png
 debug/
 target/
 
+## Essence solution JSON
+./tools/ci-test-benchmarks/data/*.json
+
 ## C++
 solvers/**/vendor/build
 *.o
@@ -25,6 +28,7 @@ solvers/**/vendor/build
 ## Python
 .env
 venv
+env
 __pycache__
 .ruff_cache
 

.vscode/settings.json

@@ -7,4 +7,10 @@
     "coverage-gutters.coverageFileNames": [
         "lcov.info",
     ],
+    "rust-analyzer.linkedProjects": [
+        "./tools/ci-test-benchmarks/Cargo.toml",
+        "./tools/ci-test-benchmarks/Cargo.toml",
+        "./tools/ci-test-benchmarks/Cargo.toml"
+    ],
+    "rust-analyzer.showUnlinkedFileNotification": false,
 }

Cargo.toml

@@ -18,7 +18,7 @@ members = [
     "crates/uniplate_derive",
     "solvers/kissat",
     "solvers/minion",
-    "solvers/chuffed",
+    "solvers/chuffed", "tools/benchmarks-visualizer",
 ]
 
 [workspace.lints.clippy]

conjure_oxide/tests/cpu_time_tests.rs

@@ -0,0 +1,52 @@
+// // cpu_time_tests.rs
+// // @author Pedro Gronda Garrigues
+
+// use serde::{Deserialize, Serialize};
+// use std::collections::HashMap;
+// use std::time::{Duration, Instant};
+
+// // struct for CPU time data
+// #[derive(Serialize, Deserialize, Debug)]
+// struct TestCPUTime {
+//     name: String,
+//     cpu_time: Duration,
+// }
+
+// // function to run test & measure CPU time
+// fn run_test_cpu_time<F: FnOnce()>(name: &str, test_fn: F) -> TestCPUTime {
+//     let start_time = Instant::now();
+//     test_fn();
+//     let end_time = Instant::now();
+//     let cpu_time = end_time - start_time;
+//     TestCPUTime {
+//         name: name.to_string(),
+//         cpu_time,
+//     }
+// }
+
+// // function to write CPU tiem data to file
+// fn write_cpu_time_data(data: HashMap<String, Vec<TestCPUTime>) -> Result<(), Box<dyn std::error::Error>> {
+//     let file = std::fs::File::create("cpu_time_data.json")?;
+//     serde_json::to_writer(file, &data)?;
+//     Ok(())
+// }
+
+// fn write_tests() {
+//     // assume import of tests (Work in Progress...)
+//     //
+
+//     let mut cpu_time_data: HashMap<String, Vec<TestCPUTime>> = HashMap::new();
+//     let tests = vec![
+//         // format: ("name", tests::test_<...>)
+//     ];
+
+//     for (name, test_fn) in tests {
+//         let test_cpu_time = run_test_cpu_time(name, test_fn);
+//         let category = "Testing".to_string(); // assume test category (to be changed)
+//         cpu_time_data.entry(name).or_insert_with(Vec::new).push(test_cpu_time);
+//     }
+
+//     if let Err(err) = write_cpu_time_data(cpu_time_data) {
+//         eprintln!("Error writing CPU time data: {}", err);
+//     }
+// }

tools/benchmarks-visualizer/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "benchmarks-visualizer"
+version = "0.1.0"
+authors = ["Pedro Gronda Garrigues"]
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+rayon = "1.5.1" # for conjure native test parallelization
+
+[lints]
+workspace = true

tools/benchmarks-visualizer/README.md

@@ -0,0 +1,80 @@
+# BENCHMARKS VISUALIZER
+
+A dashboard to visualize useful statistics comparing Conjure (native) and Conjure Oxide
+
+Stats included thus far:
+ - Num of Nodes (incomplete, need comparison between node count for native vs oxide)
+ - Time elapsed (milliseconds, ms)
+
+Author: Pedro Gronda Garrigues (220027583)
+
+## DEPENDENCIES
+
+To download requirement dependencies, follow the folling steps.
+
+*Note: this is done manually instead of in build.sh for safety reasons.
+
+1. Make sure you are in the correct repo directory
+```bash
+cd ./tools/benchmarks-visualizer
+```
+
+2. Create virtual environment
+```bash
+python -m venv env
+```
+
+3. Activate virtual environment
+```bash
+# for Windows 10/11
+.\env\Scripts\activate
+
+# for macOS/Linux
+source env/bin/activate
+```
+
+4. Install required packages
+```bash
+pip install -r requirements.txt
+```
+
+## EXECUTE
+
+A shell script is provided, which includes a prompted option to rerun all exhaustive tests for both Conjure and Conjure Oxide:
+```bash
+./build.sh
+```
+
+## UPDATE MANUALLY
+
+To update conjure native solver solutions statistics and oxide statistics, please execute the following (already included in build.sh):
+```bash
+# ensure you are in ./tools/benchmarks-visualizer directory
+cd ./tools/benchmarks-visualizer
+
+# to write data to ./data for conjure native for solver(s)
+CARGO_TARGET_DIR=./tools/benchmarks-visualizer cargo build --bin conjure_native_benchmarks
+./tools/benchmarks-visualizer/debug/conjure_native_benchmarks
+
+# to write data to ./data for conjure oxide (Minion supported for now) -- warning: very verbose
+./src/conjure_oxide_benchmarks.sh
+```
+
+## RUN DASHBOARD MANUALLY
+
+To run Dash dashboard manually simply execute the app.py script:
+```bash
+python3 src/app.py
+```
+
+To then visualize the dashboard (for now, until hosted), open a web browser and navigate to `http://127.0.0.1:8050/`:
+```bash
+# for windows 10/11
+start http://127.0.0.1:8050/
+
+# for macOS
+open http://127.0.0.1:8050/ 
+
+# for Linux (most desktop env)
+xdg-open http://127.0.0.1:8050/
+```

tools/benchmarks-visualizer/build.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# @author: Pedro Gronda Garrigues
+
+# prompt for updating Conjure Native data
+read -p "Do you want to update Conjure Native exhaustive test data? (yes/no) " update_native
+if [[ $update_native == "yes" ]]; then
+    # build and run conjure_native_benchmarks
+    echo "Building Conjure Native benchmarks..."
+    CARGO_TARGET_DIR=./tools/benchmarks-visualizer cargo build --bin conjure_native_benchmarks
+
+    if [[ $? -eq 0 ]]; then
+        echo "STATUS: Running Conjure Native exhaustive tests..."
+        ./tools/benchmarks-visualizer/debug/conjure_native_benchmarks
+    else
+        echo "STATUS: Build failed for Conjure Native benchmarks."
+        exit 1
+    fi
+fi
+
+# prompt for updating Conjure Oxide data
+read -p "Do you want to update Conjure Oxide data? (yes/no) " update_oxide
+if [[ $update_oxide == "yes" ]]; then
+    # run conjure_oxide_benchmarks.sh script
+    echo "STATUS: Running Conjure Oxide benchmarks..."
+    ./src/conjure_oxide_benchmarks.sh
+fi
+
+# execute the Python script for visualization dashboard
+echo "STATUS: Running the Python visualization app..."
+python3 ./src/app.py
+
+# check for any errors in Python script execution
+if [[ $? -ne 0 ]]; then
+    echo "STATUS: Error occurred while running the Python visualization app."
+    exit 2
+fi
+
+echo "STATUS: Benchmark visualization process completed successfully."

tools/benchmarks-visualizer/requirements.txt

@@ -0,0 +1,4 @@
+dash
+plotly
+dash-bootstrap-components
+pandas