everything should be done except ext4 via FUSE and 2/4 bulk functions

Cargo.lock

@@ -118,6 +118,27 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 
+[[package]]
+name = "csv"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "curl"
 version = "0.4.47"
@@ -153,6 +174,7 @@ name = "disk-read-benchmark"
 version = "0.1.0"
 dependencies = [
  "clap",
+ "csv",
  "curl",
  "rand",
  "rand_xorshift",
@@ -181,6 +203,12 @@ version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
+[[package]]
+name = "itoa"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+
 [[package]]
 name = "libc"
 version = "0.2.161"
@@ -199,6 +227,12 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
@@ -289,6 +323,12 @@ dependencies = [
  "rand_core",
 ]
 
+[[package]]
+name = "ryu"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+
 [[package]]
 name = "schannel"
 version = "0.1.26"
@@ -298,6 +338,26 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "serde"
+version = "1.0.214"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.214"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "shlex"
 version = "1.3.0"

Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 clap = { version = "4.5.20", features = ["cargo", "derive"] }
+csv = "1.3.0"
 curl = "0.4.47"
 rand = "0.8.5"
 rand_xorshift = "0.3.0"

prepare.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+cd $DIR
+
+if [ ! -f ./data/ext-workdir/dwarfs ]; then
+    time mkdwarfs -l 1 -i ./data/datasets/ -o ./data/ext-workdir/dwarfs
+fi
+mkdir ./data/mountpoints/dwarfs/
+time dwarfs ./data/ext-workdir/dwarfs ./data/mountpoints/dwarfs/
+
+# **DISABLED** - far too slow
+#cd ./data/datasets/
+#if [ ! -f ./data/ext-workdir/fuse-archive.tar.gz ]; then
+#    time tar -czf ../ext-workdir/fuse-archive.tar.gz .
+#fi
+#cd -
+#mkdir ./data/mountpoints/fuse-archive-tar-gz/
+#time fuse-archive ./data/ext-workdir/fuse-archive.tar.gz ./data/mountpoints/fuse-archive-tar-gz/
+
+cd ./data/datasets/
+if [ ! -f ./data/ext-workdir/fuse-archive.tar ]; then
+    time tar -cf ../ext-workdir/fuse-archive.tar .
+fi
+cd -
+mkdir ./data/mountpoints/fuse-archive-tar/
+time fuse-archive ./data/ext-workdir/fuse-archive.tar ./data/mountpoints/fuse-archive-tar/
+
+# **DISABLED** - also too slow
+#cd ./data/datasets/
+#if [ ! -f ./data/ext-workdir/fuse-archive.tar.zst ]; then
+#    time bash -c 'tar -cf - . | zstd -1 - -o ../ext-workdir/fuse-archive.tar.zst'
+#fi
+#cd -
+#mkdir ./data/mountpoints/fuse-archive-tar-zst/
+#time fuse-archive ./data/ext-workdir/fuse-archive.tar.zst ./data/mountpoints/fuse-archive-tar-zst/

src/main.rs

@@ -1,17 +1,25 @@
 use curl::easy::Easy as easy_curl;
-use rand::{self, RngCore, SeedableRng};
+use rand::{self, Rng, RngCore, SeedableRng};
 use rand_xorshift::XorShiftRng;
+use std::io::Read;
+use std::time::{Duration, Instant};
 use std::{
     env::current_dir,
-    fs::{create_dir_all, exists, File},
+    fs::{create_dir_all, exists, remove_dir_all, remove_file, File},
     io::{Error, Write},
     os::unix::fs::FileExt,
     process::Command,
     sync::{Arc, Mutex},
     thread::{self, JoinHandle},
 };
-use clap::Parser;
+
-use clap::command;
+/*
+    =================================================================
+    ====                                                         ====
+    ====                 ↓ DATASET GATHERING ↓                   ====
+    ====                                                         ====
+    =================================================================
+*/
 
 fn large_random_file_generation(path: String) {
     // https://stackoverflow.com/a/65235966
@@ -71,26 +79,42 @@ fn single_threaded_large_random_file_generation(path: String) {
 fn small_random_files_generation(folder: String) {
     let mut rng = XorShiftRng::seed_from_u64(2484345508);
     let mut data: [u8; 1024] = [0u8; 1024];
-    for i in 1..1001 {
+    for i in 1..1025 {
         let mut out = File::create(format!("{folder}/{i}")).unwrap();
         rng.fill_bytes(&mut data);
         out.write_all(&data).unwrap();
     }
 }
 
-fn create_empty_file(path: String, size: u64) {
+fn random_file_generator(path: String, size_mib: u64) {
+    let mut out = File::create(path).unwrap();
+    let mut rng = XorShiftRng::seed_from_u64(2484345508);
+
+    let mut data = [0u8; 1310720];
+    let block_size = 1310720;
+    let blocks: u64 = (size_mib * 1024 * 1024) / block_size;
+
+    for _ in 0..blocks {
+        rng.fill_bytes(&mut data);
+        out.write_all(&data).unwrap();
+    }
+}
+
+fn create_null_file(path: String, size: u64) {
     let out = File::create(path).unwrap();
     out.write_all_at(&[0], size - 1).unwrap();
 }
 
-fn small_empty_files_generation(folder: String) {
+// no reason for it not to be multithreaded, but there's not much point either, it hardly takes any time... if anything, the overhead from multithreading might be worse?
-    for i in 1..1001 {
+fn small_null_files_generation(folder: String) {
-        create_empty_file(format!("{folder}/{i}"), 1024);
+    for i in 1..1025 {
+        create_null_file(format!("{folder}/{i}"), 1024);
     }
 }
 
-fn grab_kernel(folder: String, kernel_version: String) {
+fn grab_kernel(folder: String, kernel_version: String) -> Result<bool, String> {
-    // https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.6.58.tar.xz
+    // maybe i should've just used reqwest, but that's no fun (also much more dependencies and stuff i'm sure)
+    // NOTE: requires openssl-devel to be installed for compilation (presumably requires openssl-libs for execution)
     if !(exists(format!("{folder}/linux-{kernel_version}.tar.xz")).unwrap()) {
         let mut curl = easy_curl::new();
         curl.url(&format!(
@@ -99,86 +123,277 @@ fn grab_kernel(folder: String, kernel_version: String) {
         .unwrap();
         curl.follow_location(true).unwrap();
         let mut out = File::create(format!("{folder}/linux-{kernel_version}.tar.xz")).unwrap();
-        curl.write_function(move |data| {
+        match curl.write_function(move |data| {
             out.write_all(data).unwrap();
             Ok(data.len())
-        })
+        }) {
-        .unwrap();
+            Ok(_) => (),
+            Err(e) => return Err(e.to_string()),
+        }
         curl.perform().unwrap();
     }
 
     // i'm too lazy to do this in rust
-    let mut dir = current_dir().unwrap();
+    if !(exists(format!("{folder}/linux-{kernel_version}")).unwrap()) {
-    dir.push(folder);
+        let mut dir = current_dir().unwrap();
-    Command::new("tar")
+        dir.push(folder);
-        .current_dir(dir)
+        match Command::new("tar")
-        .arg("-xf")
+            .current_dir(dir)
-        .arg(&format!("linux-{kernel_version}.tar.xz"))
+            .arg("-xf")
-        .arg("");
+            .arg(&format!("linux-{kernel_version}.tar.xz"))
+            .arg("")
+            .output()
+        {
+            Ok(_) => (),
+            Err(e) => return Err(e.to_string()),
+        }
+    }
+
+    return Ok(true);
 }
 
-fn grab_datasets() {
+fn grab_datasets() -> Result<bool, String> {
     let kernel_version = "6.6.58";
 
-    create_dir_all("data/kernel").unwrap();
+    if !exists(format!("data/datasets/kernel/linux-{kernel_version}")).unwrap() {
-
-    if !(exists(format!("data/kernel/linux-{kernel_version}")).unwrap()) {
         println!("Downloading kernel...");
-        grab_kernel("data/kernel".to_string(), kernel_version.to_string());
+        create_dir_all("data/datasets/kernel").unwrap();
+        match grab_kernel(
+            "data/datasets/kernel".to_string(),
+            kernel_version.to_string(),
+        ) {
+            Ok(_) => (),
+            Err(e) => {
+                remove_dir_all(format!("data/datasets/kernel/linux-{kernel_version}")).unwrap();
+                remove_file(format!(
+                    "data/datasets/kernel/linux-{kernel_version}.tar.xz"
+                ))
+                .unwrap();
+                panic!("{}", e.to_string());
+            }
+        }
         println!("Kernel downloaded");
-    } else {
-        println!("Kernel already downloaded");
     }
 
-    if !(exists(format!("data/25G-random.bin")).unwrap()) {
+    if !exists(format!("data/datasets/25G-random.bin")).unwrap() {
         println!("Generating random 25 GiB file...");
-        large_random_file_generation("data/25G-random.bin".to_string());
+        large_random_file_generation("data/datasets/25G-random.bin".to_string());
         println!("Random 25 GiB file generated");
-    } else {
-        println!("Random 25 GiB file already generated");
     }
 
-    if !(exists(format!("data/small-files/random")).unwrap()) {
+    if !exists(format!("data/datasets/small-files/random")).unwrap() {
         println!("Generating random 1 KiB files...");
-        create_dir_all("data/small-files/random").unwrap();
+        create_dir_all("data/datasets/small-files/random").unwrap();
-        small_random_files_generation("data/small-files/random".to_string());
+        small_random_files_generation("data/datasets/small-files/random".to_string());
         println!("Random 1 KiB files generated...");
-    } else {
-        println!("Random 1 KiB files already generated")
     }
 
-    if !(exists(format!("data/25G-null.bin")).unwrap()) {
+    if !exists(format!("data/datasets/25G-null.bin")).unwrap() {
-        println!("Generating empty 25 GiB file...");
+        println!("Generating null 25 GiB file...");
-        create_empty_file("data/25G-null.bin".to_string(), 26843545600);
+        create_null_file("data/datasets/25G-null.bin".to_string(), 26843545600);
-        println!("Empty 25 GiB file generated...");
+        println!("Null 25 GiB file generated...");
-    } else {
-        println!("Empty 25 GiB file already generated");
     }
 
-    if !(exists("data/small-files/null").unwrap()) {
+    if !exists("data/datasets/small-files/null").unwrap() {
-        println!("Generating empty 1 KiB files...");
+        println!("Generating null 1 KiB files...");
-        create_dir_all("data/small-files/null").unwrap();
+        create_dir_all("data/datasets/small-files/null").unwrap();
-        small_empty_files_generation("data/small-files/null".to_string());
+        small_null_files_generation("data/datasets/small-files/null".to_string());
-        println!("Empty 1 KiB files generated...");
+        println!("Null 1 KiB files generated...");
-    } else {
-        println!("Empty 1 KiB files already generated")
     }
 
-    if !(exists("data/100M-polygon.txt").unwrap()) {
+    if !exists("data/datasets/100M-polygon.txt").unwrap() {
-        println!("*** MANUAL: Get 100M-sided regular polygon data and put it at `./data/100M-polygon.txt` ***");
+        return Err("*** MANUAL: Get 100M-sided regular polygon data and put it at `./data/datasets/100M-polygon.txt` ***".to_string());
-    } else {
+    };
-        println!("100M-sided regular polygon data exists")
+
-    }
+    return Ok(true);
 }
 
-/// A simple read-only benchmark testing latency, sequential reads, and random reads.
+fn prep_other_dirs() -> bool {
-#[derive(Parser, Debug)]
+    if !exists("data/ext-workdir").unwrap() {
-struct Args {
+        create_dir_all("data/ext-workdir").unwrap();
-    /// A test thing
+    };
-    #[arg(short, long, default_value = "hellooooo")]
+
-    this_is_a_testtttt: String
+    if !exists("data/benchmark-workdir").unwrap() {
+        create_dir_all("data/benchmark-workdir").unwrap();
+    }
+
+    if !exists("data/mountpoints").unwrap() {
+        create_dir_all("data/mountpoints").unwrap();
+    };
+
+    return true;
+}
+
+/*
+    =================================================================
+    ====                                                         ====
+    ====                     ↓ BENCHMARKS ↓                      ====
+    ====                                                         ====
+    =================================================================
+*/
+
+fn sequential_read(path: String) -> Duration {
+    let mut f: File = File::open(path).unwrap();
+    let size = f.metadata().unwrap().len();
+
+    let mut data: [u8; 1310720] = [0u8; 1310720];
+    // benchmarking/elapsed: https://stackoverflow.com/a/40953863
+    let now = Instant::now();
+    for _ in 0..(size / 1310720) {
+        f.read(&mut data).unwrap();
+    }
+    let elapsed = now.elapsed();
+    return elapsed;
+}
+
+/// Reads 1 byte from the start of file
+fn sequential_read_latency(path: String) -> Duration {
+    let mut f: File = File::open(path).unwrap();
+    let mut data: [u8; 1] = [0u8; 1];
+    let now = Instant::now();
+    f.read(&mut data).unwrap();
+    let elapsed = now.elapsed();
+    return elapsed;
+}
+
+/// Reads 1 GiB from the file at `path` in random 1 MiB chunks
+fn random_read(path: String) -> Duration {
+    let mut rng = XorShiftRng::seed_from_u64(9198675309);
+    let f: File = File::open(path).unwrap();
+    let size = f.metadata().unwrap().len();
+
+    let mut data: [u8; 1048576] = [0u8; 1048576];
+    let now = Instant::now();
+    for _ in 0..1024 {
+        let offset = rng.gen_range(0..((size - 1048576) / 1048576));
+        f.read_at(&mut data, offset).unwrap();
+    }
+    let elapsed = now.elapsed();
+    return elapsed;
+}
+
+/// Reads 1 random byte from the file at `path` 1024 times
+fn random_read_latency(path: String) -> Duration {
+    let mut rng = XorShiftRng::seed_from_u64(9198675309);
+    let f: File = File::open(path).unwrap();
+    let size = f.metadata().unwrap().len();
+    let mut data: [u8; 1] = [0u8; 1];
+    let now = Instant::now();
+    for _ in 0..1024 {
+        let offset = rng.gen_range(0..(size - 1));
+        f.read_at(&mut data, offset).unwrap();
+    }
+    let elapsed = now.elapsed();
+    return elapsed;
+}
+
+fn bulk_sequential_read(path: String) -> Vec<Duration> {
+    let mut data: [u8; 1024] = [0u8; 1024];
+    let mut times: Vec<Duration> = Vec::new();
+    for i in 1..1025 {
+        let mut f: File = File::open(format!("{path}/{i}")).unwrap();
+        let now = Instant::now();
+        f.read(&mut data).unwrap();
+        let elapsed = now.elapsed();
+        times.push(elapsed);
+    }
+
+    return times;
+}
+
+fn bulk_sequential_read_latency(path: String) -> Vec<Duration> {
+    let mut data: [u8; 1] = [0u8; 1];
+    let mut times: Vec<Duration> = Vec::new();
+    for i in 1..1025 {
+        let now = Instant::now();
+        let mut f: File = File::open(format!("{path}/{i}")).unwrap();
+        f.read(&mut data).unwrap();
+        let elapsed = now.elapsed();
+        times.push(elapsed);
+    }
+
+    return times;
+}
+
+
+fn benchmark() {
+    let mut recorder = csv::Writer::from_path("data/benchmark-data.csv").unwrap();
+    let mut bulk_recorder = csv::Writer::from_path("data/bulk.csv").unwrap();
+    let mountpoint_dir = "data/mountpoints";
+    let mut filesystems = std::fs::read_dir(mountpoint_dir)
+        .unwrap()
+        .map(|item| {
+            let tmp = item.unwrap().file_name().into_string().unwrap();
+            format!("{mountpoint_dir}/{tmp}")
+        })
+        .collect::<Vec<String>>();
+
+    filesystems.push("data/datasets".to_string());
+
+    for fs in filesystems {
+        let single_files = vec![
+            "25G-null.bin".to_string(),
+            "25G-random.bin".to_string(),
+            "100M-polygon.txt".to_string(),
+            "kernel/linux-6.6.58.tar.xz".to_string(),
+        ];
+
+        let bulk_files = vec!["small-files/null", "small-files/random"];
+
+        for filename in single_files {
+            println!("=== {} ===", filename);
+
+            let path = format!("{fs}/{filename}");
+            println!("{}", path);
+            //panic!("hi");
+
+            let seq_read = format!("{:.5?}", sequential_read(path.clone()));
+            println!("Sequential read (complete file read): {}", seq_read.clone());
+
+            let seq_latency = format!("{:.5?}", sequential_read_latency(path.clone()));
+            println!("Sequential latency (1 byte read): {}", seq_latency);
+
+            let rand_read = format!("{:.5?}", random_read(path.clone()));
+            println!("Random read (1024x 1 MiB): {}", rand_read);
+
+            let rand_latency = format!("{:.5?}", random_read_latency(path.clone()));
+            println!("Random latency (1024x 1 byte read): {}", rand_latency);
+
+            let data: Vec<String> = vec![
+                fs.clone(),
+                filename,
+                seq_read,
+                seq_latency,
+                rand_read,
+                rand_latency,
+            ];
+            recorder.write_record(data).unwrap();
+
+            println!();
+        }
+
+        for folder in bulk_files {
+            bulk_recorder.write_record(_vec_duration_to_string(bulk_sequential_read(folder.to_string()))).unwrap();
+            bulk_recorder.write_record(_vec_duration_to_string(bulk_sequential_read_latency(folder.to_string()))).unwrap();
+            //bulk_recorder.write_record(_vec_duration_to_string(bulk_random_read(folder.to_string()))).unwrap();
+            //bulk_recorder.write_record(_vec_duration_to_string(bulk_random_read_latency(folder.to_string()))).unwrap();
+        }
+
+        println!("=== === === === === === === === === === ===\n")
+    }
 }
 
 fn main() {
-    let args = Args::parse();
+    grab_datasets().unwrap();
+    prep_other_dirs();
+    benchmark();
+}
+
+fn _vec_duration_to_string(vector_committing_crimes_with_both_direction_and_magnitude: Vec<Duration>) -> Vec<String> {
+    return vector_committing_crimes_with_both_direction_and_magnitude.iter()
+    .map(|item| {
+        format!("{:.5?}", item)
+    })
+    .collect::<Vec<String>>();
+
 }