askiiart/disk-read-benchmark
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

everything should be done except ext4 via FUSE and 2/4 bulk functions

Browse source
This commit is contained in:
askiiart 2024-10-29 13:53:31 -05:00
parent 909dca377b
commit 4a68f2ce29
Signed by untrusted user who does not match committer: askiiart
GPG key ID: EA85979611654C30
4 changed files with 373 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
Cargo.lock generated
View file

@ -118,6 +118,27 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "csv"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
dependencies = [
"csv-core",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "csv-core"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
dependencies = [
"memchr",
]
[[package]]
name = "curl"
version = "0.4.47"
@ -153,6 +174,7 @@ name = "disk-read-benchmark"
version = "0.1.0"
dependencies = [
"clap",
"csv",
"curl",
"rand",
"rand_xorshift",
@ -181,6 +203,12 @@ version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
[[package]]
name = "itoa"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "libc"
version = "0.2.161"
@ -199,6 +227,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "memchr"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "openssl-probe"
version = "0.1.5"
@ -289,6 +323,12 @@ dependencies = [
"rand_core",
]
[[package]]
name = "ryu"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "schannel"
version = "0.1.26"
@ -298,6 +338,26 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "serde"
version = "1.0.214"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.214"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "shlex"
version = "1.3.0"

1
Cargo.toml
View file

@ -5,6 +5,7 @@ edition = "2021"
[dependencies]
clap = { version = "4.5.20", features = ["cargo", "derive"] }
csv = "1.3.0"
curl = "0.4.47"
rand = "0.8.5"
rand_xorshift = "0.3.0"

35
prepare.sh Normal file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
cd $DIR
if [ ! -f ./data/ext-workdir/dwarfs ]; then
time mkdwarfs -l 1 -i ./data/datasets/ -o ./data/ext-workdir/dwarfs
fi
mkdir ./data/mountpoints/dwarfs/
time dwarfs ./data/ext-workdir/dwarfs ./data/mountpoints/dwarfs/
# **DISABLED** - far too slow
#cd ./data/datasets/
#if [ ! -f ./data/ext-workdir/fuse-archive.tar.gz ]; then
# time tar -czf ../ext-workdir/fuse-archive.tar.gz .
#fi
#cd -
#mkdir ./data/mountpoints/fuse-archive-tar-gz/
#time fuse-archive ./data/ext-workdir/fuse-archive.tar.gz ./data/mountpoints/fuse-archive-tar-gz/
cd ./data/datasets/
if [ ! -f ./data/ext-workdir/fuse-archive.tar ]; then
time tar -cf ../ext-workdir/fuse-archive.tar .
fi
cd -
mkdir ./data/mountpoints/fuse-archive-tar/
time fuse-archive ./data/ext-workdir/fuse-archive.tar ./data/mountpoints/fuse-archive-tar/
# **DISABLED** - also too slow
#cd ./data/datasets/
#if [ ! -f ./data/ext-workdir/fuse-archive.tar.zst ]; then
# time bash -c 'tar -cf - . | zstd -1 - -o ../ext-workdir/fuse-archive.tar.zst'
#fi
#cd -
#mkdir ./data/mountpoints/fuse-archive-tar-zst/
#time fuse-archive ./data/ext-workdir/fuse-archive.tar.zst ./data/mountpoints/fuse-archive-tar-zst/

329
src/main.rs
View file

@ -1,17 +1,25 @@
use curl::easy::Easy as easy_curl;
use rand::{self, RngCore, SeedableRng};
use rand::{self, Rng, RngCore, SeedableRng};
use rand_xorshift::XorShiftRng;
use std::io::Read;
use std::time::{Duration, Instant};
use std::{
env::current_dir,
fs::{create_dir_all, exists, File},
fs::{create_dir_all, exists, remove_dir_all, remove_file, File},
io::{Error, Write},
os::unix::fs::FileExt,
process::Command,
sync::{Arc, Mutex},
thread::{self, JoinHandle},
};
use clap::Parser;
use clap::command;
/*
=================================================================
==== ====
==== ↓ DATASET GATHERING ↓ ====
==== ====
=================================================================
*/
fn large_random_file_generation(path: String) {
// https://stackoverflow.com/a/65235966
@ -71,26 +79,42 @@ fn single_threaded_large_random_file_generation(path: String) {
fn small_random_files_generation(folder: String) {
let mut rng = XorShiftRng::seed_from_u64(2484345508);
let mut data: [u8; 1024] = [0u8; 1024];
for i in 1..1001 {
for i in 1..1025 {
let mut out = File::create(format!("{folder}/{i}")).unwrap();
rng.fill_bytes(&mut data);
out.write_all(&data).unwrap();
}
}
fn create_empty_file(path: String, size: u64) {
fn random_file_generator(path: String, size_mib: u64) {
let mut out = File::create(path).unwrap();
let mut rng = XorShiftRng::seed_from_u64(2484345508);
let mut data = [0u8; 1310720];
let block_size = 1310720;
let blocks: u64 = (size_mib * 1024 * 1024) / block_size;
for _ in 0..blocks {
rng.fill_bytes(&mut data);
out.write_all(&data).unwrap();
}
}
fn create_null_file(path: String, size: u64) {
let out = File::create(path).unwrap();
out.write_all_at(&[0], size - 1).unwrap();
}
fn small_empty_files_generation(folder: String) {
for i in 1..1001 {
create_empty_file(format!("{folder}/{i}"), 1024);
// no reason for it not to be multithreaded, but there's not much point either, it hardly takes any time... if anything, the overhead from multithreading might be worse?
fn small_null_files_generation(folder: String) {
for i in 1..1025 {
create_null_file(format!("{folder}/{i}"), 1024);
}
}
fn grab_kernel(folder: String, kernel_version: String) {
// https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.6.58.tar.xz
fn grab_kernel(folder: String, kernel_version: String) -> Result<bool, String> {
// maybe i should've just used reqwest, but that's no fun (also much more dependencies and stuff i'm sure)
// NOTE: requires openssl-devel to be installed for compilation (presumably requires openssl-libs for execution)
if !(exists(format!("{folder}/linux-{kernel_version}.tar.xz")).unwrap()) {
let mut curl = easy_curl::new();
curl.url(&format!(
@ -99,86 +123,277 @@ fn grab_kernel(folder: String, kernel_version: String) {
.unwrap();
curl.follow_location(true).unwrap();
let mut out = File::create(format!("{folder}/linux-{kernel_version}.tar.xz")).unwrap();
curl.write_function(move |data| {
match curl.write_function(move |data| {
out.write_all(data).unwrap();
Ok(data.len())
})
.unwrap();
}) {
Ok(_) => (),
Err(e) => return Err(e.to_string()),
}
curl.perform().unwrap();
}
// i'm too lazy to do this in rust
if !(exists(format!("{folder}/linux-{kernel_version}")).unwrap()) {
let mut dir = current_dir().unwrap();
dir.push(folder);
Command::new("tar")
match Command::new("tar")
.current_dir(dir)
.arg("-xf")
.arg(&format!("linux-{kernel_version}.tar.xz"))
.arg("");
.arg("")
.output()
{
Ok(_) => (),
Err(e) => return Err(e.to_string()),
}
}
return Ok(true);
}
fn grab_datasets() {
fn grab_datasets() -> Result<bool, String> {
let kernel_version = "6.6.58";
create_dir_all("data/kernel").unwrap();
if !(exists(format!("data/kernel/linux-{kernel_version}")).unwrap()) {
if !exists(format!("data/datasets/kernel/linux-{kernel_version}")).unwrap() {
println!("Downloading kernel...");
grab_kernel("data/kernel".to_string(), kernel_version.to_string());
create_dir_all("data/datasets/kernel").unwrap();
match grab_kernel(
"data/datasets/kernel".to_string(),
kernel_version.to_string(),
) {
Ok(_) => (),
Err(e) => {
remove_dir_all(format!("data/datasets/kernel/linux-{kernel_version}")).unwrap();
remove_file(format!(
"data/datasets/kernel/linux-{kernel_version}.tar.xz"
))
.unwrap();
panic!("{}", e.to_string());
}
}
println!("Kernel downloaded");
} else {
println!("Kernel already downloaded");
}
if !(exists(format!("data/25G-random.bin")).unwrap()) {
if !exists(format!("data/datasets/25G-random.bin")).unwrap() {
println!("Generating random 25 GiB file...");
large_random_file_generation("data/25G-random.bin".to_string());
large_random_file_generation("data/datasets/25G-random.bin".to_string());
println!("Random 25 GiB file generated");
} else {
println!("Random 25 GiB file already generated");
}
if !(exists(format!("data/small-files/random")).unwrap()) {
if !exists(format!("data/datasets/small-files/random")).unwrap() {
println!("Generating random 1 KiB files...");
create_dir_all("data/small-files/random").unwrap();
small_random_files_generation("data/small-files/random".to_string());
create_dir_all("data/datasets/small-files/random").unwrap();
small_random_files_generation("data/datasets/small-files/random".to_string());
println!("Random 1 KiB files generated...");
} else {
println!("Random 1 KiB files already generated")
}
if !(exists(format!("data/25G-null.bin")).unwrap()) {
println!("Generating empty 25 GiB file...");
create_empty_file("data/25G-null.bin".to_string(), 26843545600);
println!("Empty 25 GiB file generated...");
} else {
println!("Empty 25 GiB file already generated");
if !exists(format!("data/datasets/25G-null.bin")).unwrap() {
println!("Generating null 25 GiB file...");
create_null_file("data/datasets/25G-null.bin".to_string(), 26843545600);
println!("Null 25 GiB file generated...");
}
if !(exists("data/small-files/null").unwrap()) {
println!("Generating empty 1 KiB files...");
create_dir_all("data/small-files/null").unwrap();
small_empty_files_generation("data/small-files/null".to_string());
println!("Empty 1 KiB files generated...");
} else {
println!("Empty 1 KiB files already generated")
if !exists("data/datasets/small-files/null").unwrap() {
println!("Generating null 1 KiB files...");
create_dir_all("data/datasets/small-files/null").unwrap();
small_null_files_generation("data/datasets/small-files/null".to_string());
println!("Null 1 KiB files generated...");
}
if !(exists("data/100M-polygon.txt").unwrap()) {
println!("*** MANUAL: Get 100M-sided regular polygon data and put it at `./data/100M-polygon.txt` ***");
} else {
println!("100M-sided regular polygon data exists")
}
if !exists("data/datasets/100M-polygon.txt").unwrap() {
return Err("*** MANUAL: Get 100M-sided regular polygon data and put it at `./data/datasets/100M-polygon.txt` ***".to_string());
};
return Ok(true);
}
/// A simple read-only benchmark testing latency, sequential reads, and random reads.
#[derive(Parser, Debug)]
struct Args {
/// A test thing
#[arg(short, long, default_value = "hellooooo")]
this_is_a_testtttt: String
fn prep_other_dirs() -> bool {
if !exists("data/ext-workdir").unwrap() {
create_dir_all("data/ext-workdir").unwrap();
};
if !exists("data/benchmark-workdir").unwrap() {
create_dir_all("data/benchmark-workdir").unwrap();
}
if !exists("data/mountpoints").unwrap() {
create_dir_all("data/mountpoints").unwrap();
};
return true;
}
/*
=================================================================
==== ====
==== ↓ BENCHMARKS ↓ ====
==== ====
=================================================================
*/
fn sequential_read(path: String) -> Duration {
let mut f: File = File::open(path).unwrap();
let size = f.metadata().unwrap().len();
let mut data: [u8; 1310720] = [0u8; 1310720];
// benchmarking/elapsed: https://stackoverflow.com/a/40953863
let now = Instant::now();
for _ in 0..(size / 1310720) {
f.read(&mut data).unwrap();
}
let elapsed = now.elapsed();
return elapsed;
}
/// Reads 1 byte from the start of file
fn sequential_read_latency(path: String) -> Duration {
let mut f: File = File::open(path).unwrap();
let mut data: [u8; 1] = [0u8; 1];
let now = Instant::now();
f.read(&mut data).unwrap();
let elapsed = now.elapsed();
return elapsed;
}
/// Reads 1 GiB from the file at `path` in random 1 MiB chunks
fn random_read(path: String) -> Duration {
let mut rng = XorShiftRng::seed_from_u64(9198675309);
let f: File = File::open(path).unwrap();
let size = f.metadata().unwrap().len();
let mut data: [u8; 1048576] = [0u8; 1048576];
let now = Instant::now();
for _ in 0..1024 {
let offset = rng.gen_range(0..((size - 1048576) / 1048576));
f.read_at(&mut data, offset).unwrap();
}
let elapsed = now.elapsed();
return elapsed;
}
/// Reads 1 random byte from the file at `path` 1024 times
fn random_read_latency(path: String) -> Duration {
let mut rng = XorShiftRng::seed_from_u64(9198675309);
let f: File = File::open(path).unwrap();
let size = f.metadata().unwrap().len();
let mut data: [u8; 1] = [0u8; 1];
let now = Instant::now();
for _ in 0..1024 {
let offset = rng.gen_range(0..(size - 1));
f.read_at(&mut data, offset).unwrap();
}
let elapsed = now.elapsed();
return elapsed;
}
fn bulk_sequential_read(path: String) -> Vec<Duration> {
let mut data: [u8; 1024] = [0u8; 1024];
let mut times: Vec<Duration> = Vec::new();
for i in 1..1025 {
let mut f: File = File::open(format!("{path}/{i}")).unwrap();
let now = Instant::now();
f.read(&mut data).unwrap();
let elapsed = now.elapsed();
times.push(elapsed);
}
return times;
}
fn bulk_sequential_read_latency(path: String) -> Vec<Duration> {
let mut data: [u8; 1] = [0u8; 1];
let mut times: Vec<Duration> = Vec::new();
for i in 1..1025 {
let now = Instant::now();
let mut f: File = File::open(format!("{path}/{i}")).unwrap();
f.read(&mut data).unwrap();
let elapsed = now.elapsed();
times.push(elapsed);
}
return times;
}
fn benchmark() {
let mut recorder = csv::Writer::from_path("data/benchmark-data.csv").unwrap();
let mut bulk_recorder = csv::Writer::from_path("data/bulk.csv").unwrap();
let mountpoint_dir = "data/mountpoints";
let mut filesystems = std::fs::read_dir(mountpoint_dir)
.unwrap()
.map(|item| {
let tmp = item.unwrap().file_name().into_string().unwrap();
format!("{mountpoint_dir}/{tmp}")
})
.collect::<Vec<String>>();
filesystems.push("data/datasets".to_string());
for fs in filesystems {
let single_files = vec![
"25G-null.bin".to_string(),
"25G-random.bin".to_string(),
"100M-polygon.txt".to_string(),
"kernel/linux-6.6.58.tar.xz".to_string(),
];
let bulk_files = vec!["small-files/null", "small-files/random"];
for filename in single_files {
println!("=== {} ===", filename);
let path = format!("{fs}/{filename}");
println!("{}", path);
//panic!("hi");
let seq_read = format!("{:.5?}", sequential_read(path.clone()));
println!("Sequential read (complete file read): {}", seq_read.clone());
let seq_latency = format!("{:.5?}", sequential_read_latency(path.clone()));
println!("Sequential latency (1 byte read): {}", seq_latency);
let rand_read = format!("{:.5?}", random_read(path.clone()));
println!("Random read (1024x 1 MiB): {}", rand_read);
let rand_latency = format!("{:.5?}", random_read_latency(path.clone()));
println!("Random latency (1024x 1 byte read): {}", rand_latency);
let data: Vec<String> = vec![
fs.clone(),
filename,
seq_read,
seq_latency,
rand_read,
rand_latency,
];
recorder.write_record(data).unwrap();
println!();
}
for folder in bulk_files {
bulk_recorder.write_record(_vec_duration_to_string(bulk_sequential_read(folder.to_string()))).unwrap();
bulk_recorder.write_record(_vec_duration_to_string(bulk_sequential_read_latency(folder.to_string()))).unwrap();
//bulk_recorder.write_record(_vec_duration_to_string(bulk_random_read(folder.to_string()))).unwrap();
//bulk_recorder.write_record(_vec_duration_to_string(bulk_random_read_latency(folder.to_string()))).unwrap();
}
println!("=== === === === === === === === === === ===\n")
}
}
fn main() {
let args = Args::parse();
grab_datasets().unwrap();
prep_other_dirs();
benchmark();
}
fn _vec_duration_to_string(vector_committing_crimes_with_both_direction_and_magnitude: Vec<Duration>) -> Vec<String> {
return vector_committing_crimes_with_both_direction_and_magnitude.iter()
.map(|item| {
format!("{:.5?}", item)
})
.collect::<Vec<String>>();
}