From bd880f4f5fba39731f0acb2a4ac561be37fca314 Mon Sep 17 00:00:00 2001 From: askiiart Date: Thu, 26 Dec 2024 12:18:16 -0600 Subject: [PATCH] add reading config, add and update docs --- Cargo.lock | 160 ++++++++++++++++++------------------ Cargo.toml | 3 +- docs/config-reference.md | 7 +- docs/default-max-threads.md | 38 +++++++++ src/README.md | 2 + src/benchmark.rs | 35 ++++++++ src/data.rs | 91 ++++++++++++++++++++ src/main.rs | 16 ++-- src/tests.rs | 2 +- 9 files changed, 265 insertions(+), 89 deletions(-) create mode 100644 docs/default-max-threads.md create mode 100644 src/benchmark.rs create mode 100644 src/data.rs diff --git a/Cargo.lock b/Cargo.lock index 768ff33..2ef1402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "alphanumeric-sort" version = "1.5.3" @@ -70,16 +58,10 @@ dependencies = [ ] [[package]] -name = "arraydeque" -version = "0.5.1" +name = "anyhow" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "clap" @@ -137,13 +119,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] -name = "encoding_rs" -version = "0.8.35" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "gregory" @@ -152,26 +131,15 @@ dependencies = [ "alphanumeric-sort", "clap", "clap_complete", - "yaml-rust2", + "serde", + "serde_yml", ] [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" -dependencies = [ - "hashbrown", -] +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heck" @@ -179,6 +147,16 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indexmap" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -186,10 +164,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] -name = "once_cell" -version = "1.20.2" +name = "itoa" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "libyml" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3302702afa434ffa30847a83305f0a69d6abd74293b6554c18ec85c7ef30c980" +dependencies = [ + "anyhow", + "version_check", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "proc-macro2" @@ -209,6 +203,47 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yml" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" +dependencies = [ + "indexmap", + "itoa", + "libyml", + "memchr", + "ryu", + "serde", + "version_check", +] + [[package]] name = "strsim" version = "0.11.1" @@ -217,9 +252,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.90" +version = "2.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" dependencies = [ "proc-macro2", "quote", @@ -316,34 +351,3 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "yaml-rust2" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a1a1c0bc9823338a3bdf8c61f994f23ac004c6fa32c08cd152984499b445e8d" -dependencies = [ - "arraydeque", - "encoding_rs", - "hashlink", -] - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/Cargo.toml b/Cargo.toml index eb17f79..1f102c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,8 @@ edition = "2021" alphanumeric-sort = "1.5.3" clap = { version = "4.5.23", features = ["derive"] } clap_complete = "4.5.40" -yaml-rust2 = "0.9.0" +serde = { version = "1.0.216", features = ["derive"] } +serde_yml = "0.0.12" [profile.release] opt-level = 3 diff --git a/docs/config-reference.md b/docs/config-reference.md index 6745214..899ea29 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -8,9 +8,12 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr ## Top-level config - `log-level`: Log level `0`-`3` (error, warning, info, or debug) + - Default: 1 - warning - `max-threads`: The maximum number of threads to be used - **See also**: [`threads`](#job-config) + - Default is CPU's threads - 2 - `max-jobs`: The maximum number of jobs to be run at once + - Default is 1 **Multithreading notes (IMPORTANT)**: Gregory will first run compilation jobs, then packaging jobs for whatever programs are done, then run the `update-repo` for whichever distros are finished. For this reason, the distro names listed under `packaging` and `update-repo` *must* match. @@ -21,8 +24,8 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr - See `--cpus` in the [`podman run` docs](https://docs.podman.io/en/latest/markdown/podman-run.1.html#cpus) - *Root may be required for this argument* - If not specified, it will fall back to `max-threads` -- `image`: The Docker image to run the job in -- `commands`: The commands to run +- `image`: The Docker image to run the job in *(required)* +- `commands`: The commands to run *(required)* - TODO: Add command file/bash script instead - `volumes`: Names of volumes as defined in [`volumes` (top level)](#volumes) diff --git a/docs/default-max-threads.md b/docs/default-max-threads.md new file mode 100644 index 0000000..826e793 --- /dev/null +++ b/docs/default-max-threads.md @@ -0,0 +1,38 @@ +# Default `max-threads` + +The default `max-threads` uses a simple algorithm to get how many threads. + +```rs +if total_threads >= 32 { + return total_threads - 4; +} else if total_threads >= 12 { + return total_threads - 2; +} else if total_threads >= 3 { + return total_threads - 1; +} else { + return total_threads; +} +``` + +i.e. with `total_threads` as the number of threads the CPU(s) has: + +- If the CPU has 32 or more threads, it will use all but 4 threads +- If the CPU has 12 or more threads, it will use all but 2 threads +- If the CPU has 3 or more threads, it will use all but 1 thread +- Otherwise, i.e. if the CPU has 1 to 2 threads, it will use all threads + +--- + +Alternative algorithms I tried: + +```rs +if total_threads >= 32 { + return total_threads - total_threads.div_ceil(10); +} else if total_threads >= 12 { + return total_threads - 2; +} else if total_threads >= 3 { + return total_threads - 1; +} else { + return total_threads; +} +``` diff --git a/src/README.md b/src/README.md index cf0e9d6..faeacd4 100644 --- a/src/README.md +++ b/src/README.md @@ -8,6 +8,8 @@ This is Gregory. Gregory controls repos. Gregory keeps track of updating repos, - Add support for loading scripts rather than listing commands - Add multithreading - Add better/custom grouping for when to run jobs (dependency system?) +- Add dependency system (automatic detection?) +- Add hook system ## Other stuff diff --git a/src/benchmark.rs b/src/benchmark.rs new file mode 100644 index 0000000..cab81a6 --- /dev/null +++ b/src/benchmark.rs @@ -0,0 +1,35 @@ +use std::time::Instant; + +fn log_thing(total_threads: u32) -> u32 { + return total_threads - (f64::log(total_threads.into(), 5.0).round() as u32); +} + +fn my_thing(total_threads: u32) -> u32 { + if total_threads >= 32 { + return total_threads - 4; + } else if total_threads >= 12 { + return total_threads - 2; + } else if total_threads >= 3 { + return total_threads - 1; + } else { + return total_threads; + } + + //println!("{}", max_threads) +} + +// let mut total_threads = thread::available_parallelism().unwrap().get() as u32; + +fn main() { + /* + let now = Instant::now(); + for _ in 0..100000000 { + + } + let elapsed = now.elapsed(); + */ + let total_threads: u32 = 128; + println!("{}", log_thing(total_threads)); + println!("{}", my_thing(total_threads)); + //println!("{}", elapsed.as_nanos()); +} diff --git a/src/data.rs b/src/data.rs new file mode 100644 index 0000000..2d6cf76 --- /dev/null +++ b/src/data.rs @@ -0,0 +1,91 @@ +//! Datasets used by gregory and stuff for handling them + +use serde::Deserialize; +use std::{collections::HashMap, fs, thread}; + +/// The config for gregory +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct Config { + /// What level to log at + /// + /// - 0: Error + /// - 1: Warning + /// - 2: Info + /// - 3: Debug + #[serde(default = "log_level", rename = "log-level")] + log_level: u8, + /// Maximum number of jobs to run simultaneously + #[serde(default = "max_jobs", rename = "max-jobs")] + max_jobs: u32, + /// Maximum number of threads to use + #[serde(default = "max_threads", rename = "max-threads")] + max_threads: u32, + /// Holds the packages, including their compilation and packaging + /// + /// Format: { "librewolf": Package { compilation, packaging } } + /// + /// See [`Package`] for details + packages: HashMap, + #[serde(rename = "update-repo")] + update_repo: HashMap, + #[serde(default = "volumes")] + volumes: HashMap, +} + +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct Job { + #[serde(default = "job_threads")] + /// How many threads to limit this job to; recommended to set it to the max threads the job will use + /// + /// If `threads` isn't specified, it will fall back to `max_threads` (from [`Config`]); the same behavior applies if `threads` is greater than `max_threads` + threads: u32, + image: String, + commands: Vec, + volumes: Option>, +} + +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct Package { + compilation: Option, + packaging: HashMap, +} + +pub(crate) fn config_from_file(filename: String) -> Config { + let yaml: Config = serde_yml::from_str(fs::read_to_string(filename).unwrap().as_str()).unwrap(); + return yaml; +} + +// ========================== +// === === +// === ↓ DEFAULTS ↓ === +// === === +// ========================== + +pub(crate) fn log_level() -> u8 { + return 1; +} + +pub(crate) fn max_threads() -> u32 { + let total_threads = thread::available_parallelism().unwrap().get() as u32; + if total_threads >= 32 { + return total_threads - 4; + } else if total_threads >= 12 { + return total_threads - 2; + } else if total_threads >= 3 { + return total_threads - 1; + } else { + return total_threads; + } +} + +pub(crate) fn max_jobs() -> u32 { + return 1; +} + +pub(crate) fn volumes() -> HashMap { + return HashMap::new(); +} + +pub(crate) fn job_threads() -> u32 { + return max_threads(); +} diff --git a/src/main.rs b/src/main.rs index 3a5e61d..f3f6418 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,12 @@ use crate::cli::*; +use crate::data::*; use alphanumeric_sort::sort_str_slice; use clap::{CommandFactory, Parser}; use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh}; -use std::fs; use std::io::stdout; -use yaml_rust2::YamlLoader; + mod cli; +mod data; mod tests; fn main() { @@ -29,14 +30,15 @@ fn main() { generate(PowerShell, &mut Cli::command(), binary_name, &mut stdout()); } }, - Commands::Run { config} => { - println!("{}", config); + Commands::Run { config } => { + println!("Config path: {}", config); + run(config); } } } fn run(config_path: String) { - let tmp = fs::read_to_string(config_path.as_str()).unwrap(); - let config = YamlLoader::load_from_str(tmp.as_str()).unwrap()[0].clone(); - println!("{:?}", config) + let config = config_from_file(config_path); + + println!("{:?}", config); } diff --git a/src/tests.rs b/src/tests.rs index b99ceb1..c3cdf2c 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,7 +1,7 @@ use alphanumeric_sort::sort_str_slice; #[test] -/// This isn't to test the program, more to test the crate works how I want, especially if I switch crates +/// There sorting tests aren't to test the program, more to test the crate works how I want, especially if I switch crates fn test_semver_sorting() { // copied from https://pkgs.org/download/xorg-x11-xauth let mut versions = [