add reading config, add and update docs

Cargo.lock

@@ -2,18 +2,6 @@
 # It is not intended for manual editing.
 version = 4
 
-[[package]]
-name = "ahash"
-version = "0.8.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
-dependencies = [
- "cfg-if",
- "once_cell",
- "version_check",
- "zerocopy",
-]
-
 [[package]]
 name = "alphanumeric-sort"
 version = "1.5.3"
@@ -70,16 +58,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "arraydeque"
+name = "anyhow"
-version = "0.5.1"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236"
+checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
@@ -137,13 +119,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 
 [[package]]
-name = "encoding_rs"
+name = "equivalent"
-version = "0.8.35"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
-dependencies = [
- "cfg-if",
-]
 
 [[package]]
 name = "gregory"
@@ -152,26 +131,15 @@ dependencies = [
  "alphanumeric-sort",
  "clap",
  "clap_complete",
- "yaml-rust2",
+ "serde",
+ "serde_yml",
 ]
 
 [[package]]
 name = "hashbrown"
-version = "0.14.5"
+version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
-dependencies = [
- "ahash",
-]
-
-[[package]]
-name = "hashlink"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
-dependencies = [
- "hashbrown",
-]
 
 [[package]]
 name = "heck"
@@ -179,6 +147,16 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "indexmap"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.1"
@@ -186,10 +164,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
 [[package]]
-name = "once_cell"
+name = "itoa"
-version = "1.20.2"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
+checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
+
+[[package]]
+name = "libyml"
+version = "0.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3302702afa434ffa30847a83305f0a69d6abd74293b6554c18ec85c7ef30c980"
+dependencies = [
+ "anyhow",
+ "version_check",
+]
+
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
 [[package]]
 name = "proc-macro2"
@@ -209,6 +203,47 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "ryu"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+
+[[package]]
+name = "serde"
+version = "1.0.216"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.216"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_yml"
+version = "0.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "libyml",
+ "memchr",
+ "ryu",
+ "serde",
+ "version_check",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -217,9 +252,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.90"
+version = "2.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
+checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -316,34 +351,3 @@ name = "windows_x86_64_msvc"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
-
-[[package]]
-name = "yaml-rust2"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a1a1c0bc9823338a3bdf8c61f994f23ac004c6fa32c08cd152984499b445e8d"
-dependencies = [
- "arraydeque",
- "encoding_rs",
- "hashlink",
-]
-
-[[package]]
-name = "zerocopy"
-version = "0.7.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
-dependencies = [
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.7.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]

Cargo.toml

@@ -7,7 +7,8 @@ edition = "2021"
 alphanumeric-sort = "1.5.3"
 clap = { version = "4.5.23", features = ["derive"] }
 clap_complete = "4.5.40"
-yaml-rust2 = "0.9.0"
+serde = { version = "1.0.216", features = ["derive"] }
+serde_yml = "0.0.12"
 
 [profile.release]
 opt-level = 3

docs/config-reference.md

@@ -8,9 +8,12 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr
 ## Top-level config
 
 - `log-level`: Log level `0`-`3` (error, warning, info, or debug)
+  - Default: 1 - warning
 - `max-threads`: The maximum number of threads to be used
   - **See also**: [`threads`](#job-config)
+  - Default is CPU's threads - 2
 - `max-jobs`: The maximum number of jobs to be run at once
+  - Default is 1
 
 **Multithreading notes (IMPORTANT)**: Gregory will first run compilation jobs, then packaging jobs for whatever programs are done, then run the `update-repo` for whichever distros are finished. For this reason, the distro names listed under `packaging` and `update-repo` *must* match.
 
@@ -21,8 +24,8 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr
   - See `--cpus` in the [`podman run` docs](https://docs.podman.io/en/latest/markdown/podman-run.1.html#cpus)
   - *Root may be required for this argument*
   - If not specified, it will fall back to `max-threads`
-- `image`: The Docker image to run the job in
+- `image`: The Docker image to run the job in *(required)*
-- `commands`: The commands to run
+- `commands`: The commands to run *(required)*
   - TODO: Add command file/bash script instead
 - `volumes`: Names of volumes as defined in [`volumes` (top level)](#volumes)
 

docs/default-max-threads.md

@@ -0,0 +1,38 @@
+# Default `max-threads`
+
+The default `max-threads` uses a simple algorithm to get how many threads.
+
+```rs
+if total_threads >= 32 {
+    return total_threads - 4;
+} else if total_threads >= 12 {
+    return total_threads - 2;
+} else if total_threads >= 3 {
+    return total_threads - 1;
+} else {
+    return total_threads;
+}
+```
+
+i.e. with `total_threads` as the number of threads the CPU(s) has:
+
+- If the CPU has 32 or more threads, it will use all but 4 threads
+- If the CPU has 12 or more threads, it will use all but 2 threads
+- If the CPU has 3 or more threads, it will use all but 1 thread
+- Otherwise, i.e. if the CPU has 1 to 2 threads, it will use all threads
+
+---
+
+Alternative algorithms I tried:
+
+```rs
+if total_threads >= 32 {
+    return total_threads - total_threads.div_ceil(10);
+} else if total_threads >= 12 {
+    return total_threads - 2;
+} else if total_threads >= 3 {
+    return total_threads - 1;
+} else {
+    return total_threads;
+}
+```

src/README.md

@@ -8,6 +8,8 @@ This is Gregory. Gregory controls repos. Gregory keeps track of updating repos,
 - Add support for loading scripts rather than listing commands
 - Add multithreading
 - Add better/custom grouping for when to run jobs (dependency system?)
+- Add dependency system (automatic detection?)
+- Add hook system
 
 ## Other stuff
 

src/benchmark.rs

@@ -0,0 +1,35 @@
+use std::time::Instant;
+
+fn log_thing(total_threads: u32) -> u32 {
+    return total_threads - (f64::log(total_threads.into(), 5.0).round() as u32);
+}
+
+fn my_thing(total_threads: u32) -> u32 {
+    if total_threads >= 32 {
+        return total_threads - 4;
+    } else if total_threads >= 12 {
+        return total_threads - 2;
+    } else if total_threads >= 3 {
+        return total_threads - 1;
+    } else {
+        return total_threads;
+    }
+
+    //println!("{}", max_threads)
+}
+
+//     let mut total_threads = thread::available_parallelism().unwrap().get() as u32;
+
+fn main() {
+    /*
+    let now = Instant::now();
+    for _ in 0..100000000 {
+
+    }
+    let elapsed = now.elapsed();
+    */
+    let total_threads: u32 = 128;
+    println!("{}", log_thing(total_threads));
+    println!("{}", my_thing(total_threads));
+    //println!("{}", elapsed.as_nanos());
+}

src/data.rs

@@ -0,0 +1,91 @@
+//! Datasets used by gregory and stuff for handling them
+
+use serde::Deserialize;
+use std::{collections::HashMap, fs, thread};
+
+/// The config for gregory
+#[derive(Debug, Clone, Deserialize)]
+pub(crate) struct Config {
+    /// What level to log at
+    ///
+    /// - 0: Error
+    /// - 1: Warning
+    /// - 2: Info
+    /// - 3: Debug
+    #[serde(default = "log_level", rename = "log-level")]
+    log_level: u8,
+    /// Maximum number of jobs to run simultaneously
+    #[serde(default = "max_jobs", rename = "max-jobs")]
+    max_jobs: u32,
+    /// Maximum number of threads to use
+    #[serde(default = "max_threads", rename = "max-threads")]
+    max_threads: u32,
+    /// Holds the packages, including their compilation and packaging
+    /// 
+    /// Format: { "librewolf": Package { compilation, packaging } }
+    /// 
+    /// See [`Package`] for details
+    packages: HashMap<String, Package>,
+    #[serde(rename = "update-repo")]
+    update_repo: HashMap<String, Job>,
+    #[serde(default = "volumes")]
+    volumes: HashMap<String, String>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub(crate) struct Job {
+    #[serde(default = "job_threads")]
+    /// How many threads to limit this job to; recommended to set it to the max threads the job will use
+    /// 
+    /// If `threads` isn't specified, it will fall back to `max_threads` (from [`Config`]); the same behavior applies if `threads` is greater than `max_threads`
+    threads: u32,
+    image: String,
+    commands: Vec<String>,
+    volumes: Option<Vec<String>>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub(crate) struct Package {
+    compilation: Option<Job>,
+    packaging: HashMap<String, Job>,
+}
+
+pub(crate) fn config_from_file(filename: String) -> Config {
+    let yaml: Config = serde_yml::from_str(fs::read_to_string(filename).unwrap().as_str()).unwrap();
+    return yaml;
+}
+
+// ==========================
+// ===                    ===
+// ===    ↓ DEFAULTS ↓    ===
+// ===                    ===
+// ==========================
+
+pub(crate) fn log_level() -> u8 {
+    return 1;
+}
+
+pub(crate) fn max_threads() -> u32 {
+    let total_threads = thread::available_parallelism().unwrap().get() as u32;
+    if total_threads >= 32 {
+        return total_threads - 4;
+    } else if total_threads >= 12 {
+        return total_threads - 2;
+    } else if total_threads >= 3 {
+        return total_threads - 1;
+    } else {
+        return total_threads;
+    }
+}
+
+pub(crate) fn max_jobs() -> u32 {
+    return 1;
+}
+
+pub(crate) fn volumes() -> HashMap<String, String> {
+    return HashMap::new();
+}
+
+pub(crate) fn job_threads() -> u32 {
+    return max_threads();
+}

src/main.rs

@@ -1,11 +1,12 @@
 use crate::cli::*;
+use crate::data::*;
 use alphanumeric_sort::sort_str_slice;
 use clap::{CommandFactory, Parser};
 use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh};
-use std::fs;
 use std::io::stdout;
-use yaml_rust2::YamlLoader;
+
 mod cli;
+mod data;
 mod tests;
 
 fn main() {
@@ -30,13 +31,14 @@ fn main() {
             }
         },
         Commands::Run { config } => {
-            println!("{}", config);
+            println!("Config path: {}", config);
+            run(config);
         }
     }
 }
 
 fn run(config_path: String) {
-    let tmp = fs::read_to_string(config_path.as_str()).unwrap();
+    let config = config_from_file(config_path);
-    let config = YamlLoader::load_from_str(tmp.as_str()).unwrap()[0].clone();
+
-    println!("{:?}", config)
+    println!("{:?}", config);
 }

src/tests.rs

@@ -1,7 +1,7 @@
 use alphanumeric_sort::sort_str_slice;
 
 #[test]
-/// This isn't to test the program, more to test the crate works how I want, especially if I switch crates
+/// There sorting tests aren't to test the program, more to test the crate works how I want, especially if I switch crates
 fn test_semver_sorting() {
     // copied from https://pkgs.org/download/xorg-x11-xauth
     let mut versions = [