add reading config, add and update docs
This commit is contained in:
askiiart
parent
e9af17baf0
commit
bd880f4f5f
GPG key ID:
EA85979611654C30
9 changed files with 265 additions and 89 deletions
160
Cargo.lock
generated
160
Cargo.lock
generated
|
|
@ -2,18 +2,6 @@
|
|||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alphanumeric-sort"
|
||||
version = "1.5.3"
|
||||
|
|
@ -70,16 +58,10 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "arraydeque"
|
||||
version = "0.5.1"
|
||||
name = "anyhow"
|
||||
version = "1.0.95"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
|
|
@ -137,13 +119,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
name = "equivalent"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
|
||||
|
||||
[[package]]
|
||||
name = "gregory"
|
||||
|
|
@ -152,26 +131,15 @@ dependencies = [
|
|||
"alphanumeric-sort",
|
||||
"clap",
|
||||
"clap_complete",
|
||||
"yaml-rust2",
|
||||
"serde",
|
||||
"serde_yml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
version = "0.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
|
||||
dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
|
|
@ -179,6 +147,16 @@ version = "0.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.1"
|
||||
|
|
@ -186,10 +164,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.20.2"
|
||||
name = "itoa"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
|
||||
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
|
||||
|
||||
[[package]]
|
||||
name = "libyml"
|
||||
version = "0.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3302702afa434ffa30847a83305f0a69d6abd74293b6554c18ec85c7ef30c980"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
|
|
@ -209,6 +203,47 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.216"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.216"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_yml"
|
||||
version = "0.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"libyml",
|
||||
"memchr",
|
||||
"ryu",
|
||||
"serde",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
|
|
@ -217,9 +252,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.90"
|
||||
version = "2.0.91"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
|
||||
checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -316,34 +351,3 @@ name = "windows_x86_64_msvc"
|
|||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "yaml-rust2"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a1a1c0bc9823338a3bdf8c61f994f23ac004c6fa32c08cd152984499b445e8d"
|
||||
dependencies = [
|
||||
"arraydeque",
|
||||
"encoding_rs",
|
||||
"hashlink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ edition = "2021"
|
|||
alphanumeric-sort = "1.5.3"
|
||||
clap = { version = "4.5.23", features = ["derive"] }
|
||||
clap_complete = "4.5.40"
|
||||
yaml-rust2 = "0.9.0"
|
||||
serde = { version = "1.0.216", features = ["derive"] }
|
||||
serde_yml = "0.0.12"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
|
|
|
|||
|
|
@ -8,9 +8,12 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr
|
|||
## Top-level config
|
||||
|
||||
- `log-level`: Log level `0`-`3` (error, warning, info, or debug)
|
||||
- Default: 1 - warning
|
||||
- `max-threads`: The maximum number of threads to be used
|
||||
- **See also**: [`threads`](#job-config)
|
||||
- Default is CPU's threads - 2
|
||||
- `max-jobs`: The maximum number of jobs to be run at once
|
||||
- Default is 1
|
||||
|
||||
**Multithreading notes (IMPORTANT)**: Gregory will first run compilation jobs, then packaging jobs for whatever programs are done, then run the `update-repo` for whichever distros are finished. For this reason, the distro names listed under `packaging` and `update-repo` *must* match.
|
||||
|
||||
|
|
@ -21,8 +24,8 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr
|
|||
- See `--cpus` in the [`podman run` docs](https://docs.podman.io/en/latest/markdown/podman-run.1.html#cpus)
|
||||
- *Root may be required for this argument*
|
||||
- If not specified, it will fall back to `max-threads`
|
||||
- `image`: The Docker image to run the job in
|
||||
- `commands`: The commands to run
|
||||
- `image`: The Docker image to run the job in *(required)*
|
||||
- `commands`: The commands to run *(required)*
|
||||
- TODO: Add command file/bash script instead
|
||||
- `volumes`: Names of volumes as defined in [`volumes` (top level)](#volumes)
|
||||
|
||||
|
|
|
|||
38
docs/default-max-threads.md
Normal file
38
docs/default-max-threads.md
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# Default `max-threads`
|
||||
|
||||
The default `max-threads` uses a simple algorithm to get how many threads.
|
||||
|
||||
```rs
|
||||
if total_threads >= 32 {
|
||||
return total_threads - 4;
|
||||
} else if total_threads >= 12 {
|
||||
return total_threads - 2;
|
||||
} else if total_threads >= 3 {
|
||||
return total_threads - 1;
|
||||
} else {
|
||||
return total_threads;
|
||||
}
|
||||
```
|
||||
|
||||
i.e. with `total_threads` as the number of threads the CPU(s) has:
|
||||
|
||||
- If the CPU has 32 or more threads, it will use all but 4 threads
|
||||
- If the CPU has 12 or more threads, it will use all but 2 threads
|
||||
- If the CPU has 3 or more threads, it will use all but 1 thread
|
||||
- Otherwise, i.e. if the CPU has 1 to 2 threads, it will use all threads
|
||||
|
||||
---
|
||||
|
||||
Alternative algorithms I tried:
|
||||
|
||||
```rs
|
||||
if total_threads >= 32 {
|
||||
return total_threads - total_threads.div_ceil(10);
|
||||
} else if total_threads >= 12 {
|
||||
return total_threads - 2;
|
||||
} else if total_threads >= 3 {
|
||||
return total_threads - 1;
|
||||
} else {
|
||||
return total_threads;
|
||||
}
|
||||
```
|
||||
|
|
@ -8,6 +8,8 @@ This is Gregory. Gregory controls repos. Gregory keeps track of updating repos,
|
|||
- Add support for loading scripts rather than listing commands
|
||||
- Add multithreading
|
||||
- Add better/custom grouping for when to run jobs (dependency system?)
|
||||
- Add dependency system (automatic detection?)
|
||||
- Add hook system
|
||||
|
||||
## Other stuff
|
||||
|
||||
|
|
|
|||
35
src/benchmark.rs
Normal file
35
src/benchmark.rs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
use std::time::Instant;
|
||||
|
||||
fn log_thing(total_threads: u32) -> u32 {
|
||||
return total_threads - (f64::log(total_threads.into(), 5.0).round() as u32);
|
||||
}
|
||||
|
||||
fn my_thing(total_threads: u32) -> u32 {
|
||||
if total_threads >= 32 {
|
||||
return total_threads - 4;
|
||||
} else if total_threads >= 12 {
|
||||
return total_threads - 2;
|
||||
} else if total_threads >= 3 {
|
||||
return total_threads - 1;
|
||||
} else {
|
||||
return total_threads;
|
||||
}
|
||||
|
||||
//println!("{}", max_threads)
|
||||
}
|
||||
|
||||
// let mut total_threads = thread::available_parallelism().unwrap().get() as u32;
|
||||
|
||||
fn main() {
|
||||
/*
|
||||
let now = Instant::now();
|
||||
for _ in 0..100000000 {
|
||||
|
||||
}
|
||||
let elapsed = now.elapsed();
|
||||
*/
|
||||
let total_threads: u32 = 128;
|
||||
println!("{}", log_thing(total_threads));
|
||||
println!("{}", my_thing(total_threads));
|
||||
//println!("{}", elapsed.as_nanos());
|
||||
}
|
||||
91
src/data.rs
Normal file
91
src/data.rs
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
//! Datasets used by gregory and stuff for handling them
|
||||
|
||||
use serde::Deserialize;
|
||||
use std::{collections::HashMap, fs, thread};
|
||||
|
||||
/// The config for gregory
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub(crate) struct Config {
|
||||
/// What level to log at
|
||||
///
|
||||
/// - 0: Error
|
||||
/// - 1: Warning
|
||||
/// - 2: Info
|
||||
/// - 3: Debug
|
||||
#[serde(default = "log_level", rename = "log-level")]
|
||||
log_level: u8,
|
||||
/// Maximum number of jobs to run simultaneously
|
||||
#[serde(default = "max_jobs", rename = "max-jobs")]
|
||||
max_jobs: u32,
|
||||
/// Maximum number of threads to use
|
||||
#[serde(default = "max_threads", rename = "max-threads")]
|
||||
max_threads: u32,
|
||||
/// Holds the packages, including their compilation and packaging
|
||||
///
|
||||
/// Format: { "librewolf": Package { compilation, packaging } }
|
||||
///
|
||||
/// See [`Package`] for details
|
||||
packages: HashMap<String, Package>,
|
||||
#[serde(rename = "update-repo")]
|
||||
update_repo: HashMap<String, Job>,
|
||||
#[serde(default = "volumes")]
|
||||
volumes: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub(crate) struct Job {
|
||||
#[serde(default = "job_threads")]
|
||||
/// How many threads to limit this job to; recommended to set it to the max threads the job will use
|
||||
///
|
||||
/// If `threads` isn't specified, it will fall back to `max_threads` (from [`Config`]); the same behavior applies if `threads` is greater than `max_threads`
|
||||
threads: u32,
|
||||
image: String,
|
||||
commands: Vec<String>,
|
||||
volumes: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub(crate) struct Package {
|
||||
compilation: Option<Job>,
|
||||
packaging: HashMap<String, Job>,
|
||||
}
|
||||
|
||||
pub(crate) fn config_from_file(filename: String) -> Config {
|
||||
let yaml: Config = serde_yml::from_str(fs::read_to_string(filename).unwrap().as_str()).unwrap();
|
||||
return yaml;
|
||||
}
|
||||
|
||||
// ==========================
|
||||
// === ===
|
||||
// === ↓ DEFAULTS ↓ ===
|
||||
// === ===
|
||||
// ==========================
|
||||
|
||||
pub(crate) fn log_level() -> u8 {
|
||||
return 1;
|
||||
}
|
||||
|
||||
pub(crate) fn max_threads() -> u32 {
|
||||
let total_threads = thread::available_parallelism().unwrap().get() as u32;
|
||||
if total_threads >= 32 {
|
||||
return total_threads - 4;
|
||||
} else if total_threads >= 12 {
|
||||
return total_threads - 2;
|
||||
} else if total_threads >= 3 {
|
||||
return total_threads - 1;
|
||||
} else {
|
||||
return total_threads;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn max_jobs() -> u32 {
|
||||
return 1;
|
||||
}
|
||||
|
||||
pub(crate) fn volumes() -> HashMap<String, String> {
|
||||
return HashMap::new();
|
||||
}
|
||||
|
||||
pub(crate) fn job_threads() -> u32 {
|
||||
return max_threads();
|
||||
}
|
||||
14
src/main.rs
14
src/main.rs
|
|
@ -1,11 +1,12 @@
|
|||
use crate::cli::*;
|
||||
use crate::data::*;
|
||||
use alphanumeric_sort::sort_str_slice;
|
||||
use clap::{CommandFactory, Parser};
|
||||
use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh};
|
||||
use std::fs;
|
||||
use std::io::stdout;
|
||||
use yaml_rust2::YamlLoader;
|
||||
|
||||
mod cli;
|
||||
mod data;
|
||||
mod tests;
|
||||
|
||||
fn main() {
|
||||
|
|
@ -30,13 +31,14 @@ fn main() {
|
|||
}
|
||||
},
|
||||
Commands::Run { config } => {
|
||||
println!("{}", config);
|
||||
println!("Config path: {}", config);
|
||||
run(config);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run(config_path: String) {
|
||||
let tmp = fs::read_to_string(config_path.as_str()).unwrap();
|
||||
let config = YamlLoader::load_from_str(tmp.as_str()).unwrap()[0].clone();
|
||||
println!("{:?}", config)
|
||||
let config = config_from_file(config_path);
|
||||
|
||||
println!("{:?}", config);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use alphanumeric_sort::sort_str_slice;
|
||||
|
||||
#[test]
|
||||
/// This isn't to test the program, more to test the crate works how I want, especially if I switch crates
|
||||
/// There sorting tests aren't to test the program, more to test the crate works how I want, especially if I switch crates
|
||||
fn test_semver_sorting() {
|
||||
// copied from https://pkgs.org/download/xorg-x11-xauth
|
||||
let mut versions = [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue