diff --git a/.gitignore b/.gitignore index ea8c4bf..f6c313c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +/data \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..1b736c3 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,49 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'gregory'", + "cargo": { + "args": [ + "build", + "--bin=gregory", + "--package=gregory", + ], + "filter": { + "name": "gregory", + "kind": "bin" + } + }, + "args": [ + "run", + "-c", + "gregory.example.yml" + ], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'gregory'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=gregory", + "--package=gregory" + ], + "filter": { + "name": "gregory", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 2ef1402..c18fa69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,6 +63,18 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "clap" version = "4.5.23" @@ -124,15 +136,27 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gregory" -version = "0.1.0" +version = "0.1.1" dependencies = [ "alphanumeric-sort", "clap", "clap_complete", "serde", "serde_yml", + "uuid", ] [[package]] @@ -169,6 +193,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + [[package]] name = "libyml" version = "0.0.5" @@ -185,6 +215,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.92" @@ -203,6 +242,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "ryu" version = "1.0.18" @@ -273,12 +342,28 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +dependencies = [ + "getrandom", + "rand", +] + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "windows-sys" version = "0.59.0" @@ -351,3 +436,24 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 1f102c3..6bd0ba4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "gregory" -version = "0.1.0" +version = "0.1.1" edition = "2021" +license = "GPL-3.0-only" [dependencies] alphanumeric-sort = "1.5.3" @@ -9,6 +10,7 @@ clap = { version = "4.5.23", features = ["derive"] } clap_complete = "4.5.40" serde = { version = "1.0.216", features = ["derive"] } serde_yml = "0.0.12" +uuid = { version = "1.11.0", features = ["v7", "fast-rng"] } [profile.release] opt-level = 3 diff --git a/README.md b/README.md index 2bca48b..db855f6 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ This is Gregory. Gregory controls repos. Gregory keeps track of updating repos, trying to be simple and elegant, but enough. +## THIS IS A PROTOTYPE + ## Documentation Go look at [`docs/`](/docs/) @@ -14,6 +16,7 @@ Go look at [`docs/`](/docs/) - Add better/custom grouping for when to run jobs (dependency system?) - Add dependency system (automatic detection?) - Add hook system +- Add SQL database (maybe using `sqlx`?) ## Other stuff diff --git a/docs/behind-the-scenes/commands.md b/docs/behind-the-scenes/commands.md new file mode 100644 index 0000000..674e5d9 --- /dev/null +++ b/docs/behind-the-scenes/commands.md @@ -0,0 +1,3 @@ +# How commands are run + +I was unable to find a way to directly run *multiple* commands via Docker/Podman. Instead of doing that, greg puts all the commands in a temporary script, mounts it inside, and then run it with \ No newline at end of file diff --git a/docs/default-max-threads.md b/docs/behind-the-scenes/default-max-threads.md similarity index 100% rename from docs/default-max-threads.md rename to docs/behind-the-scenes/default-max-threads.md diff --git a/docs/config-reference.md b/docs/config-reference.md index 400728c..e59c5d6 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -14,6 +14,8 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr - Default is CPU's threads - 2 - `max-jobs` (integer): The maximum number of jobs to be run at once - Default is 1 +- `data-dir` (string): The path to put data for job logs and stuff + - **Temporary**, will be removed once SQL database support is added **Multithreading notes (IMPORTANT)**: Gregory will first run compilation jobs, then packaging jobs for whatever programs are done, then run the `update-repo` for whichever distros are finished. For this reason, the distro names listed under `packaging` and `update-repo` *must* match. @@ -21,16 +23,23 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr ## Job config +- `id` (string): An ID to identify the job, such as the compilation of a program **(highly recommended)** + - Default is `-1` for unassigned + - If you just want to run stuff, you don't need this, but it's *highly* recommended as it allows you to filter your logs. +- `revision` (string): A revision id for the job, such as a version number for a compilation script + - Default is `1` - `threads` (integer): The maximum number of vCPUs/threads to dedicate to a job; this can be a fractional number - Set this as less than or equal to the max number of threads the thing you're running will use - See `--cpus` in the [`podman run` docs](https://docs.podman.io/en/latest/markdown/podman-run.1.html#cpus) - *Root may be required for this argument* - If not specified, it will fall back to `max-threads` -- `image` (string): The Docker image to run the job in *(required)* -- `commands` (sequence): The commands to run *(required)* +- `image` (string): The Docker image to run the job in **(required)** +- `commands` (sequence): The commands to run **(required)** - TODO: Add command file/bash script instead - `volumes` (sequence): Names of volumes as defined in [`volumes` (top level)](#volumes) - `privileged` (bool): Whether the job's container should be privileged +- `shell` (string): The shell to run the commands in + - Default: `/bin/sh` ## Packages (`packages`) diff --git a/gregory.example.yml b/gregory.example.yml index 084b1b1..dc77bae 100644 --- a/gregory.example.yml +++ b/gregory.example.yml @@ -5,11 +5,13 @@ log-level: 0 packages: librewolf: compilation: + id: 1 + revision: 2 threads: 8 image: 'docker.io/library/debian' commands: - - 'cd ~/librewolf' - - './mach build' + - 'echo hi' + - 'echo helloooooooooo' volumes: - 'librewolf' packaging: @@ -17,8 +19,8 @@ packages: threads: 8 image: 'docker.io/library/fedora' commands: - - 'git clone http://example.com/librewolf-fedora-packaging.git && cd librewolf-fedora-packaging/' - - 'do-rpm-stuff-idk' + - 'echo did you ever hear the tragedy of darth plageuis the wise?' + - "echo it's not a story the jedi would tell you" volumes: - 'librewolf' @@ -27,7 +29,7 @@ update-repo: threads: 4 image: 'docker.io/library/fedora' commands: - - 'idkkkkk' + - 'echo hai' volumes: - 'librewolf' diff --git a/src/data.rs b/src/data.rs index 80f707a..5f728ac 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,6 +1,7 @@ //! Data structs. used by gregory and stuff for handling them use serde::Deserialize; +use std::time; use std::{collections::HashMap, fs, thread}; /// The config for gregory @@ -12,57 +13,88 @@ pub(crate) struct Config { /// - 1: Warning /// - 2: Info /// - 3: Debug - #[serde(default = "log_level", rename = "log-level")] // the rename lets it use `log-level` instead in the yaml file - this is not an alias, `log_level` in the yaml will *not* work - log_level: u8, + #[serde(default = "log_level", rename = "log-level")] + // the rename lets it use `log-level` instead in the yaml file - this is not an alias, `log_level` in the yaml will *not* work + pub(crate) log_level: u8, /// Maximum number of jobs to run simultaneously #[serde(default = "max_jobs", rename = "max-jobs")] - max_jobs: u32, + pub(crate) max_jobs: u32, /// Maximum number of threads to use #[serde(default = "max_threads", rename = "max-threads")] - max_threads: u32, + pub(crate) max_threads: u32, + #[serde(default = "data", rename = "data-dir")] + pub(crate) data_dir: String, /// Holds the packages, including their compilation and packaging /// /// Format: `{ "librewolf": Package { compilation, packaging } }` /// /// See [`Package`] for details - packages: HashMap, + pub(crate) packages: HashMap, /// The jobs for updating the repo, organized by distro/repo name #[serde(rename = "update-repo")] - update_repo: HashMap, + pub(crate) update_repo: HashMap, /// All volumes, organized like this: /// /// Format: `{ "librewolf": "./data/librewolf:/librewolf" }` - like Docker/Podman formatting #[serde(default = "volumes")] - volumes: HashMap, + pub(crate) volumes: HashMap, } #[derive(Debug, Clone, Deserialize)] pub(crate) struct Job { + /// An ID to identify the job, such as the compilation of a program + #[serde(default = "id")] + pub(crate) id: String, + #[serde(default = "revision")] + pub(crate) revision: String, /// How many threads to limit this job to; recommended to set it to the max threads the job will use /// /// If `threads` isn't specified, it will fall back to `max_threads` (from [`Config`]); the same behavior applies if `threads` is greater than `max_threads` #[serde(default = "job_threads")] - threads: u32, - /// The OCi image to run it in + pub(crate) threads: u32, + /// The OCI image to run it in /// /// For example, `docker.io/library/debian:latest` - image: String, + pub(crate) image: String, /// The commands to run in the job - commands: Vec, - volumes: Option>, - /// Whether the job should be privileged + pub(crate) commands: Vec, + /// A list of all volumes given their name - see [`Config`] -> `volumes` + pub(crate) volumes: Option>, + /// Whether the job W be privileged /// /// Defauolt: false #[serde(default = "privileged")] - privileged: bool, + pub(crate) privileged: bool, + #[serde(default = "shell")] + pub(crate) shell: String, } +/// Holds the data for a certain package's config #[derive(Debug, Clone, Deserialize)] pub(crate) struct Package { /// The compilation [`Job`] - optional - compilation: Option, + pub(crate) compilation: Option, /// The packaging [`Job`]s, organized by the distro/repo name - packaging: HashMap, + pub(crate) packaging: HashMap, +} + +/// The exit status and stuff for a [`Job`] +#[derive(Debug, Clone)] +pub(crate) struct JobExitStatus { + pub(crate) job: Job, + /// The [`Job`] this status is from + /// + /// This is stored as a u16 rather than a u8 so that 65535 can be returned if there is no exit code rather than doing an Option or something, which I fear will probably come back to haunt me, but whatever + pub(crate) exit_code: u16, + /// Where the log is + /// + /// TEMPORARY + /// TODO: Have main() handle logs and writing them to the database, not doing it in run_job() + pub(crate) log_path: String, + /// How long it took to run the job + pub(crate) duration: time::Duration, + /// The name of the container this job ran in + pub(crate) container_name: String, } pub(crate) fn config_from_file(filename: String) -> Config { @@ -114,3 +146,22 @@ pub(crate) fn job_threads() -> u32 { pub(crate) fn privileged() -> bool { return false; } + +/// Default (`/bin/sh`) for which shell to use +pub(crate) fn shell() -> String { + return "/bin/sh".to_string(); +} + +/// Default id (`-1`) +pub(crate) fn id() -> String { + return "-1".to_string(); +} + +/// Default revision (`1`) +pub(crate) fn revision() -> String { + return "1".to_string(); +} + +pub(crate) fn data() -> String { + return "./data".to_string(); +} diff --git a/src/main.rs b/src/main.rs index 05a0544..3baa4ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,16 @@ use crate::data::*; use alphanumeric_sort::sort_str_slice; use clap::{CommandFactory, Parser}; use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh}; +use std::fs; +use std::fs::create_dir_all; +use std::fs::write; +use std::fs::File; use std::io::stdout; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::Command; +use std::time::Instant; +use uuid::Uuid; mod cli; mod data; @@ -40,9 +49,75 @@ fn main() { fn run(config_path: String) { let config = config_from_file(config_path); - println!("{:?}", config); + println!("{:#?}", config); + + let mut jobs: Vec = Vec::new(); + + for (_, package) in config.clone().packages { + match package.compilation { + Some(tmp) => { + jobs.push(tmp); + } + None => {} + } + + for (_, job) in package.packaging { + jobs.push(job); + } + } + + for (_, job) in config.clone().update_repo { + jobs.push(job); + } + + for job in jobs { + println!("{:#?}", run_job(config.clone(), job)); + } } -fn run_job(max_threads: u32, job: Job) { - -} \ No newline at end of file +fn run_job(conf: Config, job: Job) -> JobExitStatus { + // limit threads to max_threads in the config + let mut threads: u32 = job.threads; + if job.threads > conf.max_threads { + threads = conf.max_threads; + } + + let container_name: String = format!("gregory-{}-{}-{}", job.id, job.revision, Uuid::now_v7()); + + let log_path = &format!("{}/logs/{container_name}.txt", conf.data_dir); // can't select fields in the format!() {} thing, have to do this + let log_dir: &Path = Path::new(log_path); + create_dir_all(log_dir.parent().unwrap()).unwrap(); + write(log_path, job.commands.join("\n")).unwrap(); + + // set permissions - *unix specific* + let mut perms = File::open(log_path) + .unwrap() + .metadata() + .unwrap() + .permissions(); + PermissionsExt::set_mode(&mut perms, 0o755); + + let now = Instant::now(); + let cmd_args: Vec = vec![ + "run".to_string(), + format!("--name={container_name}"), + format!("--cpus={threads}"), + format!("--privileged={}", job.privileged), + format!("-v={log_path}:/gregory-entrypoint.sh"), + format!("--entrypoint=['{}', '/gregory-entrypoint.sh']", &job.shell), + job.clone().image + ]; + let cmd_output = Command::new("podman").args(cmd_args).output().unwrap(); + let elapsed = now.elapsed(); + + + println!("{:?}", cmd_output); + + return JobExitStatus { + container_name: container_name, + duration: elapsed, + job: job, + exit_code: cmd_output.status.code().ok_or_else(|| 65535).unwrap() as u16, + log_path: log_path.clone(), + }; +}