diff --git a/Cargo.lock b/Cargo.lock index e009b27..3a38daf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,21 +29,6 @@ version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d67c60c5f10f11c6ee04de72b2dd98bb9d2548cbc314d22a609bfa8bd9e87e8f" -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anstream" version = "0.6.18" @@ -160,12 +145,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bumpalo" -version = "3.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" - [[package]] name = "byteorder" version = "1.5.0" @@ -178,35 +157,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" -[[package]] -name = "cc" -version = "1.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" -dependencies = [ - "shlex", -] - [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chrono" -version = "0.4.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-targets 0.52.6", -] - [[package]] name = "clap" version = "4.5.27" @@ -277,12 +233,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "cpufeatures" version = "0.2.16" @@ -556,7 +506,6 @@ version = "0.1.1" dependencies = [ "alphanumeric-sort", "better-commands", - "chrono", "clap", "clap_complete", "serde", @@ -626,29 +575,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "iana-time-zone" -version = "0.1.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "icu_collections" version = "1.5.0" @@ -810,16 +736,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" -[[package]] -name = "js-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -1165,12 +1081,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rustversion" -version = "1.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" - [[package]] name = "ryu" version = "1.0.18" @@ -1258,12 +1168,6 @@ dependencies = [ "digest", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -1829,64 +1733,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" -[[package]] -name = "wasm-bindgen" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" -dependencies = [ - "unicode-ident", -] - [[package]] name = "whoami" version = "1.5.2" @@ -1897,15 +1743,6 @@ dependencies = [ "wasite", ] -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index a91d93a..2baf5ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ readme = "README.md" [dependencies] alphanumeric-sort = "1.5.3" better-commands = "1.0.2" -chrono = "0.4.39" clap = { version = "4.5.23", features = ["derive"] } clap_complete = "4.5.40" serde = { version = "1.0.216", features = ["derive"] } diff --git a/README.md b/README.md index 7ea4976..1c545f4 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,17 @@ This is Gregory. Gregory controls repos. Gregory keeps track of updating repos, ## Documentation -Go look at [`docs/`](/docs/), and check out the [example config](/gregory.example.toml) +Go look at [`docs/`](/docs/) ## TODO - Add multithreading - Add hook system +- Add SQL database (maybe using `sqlx`?) + - Log stderr too +- Add dependency system + - Automatic detection with features (extensibility?) for each distro? +- Add podman errors ## Other stuff @@ -17,4 +22,3 @@ Go look at [`docs/`](/docs/), and check out the [example config](/gregory.exampl - Why the name? - I was thinking to go with something dark and foreboding, since this is a program to control *everything* about a repo - it's the high command. But I couldn't think of anything and thought just naming it some lame random name instead would be way funnier. Hence, Gregory. - Gregory is a program, so it uses it/its pronouns. It also doesn't mind whether you capitalize its name or not, "gregory" or "Gregory" are fine, you can even shorten it if you want. -- It's built for updating package repositories, but can be used to run pretty much anything. This isn't to say support won't be offered unless you're using it for a repo, but development will be focused on updating repos. diff --git a/dev-setup.sh b/dev-setup.sh index 8b0f914..e16fc8f 100755 --- a/dev-setup.sh +++ b/dev-setup.sh @@ -1,9 +1,4 @@ #!/usr/bin/env bash -set -ex - -command_exists() { type "$1" &>/dev/null; } - -./test.sh rm -rf ./data/ mkdir -p ./data/{fedora-repo,librewolf,other-workspace} @@ -11,16 +6,8 @@ mkdir -p ./data/{fedora-repo,librewolf,other-workspace} mkdir -p ./dev/{pgadmin,gregory-pg} chmod -R 777 ./dev/pgadmin -if command_exists "docker-compose"; then - docker-compose -f podman-compose.dev.yml down - docker-compose -f podman-compose.dev.yml up -d -elif command_exists "podman-compose"; then - podman-compose -f podman-compose.dev.yml down - podman-compose -f podman-compose.dev.yml up -d -else - echo "[ERROR] neither docker-compose nor podman-compose were found" - exit 127 -fi +podman-compose down +podman-compose -f podman-compose.dev.yml up -d echo " --- @@ -32,4 +19,4 @@ echo 'pgadmin login: echo 'pgadmin settings: Hostname: "postgres" Username: "gregory" - Password: "pass"' + Password: "pass"' \ No newline at end of file diff --git a/docs/database.md b/docs/database.md deleted file mode 100644 index e15a59f..0000000 --- a/docs/database.md +++ /dev/null @@ -1,31 +0,0 @@ -# Database docs - -Gregory's database is described as follows: - -```sql -CREATE TABLE IF NOT EXISTS job_logs ( - start_time timestamp, - end_time timestamp, - duration interval GENERATED ALWAYS AS (end_time - start_time) STORED, - exit_code smallint, - job_id text, - revision text, - uuid text, - container_name text GENERATED ALWAYS AS (job_id || '-' || uuid) STORED, - log_path text -); -``` - -i.e. it uses the table `job_logs`, containing the following fields: - -| start_time | end_time | duration | exit_code | job_id | revision | uuid | container_name | log_path | -| ---------- | -------- | -------- | --------- | ------ | -------- | ---- | -------------- | -------- | - ---- - -`duration` and `container_name` don't have to be inserted, as the database generates them, so they're just inserted like this: - -```rs -INSERT INTO job_logs (start_time, end_time, exit_code, job_id, revision, uuid, log_path) - VALUES ('1970-01-01 10:10:10 idkkkkk', '1970-01-01 10:11:10 idkkkkk', 1, 'packaging.librewolf.compilation', '5', 'blahblahblahblah', './data/logs/packages.librewolf.compilation/5/blahblahblahblah'); -``` diff --git a/gregory.example.toml b/gregory.example.toml index 4c28e68..bf55cd4 100644 --- a/gregory.example.toml +++ b/gregory.example.toml @@ -12,7 +12,7 @@ max-threads = 10 revision = "2" threads = 6 image = "docker.io/library/debian" - commands = ["echo hi", "sleep 2.432", "echo helloooooooooo"] + commands = ["echo hi", "echo helloooooooooo"] volumes = ["librewolf"] [packages.librewolf.packaging.fedora] diff --git a/src/data.rs b/src/data.rs index 47e7649..b5cf0a1 100644 --- a/src/data.rs +++ b/src/data.rs @@ -32,22 +32,6 @@ pub(crate) struct Config { pub(crate) volumes: HashMap, } -impl Config { - pub(crate) fn from_file(filename: String) -> Result { - match fs::read_to_string(filename) { - Ok(raw_data) => match toml::from_str(raw_data.as_str()) { - Ok(conf) => return Ok(conf), - Err(e) => { - return Err(Error::DeserError(e)); - } - }, - Err(e) => { - return Err(Error::IOError(e)); - } - } - } -} - /// Holds the data for a job #[derive(Debug, Clone, Deserialize)] pub(crate) struct Job { @@ -105,13 +89,26 @@ pub(crate) struct JobExitStatus { /// Where the log is /// /// TEMPORARY + /// TODO: Have main() handle logs and writing them to the database, not doing it in run_job() pub(crate) log_path: String, /// How long it took to run the job pub(crate) duration: time::Duration, /// The name of the container this job ran in pub(crate) container_name: String, - /// Uuid - pub(crate) job_uuid: String +} + +pub(crate) fn config_from_file(filename: String) -> Result { + match fs::read_to_string(filename) { + Ok(raw_data) => match toml::from_str(raw_data.as_str()) { + Ok(conf) => return Ok(conf), + Err(e) => { + return Err(Error::DeserError(e)); + } + }, + Err(e) => { + return Err(Error::IOError(e)); + } + } } // ========================== diff --git a/src/errors.rs b/src/errors.rs index 72ba332..cd638e1 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -6,4 +6,6 @@ pub enum Error { IOError(#[from] std::io::Error), #[error("error while deserializing TOML: {0}")] DeserError(#[from] toml::de::Error), + #[error("Error connecting to database: {0}")] + DbConnectionError(String), } diff --git a/src/logging.rs b/src/logging.rs index 529f3d2..9865b23 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -1,51 +1,65 @@ -use uuid::Uuid; - use crate::errors::Error; -use std::env; -use std::fs::{create_dir_all, File, OpenOptions}; +use std::fs::{File, OpenOptions}; use std::io::Write; -use std::path::Path; -use std::time::{Instant, SystemTime}; + +/// The logger for gregory itself - NOT for jobs +pub(crate) struct Logger { + log_file: File, +} + +impl Logger { + pub(crate) fn new(path: String) -> Result { + match OpenOptions::new().append(true).open(path) { + Ok(f) => return Ok(Logger { log_file: f }), + Err(e) => { + return Err(Error::IOError(e)); + } + } + } + + /// Log a warning + /// + /// Fun gregory lore: I originally typo'd this as "Strign" and the linter didn't catch it for some reason + pub(crate) fn warning(&mut self, text: String) -> Result<(), Error> { + match writeln!(&mut self.log_file, "[WARNING] {}", text) { + Ok(_) => return Ok(()), + Err(e) => { + return Err(Error::IOError(e)); + } + } + } + + /// Log an error + pub(crate) fn error(&mut self, text: String) -> Result<(), Error> { + match writeln!(&mut self.log_file, "[ERROR] {}", text) { + Ok(_) => return Ok(()), + Err(e) => { + return Err(Error::IOError(e)); + } + } + } +} /// Logging for a [`Job`] -// TODO: log to postgres instead; maybe i already made a comment todo-ing this idk pub(crate) struct JobLogger { log_file: File, - path: String, } impl JobLogger { - pub(crate) fn new( - data_dir: String, - job_id: String, - revision: String, - run_id: Uuid, - ) -> JobLogger { - // get path and create the dir. - let log_path = format!("{data_dir}/logs/{job_id}/{revision}/{run_id}"); - let log_dir = Path::new(&log_path).parent().unwrap(); - create_dir_all(log_dir).unwrap(); - - return JobLogger { - log_file: OpenOptions::new() - .create_new(true) - .append(true) - .open(&log_path) - .unwrap(), - path: log_path, - }; + pub(crate) fn new(path: String) -> Result { + match OpenOptions::new().create_new(true).append(true).open(path) { + Ok(f) => return Ok(JobLogger { log_file: f }), + Err(e) => { + return Err(Error::IOError(e)); + } + } } /// Log something printed to stdout /// /// Fun gregory lore: I originally typo'd this as "Strign" and the linter didn't catch it for some reason - pub(crate) fn stdout(&mut self, text: String, start_time: Instant) -> Result<(), Error> { - match writeln!( - &mut self.log_file, - "[{:.3}] [stdout] {}", - start_time.elapsed().as_millis() as f64 / 1000.0, - text - ) { + pub(crate) fn stdout(&mut self, text: String) -> Result<(), Error> { + match writeln!(&mut self.log_file, "[stdout] {}", text) { Ok(_) => return Ok(()), Err(e) => { return Err(Error::IOError(e)); @@ -54,147 +68,12 @@ impl JobLogger { } /// Log something printed to stderr - pub(crate) fn stderr(&mut self, text: String, start_time: Instant) -> Result<(), Error> { - match writeln!( - &mut self.log_file, - "[{}] [stderr] {}", - start_time.elapsed().as_millis() / 1000, - text - ) { + pub(crate) fn stderr(&mut self, text: String) -> Result<(), Error> { + match writeln!(&mut self.log_file, "[stderr] {}", text) { Ok(_) => return Ok(()), Err(e) => { return Err(Error::IOError(e)); } } } - - /// Returns the path the job's output was logged to - pub(crate) fn path(&self) -> String { - return self.path.clone(); - } -} - -pub(crate) mod sql { - use chrono::{DateTime, Utc}; - use sqlx::{Connection, PgConnection}; - use std::{env, time::SystemTime}; - - /// Returns a new connection to postgres - /// - /// *x*: How many times to retry the reconnect - pub(crate) async fn start(x: u16) -> PgConnection { - let mut conn = db_connect_with_retries(x).await; - create_tables(&mut conn).await; - return conn; - } - - /// Returns the database environment variables - /// - /// Format: (address, username, password) - pub(crate) fn db_vars() -> (String, String, String) { - let db_address: String = match env::var("GREGORY_DB_ADDRESS") { - Ok(address) => address, - Err(_) => { - panic!("Environment variable `GREGORY_DB_ADDRESS` not set") - } - }; - let db_user: String = match env::var("GREGORY_DB_USER") { - Ok(user) => user, - Err(_) => { - panic!("Environment variable `GREGORY_DB_USER` not set") - } - }; - let db_pass: String = match env::var("GREGORY_DB_PASSWORD") { - Ok(pass) => pass, - Err(_) => { - panic!("Environment variable `GREGORY_DB_PASSWORD` not set") - } - }; - - return (db_address, db_user, db_pass); - } - - /// Returns the connection to the database - pub(crate) async fn db_connection() -> Result { - let (db_address, db_user, db_pass) = db_vars(); - let uri = format!("postgres://{db_user}:{db_pass}@{db_address}/gregory"); - return PgConnection::connect(uri.as_str()).await; - } - - pub(crate) async fn db_connect_with_retries(x: u16) -> PgConnection { - let mut conn = db_connection().await; - if conn.is_ok() { - return conn.unwrap(); - } - - for _ in 0..x { - conn = db_connection().await; - if conn.is_ok() { - break; - } - } - - return conn.unwrap(); - } - - // TODO: when adding logging to postgres directly, update this so it 1) adds the job at the start, 2) logs line-by-line, and 3) adds the end time and exit code at the end of the job - pub(crate) async fn log_job( - conn: &mut PgConnection, - start_time: SystemTime, - end_time: SystemTime, - exit_code: Option, - job_id: String, - revision: String, - uuid: String, - log_path: String, - ) { - let start_time: DateTime = start_time.into(); - let start_time = start_time.format("%+").to_string(); - let end_time: DateTime = end_time.into(); - let end_time = end_time.format("%+").to_string(); - let exit_code = match exit_code { - Some(code) => code.to_string(), - None => "NULL".to_string(), - }; - let query = format!("INSERT INTO job_logs (start_time, end_time, exit_code, job_id, revision, uuid, log_path) VALUES ('{start_time}', '{end_time}', {exit_code}, '{job_id}', '{revision}', '{uuid}', '{log_path}')"); - sqlx::query(query.as_str()) - .execute(conn.as_mut()) - .await - .unwrap(); - } - - /// Tries to connect to the database *x* times, panics after reaching that limit - - /// Creates table(s) for gregory if they don't exist already - pub(crate) async fn create_tables(conn: &mut PgConnection) { - sqlx::query( - "CREATE TABLE IF NOT EXISTS job_logs ( - start_time timestamp, - end_time timestamp, - duration interval GENERATED ALWAYS AS (end_time - start_time) STORED, - exit_code smallint, - job_id text, - revision text, - uuid text, - container_name text GENERATED ALWAYS AS (job_id || '-' || uuid) STORED, - log_path text -); -", - ) - .execute(conn) - .await - .unwrap(); - } -} - -#[test] -pub(crate) fn test_db_vars() { - assert_eq!( - ( - "postgres".to_string(), - "gregory".to_string(), - "pass".to_string() - ), - sql::db_vars() - ) } diff --git a/src/main.rs b/src/main.rs index 1919b5c..9439064 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,8 +3,6 @@ use crate::data::*; use better_commands; use clap::{CommandFactory, Parser}; use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh}; -use logging::sql; -use sqlx::PgConnection; use std::collections::HashMap; use std::fs::create_dir_all; use std::fs::remove_dir_all; @@ -16,8 +14,6 @@ use std::path::Path; use std::process::Command; use std::sync::Arc; use std::sync::Mutex; -use std::time::Instant; -use std::time::SystemTime; use uuid::Uuid; mod cli; @@ -26,8 +22,7 @@ mod errors; mod logging; mod tests; -#[tokio::main] -async fn main() { +fn main() { let cli = Cli::parse(); match cli.command { @@ -49,18 +44,37 @@ async fn main() { } }, Commands::Run { config } => { - run(config).await; + run(config); } } } -async fn run(config_path: String) { - let config = Config::from_file(config_path).unwrap(); // this reads the file to a [`Config`] thing - let state = State::from_config(config.clone()).await; +fn run(config_path: String) { + let config = config_from_file(config_path).unwrap(); // this reads the file to a [`Config`] thing + + let mut jobs: HashMap = HashMap::new(); + + // arranges all the jobs by their job id (e.g. `packages.librewolf.compilation`) + for (package_name, package) in config.clone().packages { + match package.compilation { + Some(tmp) => { + jobs.insert(format!("packages.{}.compilation", package_name), tmp); + } + None => {} + } + + for (job_name, job) in package.packaging { + jobs.insert( + format!("packages.{}.packaging.{}", package_name, job_name), + job, + ); + } + } // TODO: improve efficiency of all this logic // TODO: Also clean it up and split it into different functions, especially the job sorter // TODO: figure all this out and stuff and update the comments above this - the dependency map is done though + let dep_map = dependency_map(jobs.clone(), config.clone()); let mut ordered: Vec = Vec::new(); // holds the job ids in order of how they should be run @@ -72,57 +86,40 @@ async fn run(config_path: String) { // TODO: Add logic to add repo update repos when relevant (see dependencies) here - or maybe do that logic earlier? let failed_packages: Vec = Vec::new(); - - - let mut pg_connection = sql::start(5).await; - - // runs the jobs (will need to be updated after sorting is added) - for (job_id, job) in state.jobs { - let start_time = SystemTime::now(); - let job_exit_status = run_job(&state.conf, job_id.clone(), job.clone()); - // TODO: PUSH IT TO THE DB HERE - sql::log_job( - pg_connection.as_mut(), - start_time, - start_time + job_exit_status.duration, - job_exit_status.exit_code, - job_id, - job.revision, - job_exit_status.job_uuid, - job_exit_status.log_path, - ).await; + // runs the jobs (will need to be updated after sorting is added) + for (job_id, job) in jobs { + let job_exit_status = run_job(config.clone(), job_id, job); + println!("{:#?}", job_exit_status); } } -fn run_job( - conf: &Config, - job_id: String, - job: Job, -) -> JobExitStatus { +fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus { // limit threads to max_threads in the config let mut threads = job.threads; if job.threads > conf.max_threads { threads = conf.max_threads; } - let run_id = Uuid::now_v7(); + let container_name: String = format!("gregory-{}-{}-{}", job_id, job.revision, Uuid::now_v7()); - let job_logger = Arc::new(Mutex::new(logging::JobLogger::new( - conf.data_dir.clone(), - job_id.clone(), - job.revision.clone(), - run_id, - ))); + // do job log setup + let log_path = &format!("{}/logs/{container_name}", conf.data_dir); // can't select fields in the format!() {} thing, have to do this + let log_dir: &Path = Path::new(log_path).parent().unwrap(); + create_dir_all(log_dir).unwrap(); + + let job_logger = Arc::new(Mutex::new( + logging::JobLogger::new(log_path.clone()).unwrap(), + )); // write the script - let script_path: String = format!("{}/tmp/{}.sh", conf.data_dir, run_id); - let script_dir = Path::new(&script_path).parent().unwrap(); // create dir for the script + let script_path = &format!("{}/tmp/{container_name}.sh", conf.data_dir); + let script_dir: &Path = Path::new(script_path).parent().unwrap(); // create dir for the script create_dir_all(script_dir).unwrap(); - write(&script_path, job.commands.join("\n")).unwrap(); + write(script_path, job.commands.join("\n")).unwrap(); // set permissions - *unix specific* - let mut perms = File::open(&script_path) + let mut perms = File::open(script_path) .unwrap() .metadata() .unwrap() @@ -132,8 +129,7 @@ fn run_job( // run the job let mut cmd_args: Vec = vec![ "run".to_string(), - "--rm".to_string(), - format!("--name={job_id}-{run_id}"), + format!("--name={container_name}"), format!("--cpus={threads}"), format!("--privileged={}", job.privileged), format!("-v={script_path}:/gregory-entrypoint.sh"), @@ -155,26 +151,18 @@ fn run_job( let cmd_output = better_commands::run_funcs( Command::new("podman").args(cmd_args), { - let start_time = Instant::now(); let logger_clone = Arc::clone(&job_logger); move |stdout_lines| { for line in stdout_lines { - let _ = logger_clone - .lock() - .unwrap() - .stdout(line.unwrap(), start_time); + let _ = logger_clone.lock().unwrap().stdout(line.unwrap()); } } }, { - let start_time = Instant::now(); let logger_clone = Arc::clone(&job_logger); move |stderr_lines| { for line in stderr_lines { - let _ = logger_clone - .lock() - .unwrap() - .stderr(line.unwrap(), start_time); + let _ = logger_clone.lock().unwrap().stderr(line.unwrap()); } } }, @@ -183,15 +171,12 @@ fn run_job( // remove tmp dir/clean up remove_dir_all(script_dir).unwrap(); - let log_path = job_logger.lock().unwrap().path(); - return JobExitStatus { - container_name: script_path, + container_name: container_name, duration: cmd_output.clone().duration(), - job, + job: job, exit_code: cmd_output.status_code(), - log_path, - job_uuid: run_id.to_string(), + log_path: log_path.clone(), }; } @@ -235,6 +220,73 @@ fn order_jobs(jobs: HashMap, conf: Config) { */ } +/// Returns a hashmap mapping all job ids to what jobs depend on them (recursively) +/// +/// Example output using the example toml: +/// +/// ```json +/// { +/// "packages.some-librewolf-dependency.packaging.fedora": [ +/// "packages.librewolf.compilation", +/// "packages.librewolf.packaging.fedora", +/// ], +/// "packages.some-librewolf-dependency.compilation": [ +/// "packages.librewolf.compilation", +/// "packages.librewolf.packaging.fedora", +/// "packages.some-librewolf-dependency.packaging.fedora", +/// ], +/// "packages.librewolf.compilation": [ +/// "packages.librewolf.packaging.fedora", +/// ], +/// } +/// ``` +fn dependency_map(jobs: HashMap, conf: Config) -> HashMap> { + let mut dep_map: HashMap> = HashMap::new(); // holds job ids and every job they depend on (recursively) - not just specified dependencies, also packaging depending on compilation + + for (job_id, _) in jobs.clone() { + let (_, package_name, _) = jod_id_to_metadata(job_id.clone()); + + for dep_name in conf + .packages + .get(&package_name) + .unwrap() + .dependencies + .clone() + { + let all_deps = recursive_deps_for_package(dep_name.clone(), conf.clone()); + for dep in all_deps { + if !dep_map.contains_key(&dep) { + dep_map.insert(dep.clone(), Vec::new()); + } + dep_map.get_mut(&dep).unwrap().push(job_id.clone()); + } + } + } + + // add compilation jobs when relevant + for (package_name, package) in conf.packages { + if package.compilation.is_some() { + if !dep_map.contains_key(&format!("packages.{package_name}.compilation")) { + dep_map.insert(format!("packages.{package_name}.compilation"), Vec::new()); + } + + for (job_name, _) in package.packaging { + dep_map + .get_mut(&format!("packages.{package_name}.compilation")) + .unwrap() + .push(format!("packages.{package_name}.packaging.{job_name}")); + } + } + } + + // deduplicate dependencies + for (_, deps) in dep_map.iter_mut() { + deps.dedup(); + } + + return dep_map; +} + /// Returns all the dependencies for a package recursively, *not* including the package's own jobs (e.g. compilation) fn recursive_deps_for_package(package_name: String, conf: Config) -> Vec { let mut deps: Vec = Vec::new(); @@ -275,139 +327,3 @@ fn recursive_deps_for_package(package_name: String, conf: Config) -> Vec return deps; } - -struct State { - /// The entire config, from the config file. - conf: Config, - /// A hashmap mapping all job ids to what jobs depend on them (recursively) - /// - /// Using the example config (`gregory.example.toml`): - /// - /// ```json - /// { - /// "packages.some-librewolf-dependency.packaging.fedora": [ - /// "packages.librewolf.compilation", - /// "packages.librewolf.packaging.fedora", - /// ], - /// "packages.some-librewolf-dependency.compilation": [ - /// "packages.librewolf.compilation", - /// "packages.librewolf.packaging.fedora", - /// "packages.some-librewolf-dependency.packaging.fedora", - /// ], - /// "packages.librewolf.compilation": [ - /// "packages.librewolf.packaging.fedora", - /// ], - /// } - /// ``` - dependency_map: HashMap>, - /// A hashmap mapping all job ids to their jobs - jobs: HashMap, - /// The connection to the database - /// - /// Example (from sqlx README, modified) - /// ```ignore - /// sqlx::query("DELETE FROM table").execute(&mut state.conn).await?; - /// ``` - sql: PgConnection, -} - -impl State { - pub(crate) async fn from_file(filename: String) -> State { - let conf = Config::from_file(filename).unwrap(); - return State::from_config(conf).await; - } - - pub(crate) async fn from_config(conf: Config) -> State { - let mut jobs = HashMap::new(); - - for (package_name, package) in conf.clone().packages { - match package.compilation { - Some(tmp) => { - jobs.insert(format!("packages.{}.compilation", package_name), tmp); - } - None => {} - } - - for (job_name, job) in package.packaging { - jobs.insert( - format!("packages.{}.packaging.{}", package_name, job_name), - job, - ); - } - } - - return State { - conf: conf.clone(), - jobs: jobs.clone(), - dependency_map: State::dependency_map(jobs, conf), - sql: logging::sql::start(5).await, - }; - } - - /// Returns a hashmap mapping all job ids to what jobs depend on them (recursively) - /// - /// Example output using the example toml: - /// - /// ```json - /// { - /// "packages.some-librewolf-dependency.packaging.fedora": [ - /// "packages.librewolf.compilation", - /// "packages.librewolf.packaging.fedora", - /// ], - /// "packages.some-librewolf-dependency.compilation": [ - /// "packages.librewolf.compilation", - /// "packages.librewolf.packaging.fedora", - /// "packages.some-librewolf-dependency.packaging.fedora", - /// ], - /// "packages.librewolf.compilation": [ - /// "packages.librewolf.packaging.fedora", - /// ], - /// } - /// ``` - fn dependency_map(jobs: HashMap, conf: Config) -> HashMap> { - let mut dep_map: HashMap> = HashMap::new(); // holds job ids and every job they depend on (recursively) - not just specified dependencies, also packaging depending on compilation - - for (job_id, _) in jobs.clone() { - let (_, package_name, _) = jod_id_to_metadata(job_id.clone()); - - for dep_name in conf - .packages - .get(&package_name) - .unwrap() - .dependencies - .clone() - { - let all_deps = recursive_deps_for_package(dep_name.clone(), conf.clone()); - for dep in all_deps { - if !dep_map.contains_key(&dep) { - dep_map.insert(dep.clone(), Vec::new()); - } - dep_map.get_mut(&dep).unwrap().push(job_id.clone()); - } - } - } - - // add compilation jobs when relevant - for (package_name, package) in conf.packages { - if package.compilation.is_some() { - if !dep_map.contains_key(&format!("packages.{package_name}.compilation")) { - dep_map.insert(format!("packages.{package_name}.compilation"), Vec::new()); - } - - for (job_name, _) in package.packaging { - dep_map - .get_mut(&format!("packages.{package_name}.compilation")) - .unwrap() - .push(format!("packages.{package_name}.packaging.{job_name}")); - } - } - } - - // deduplicate dependencies - for (_, deps) in dep_map.iter_mut() { - deps.dedup(); - } - - return dep_map; - } -} diff --git a/test.sh b/test.bash similarity index 94% rename from test.sh rename to test.bash index 4a84160..b9508e6 100755 --- a/test.sh +++ b/test.bash @@ -1,6 +1,4 @@ #!/usr/bin/env bash -set -ex - # set env vars for testing export GREGORY_DB_ADDRESS=postgres export GREGORY_DB_USER=gregory