From e24eeebe2440ae2490d092660cfea32bba7e3ba6 Mon Sep 17 00:00:00 2001 From: askiiart Date: Mon, 27 Jan 2025 11:41:10 -0600 Subject: [PATCH 1/5] update dev scripts --- dev-setup.sh | 19 ++++++++++++++++--- test.bash => test.sh | 2 ++ 2 files changed, 18 insertions(+), 3 deletions(-) rename test.bash => test.sh (94%) diff --git a/dev-setup.sh b/dev-setup.sh index e16fc8f..8b0f914 100755 --- a/dev-setup.sh +++ b/dev-setup.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash +set -ex + +command_exists() { type "$1" &>/dev/null; } + +./test.sh rm -rf ./data/ mkdir -p ./data/{fedora-repo,librewolf,other-workspace} @@ -6,8 +11,16 @@ mkdir -p ./data/{fedora-repo,librewolf,other-workspace} mkdir -p ./dev/{pgadmin,gregory-pg} chmod -R 777 ./dev/pgadmin -podman-compose down -podman-compose -f podman-compose.dev.yml up -d +if command_exists "docker-compose"; then + docker-compose -f podman-compose.dev.yml down + docker-compose -f podman-compose.dev.yml up -d +elif command_exists "podman-compose"; then + podman-compose -f podman-compose.dev.yml down + podman-compose -f podman-compose.dev.yml up -d +else + echo "[ERROR] neither docker-compose nor podman-compose were found" + exit 127 +fi echo " --- @@ -19,4 +32,4 @@ echo 'pgadmin login: echo 'pgadmin settings: Hostname: "postgres" Username: "gregory" - Password: "pass"' \ No newline at end of file + Password: "pass"' diff --git a/test.bash b/test.sh similarity index 94% rename from test.bash rename to test.sh index b9508e6..4a84160 100755 --- a/test.bash +++ b/test.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +set -ex + # set env vars for testing export GREGORY_DB_ADDRESS=postgres export GREGORY_DB_USER=gregory From 7b4f389b0481e342b49e5418ede3ceace6494217 Mon Sep 17 00:00:00 2001 From: askiiart Date: Mon, 27 Jan 2025 13:24:42 -0600 Subject: [PATCH 2/5] remove unused logger --- src/logging.rs | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/src/logging.rs b/src/logging.rs index 9865b23..fbf57b7 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -1,44 +1,8 @@ use crate::errors::Error; use std::fs::{File, OpenOptions}; use std::io::Write; - -/// The logger for gregory itself - NOT for jobs -pub(crate) struct Logger { - log_file: File, -} - -impl Logger { - pub(crate) fn new(path: String) -> Result { - match OpenOptions::new().append(true).open(path) { - Ok(f) => return Ok(Logger { log_file: f }), - Err(e) => { - return Err(Error::IOError(e)); - } - } - } - - /// Log a warning - /// - /// Fun gregory lore: I originally typo'd this as "Strign" and the linter didn't catch it for some reason - pub(crate) fn warning(&mut self, text: String) -> Result<(), Error> { - match writeln!(&mut self.log_file, "[WARNING] {}", text) { - Ok(_) => return Ok(()), - Err(e) => { - return Err(Error::IOError(e)); - } - } - } - - /// Log an error - pub(crate) fn error(&mut self, text: String) -> Result<(), Error> { - match writeln!(&mut self.log_file, "[ERROR] {}", text) { - Ok(_) => return Ok(()), - Err(e) => { - return Err(Error::IOError(e)); - } - } - } -} +use std::path::Path; +use std::time::Instant; /// Logging for a [`Job`] pub(crate) struct JobLogger { From e56fb94f359a345246d55ef5e188767f4b3d4a82 Mon Sep 17 00:00:00 2001 From: askiiart Date: Tue, 28 Jan 2025 20:46:42 -0600 Subject: [PATCH 3/5] update readme --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1c545f4..7ea4976 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,12 @@ This is Gregory. Gregory controls repos. Gregory keeps track of updating repos, ## Documentation -Go look at [`docs/`](/docs/) +Go look at [`docs/`](/docs/), and check out the [example config](/gregory.example.toml) ## TODO - Add multithreading - Add hook system -- Add SQL database (maybe using `sqlx`?) - - Log stderr too -- Add dependency system - - Automatic detection with features (extensibility?) for each distro? -- Add podman errors ## Other stuff @@ -22,3 +17,4 @@ Go look at [`docs/`](/docs/) - Why the name? - I was thinking to go with something dark and foreboding, since this is a program to control *everything* about a repo - it's the high command. But I couldn't think of anything and thought just naming it some lame random name instead would be way funnier. Hence, Gregory. - Gregory is a program, so it uses it/its pronouns. It also doesn't mind whether you capitalize its name or not, "gregory" or "Gregory" are fine, you can even shorten it if you want. +- It's built for updating package repositories, but can be used to run pretty much anything. This isn't to say support won't be offered unless you're using it for a repo, but development will be focused on updating repos. From d6b78eb62c798a88853085e0e479420059d37fbf Mon Sep 17 00:00:00 2001 From: askiiart Date: Wed, 29 Jan 2025 13:07:43 -0600 Subject: [PATCH 4/5] add db connection - INCOMPLETE - remove containers after running - change log paths to make more sense (more hierarchical) - add dependency funcs (e.g. dependency_map()) - add postgres connection --- Cargo.lock | 163 ++++++++++++++++++++++++ Cargo.toml | 1 + docs/database.md | 31 +++++ gregory.example.toml | 2 +- src/data.rs | 31 ++--- src/errors.rs | 2 - src/logging.rs | 184 +++++++++++++++++++++++++-- src/main.rs | 291 ++++++++++++++++++++++++++----------------- 8 files changed, 562 insertions(+), 143 deletions(-) create mode 100644 docs/database.md diff --git a/Cargo.lock b/Cargo.lock index 3a38daf..e009b27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,21 @@ version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d67c60c5f10f11c6ee04de72b2dd98bb9d2548cbc314d22a609bfa8bd9e87e8f" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.18" @@ -145,6 +160,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + [[package]] name = "byteorder" version = "1.5.0" @@ -157,12 +178,35 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +[[package]] +name = "cc" +version = "1.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.6", +] + [[package]] name = "clap" version = "4.5.27" @@ -233,6 +277,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.16" @@ -506,6 +556,7 @@ version = "0.1.1" dependencies = [ "alphanumeric-sort", "better-commands", + "chrono", "clap", "clap_complete", "serde", @@ -575,6 +626,29 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -736,6 +810,16 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1081,6 +1165,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustversion" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" + [[package]] name = "ryu" version = "1.0.18" @@ -1168,6 +1258,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -1733,6 +1829,64 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + [[package]] name = "whoami" version = "1.5.2" @@ -1743,6 +1897,15 @@ dependencies = [ "wasite", ] +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 2baf5ac..a91d93a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ readme = "README.md" [dependencies] alphanumeric-sort = "1.5.3" better-commands = "1.0.2" +chrono = "0.4.39" clap = { version = "4.5.23", features = ["derive"] } clap_complete = "4.5.40" serde = { version = "1.0.216", features = ["derive"] } diff --git a/docs/database.md b/docs/database.md new file mode 100644 index 0000000..e15a59f --- /dev/null +++ b/docs/database.md @@ -0,0 +1,31 @@ +# Database docs + +Gregory's database is described as follows: + +```sql +CREATE TABLE IF NOT EXISTS job_logs ( + start_time timestamp, + end_time timestamp, + duration interval GENERATED ALWAYS AS (end_time - start_time) STORED, + exit_code smallint, + job_id text, + revision text, + uuid text, + container_name text GENERATED ALWAYS AS (job_id || '-' || uuid) STORED, + log_path text +); +``` + +i.e. it uses the table `job_logs`, containing the following fields: + +| start_time | end_time | duration | exit_code | job_id | revision | uuid | container_name | log_path | +| ---------- | -------- | -------- | --------- | ------ | -------- | ---- | -------------- | -------- | + +--- + +`duration` and `container_name` don't have to be inserted, as the database generates them, so they're just inserted like this: + +```rs +INSERT INTO job_logs (start_time, end_time, exit_code, job_id, revision, uuid, log_path) + VALUES ('1970-01-01 10:10:10 idkkkkk', '1970-01-01 10:11:10 idkkkkk', 1, 'packaging.librewolf.compilation', '5', 'blahblahblahblah', './data/logs/packages.librewolf.compilation/5/blahblahblahblah'); +``` diff --git a/gregory.example.toml b/gregory.example.toml index bf55cd4..4c28e68 100644 --- a/gregory.example.toml +++ b/gregory.example.toml @@ -12,7 +12,7 @@ max-threads = 10 revision = "2" threads = 6 image = "docker.io/library/debian" - commands = ["echo hi", "echo helloooooooooo"] + commands = ["echo hi", "sleep 2.432", "echo helloooooooooo"] volumes = ["librewolf"] [packages.librewolf.packaging.fedora] diff --git a/src/data.rs b/src/data.rs index b5cf0a1..fc52ac3 100644 --- a/src/data.rs +++ b/src/data.rs @@ -32,6 +32,22 @@ pub(crate) struct Config { pub(crate) volumes: HashMap, } +impl Config { + pub(crate) fn from_file(filename: String) -> Result { + match fs::read_to_string(filename) { + Ok(raw_data) => match toml::from_str(raw_data.as_str()) { + Ok(conf) => return Ok(conf), + Err(e) => { + return Err(Error::DeserError(e)); + } + }, + Err(e) => { + return Err(Error::IOError(e)); + } + } + } +} + /// Holds the data for a job #[derive(Debug, Clone, Deserialize)] pub(crate) struct Job { @@ -89,7 +105,6 @@ pub(crate) struct JobExitStatus { /// Where the log is /// /// TEMPORARY - /// TODO: Have main() handle logs and writing them to the database, not doing it in run_job() pub(crate) log_path: String, /// How long it took to run the job pub(crate) duration: time::Duration, @@ -97,20 +112,6 @@ pub(crate) struct JobExitStatus { pub(crate) container_name: String, } -pub(crate) fn config_from_file(filename: String) -> Result { - match fs::read_to_string(filename) { - Ok(raw_data) => match toml::from_str(raw_data.as_str()) { - Ok(conf) => return Ok(conf), - Err(e) => { - return Err(Error::DeserError(e)); - } - }, - Err(e) => { - return Err(Error::IOError(e)); - } - } -} - // ========================== // === === // === ↓ DEFAULTS ↓ === diff --git a/src/errors.rs b/src/errors.rs index cd638e1..72ba332 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -6,6 +6,4 @@ pub enum Error { IOError(#[from] std::io::Error), #[error("error while deserializing TOML: {0}")] DeserError(#[from] toml::de::Error), - #[error("Error connecting to database: {0}")] - DbConnectionError(String), } diff --git a/src/logging.rs b/src/logging.rs index fbf57b7..72d5083 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -1,29 +1,51 @@ +use uuid::Uuid; + use crate::errors::Error; -use std::fs::{File, OpenOptions}; +use std::env; +use std::fs::{create_dir_all, File, OpenOptions}; use std::io::Write; use std::path::Path; use std::time::Instant; /// Logging for a [`Job`] +// TODO: log to postgres instead; maybe i already made a comment todo-ing this idk pub(crate) struct JobLogger { log_file: File, + path: String, } impl JobLogger { - pub(crate) fn new(path: String) -> Result { - match OpenOptions::new().create_new(true).append(true).open(path) { - Ok(f) => return Ok(JobLogger { log_file: f }), - Err(e) => { - return Err(Error::IOError(e)); - } - } + pub(crate) fn new( + data_dir: String, + job_id: String, + revision: String, + run_id: Uuid, + ) -> JobLogger { + // get path and create the dir. + let log_path = format!("{data_dir}/logs/{job_id}/{revision}/{run_id}"); + let log_dir = Path::new(&log_path).parent().unwrap(); + create_dir_all(log_dir).unwrap(); + + return JobLogger { + log_file: OpenOptions::new() + .create_new(true) + .append(true) + .open(&log_path) + .unwrap(), + path: log_path, + }; } /// Log something printed to stdout /// /// Fun gregory lore: I originally typo'd this as "Strign" and the linter didn't catch it for some reason - pub(crate) fn stdout(&mut self, text: String) -> Result<(), Error> { - match writeln!(&mut self.log_file, "[stdout] {}", text) { + pub(crate) fn stdout(&mut self, text: String, start_time: Instant) -> Result<(), Error> { + match writeln!( + &mut self.log_file, + "[{:.3}] [stdout] {}", + start_time.elapsed().as_millis() as f64 / 1000.0, + text + ) { Ok(_) => return Ok(()), Err(e) => { return Err(Error::IOError(e)); @@ -32,12 +54,150 @@ impl JobLogger { } /// Log something printed to stderr - pub(crate) fn stderr(&mut self, text: String) -> Result<(), Error> { - match writeln!(&mut self.log_file, "[stderr] {}", text) { + pub(crate) fn stderr(&mut self, text: String, start_time: Instant) -> Result<(), Error> { + match writeln!( + &mut self.log_file, + "[{}] [stderr] {}", + start_time.elapsed().as_millis() / 1000, + text + ) { Ok(_) => return Ok(()), Err(e) => { return Err(Error::IOError(e)); } } } + + /// Returns the path the job's output was logged to + pub(crate) fn path(&self) -> String { + return self.path.clone(); + } +} + +pub(crate) mod sql { + use sqlx::{Connection, PgConnection}; + use std::{env, time::Instant}; + + /// Returns a new connection to postgres + /// + /// *x*: How many times to retry the reconnect + pub(crate) async fn start(x: u16) -> Box { + let mut conn = Box::new(db_connect_with_retries(x).await); + create_tables(&mut conn).await; + return conn; + } + + /// Returns the database environment variables + /// + /// Format: (address, username, password) + pub(crate) fn db_vars() -> (String, String, String) { + let db_address: String = match env::var("GREGORY_DB_ADDRESS") { + Ok(address) => address, + Err(_) => { + panic!("Environment variable `GREGORY_DB_ADDRESS` not set") + } + }; + let db_user: String = match env::var("GREGORY_DB_USER") { + Ok(user) => user, + Err(_) => { + panic!("Environment variable `GREGORY_DB_USER` not set") + } + }; + let db_pass: String = match env::var("GREGORY_DB_PASSWORD") { + Ok(pass) => pass, + Err(_) => { + panic!("Environment variable `GREGORY_DB_PASSWORD` not set") + } + }; + + return (db_address, db_user, db_pass); + } + + /// Returns the connection to the database + pub(crate) async fn db_connection() -> Result { + let (db_address, db_user, db_pass) = db_vars(); + let uri = format!("postgres://{db_user}:{db_pass}@{db_address}/gregory"); + return PgConnection::connect(uri.as_str()).await; + } + + pub(crate) async fn db_connect_with_retries(x: u16) -> PgConnection { + let mut conn = db_connection().await; + if conn.is_ok() { + return conn.unwrap(); + } + + for _ in 0..x { + conn = db_connection().await; + if conn.is_ok() { + break; + } + } + + return conn.unwrap(); + } + + // TODO: when adding logging to postgres directly, update this so it 1) adds the job at the start, 2) logs line-by-line, and 3) adds the end time and exit code at the end of the job + pub(crate) async fn log_job( + mut conn: Box, + start_time: Instant, + end_time: Instant, + exit_code: Option, + job_id: String, + revision: String, + uuid: String, + log_path: String, + ) { + let start_time = + chrono::DateTime::from_timestamp_millis(start_time.elapsed().as_millis() as i64) + .unwrap() + .format("%+") + .to_string(); + let end_time = + chrono::DateTime::from_timestamp_millis(end_time.elapsed().as_millis() as i64) + .unwrap() + .format("%+") + .to_string(); + let exit_code = match exit_code { + Some(code) => code.to_string(), + None => "NULL".to_string(), + }; + sqlx::query(format!("INSERT INTO job_logs (start_time, end_time, exit_code, job_id, revision, uuid, log_path) + VALUES ('{start_time}', '{end_time}', {exit_code}, '{job_id}', '{revision}', '{uuid}', '{log_path}')); +").as_str()).execute(conn.as_mut()).await.unwrap(); + } + + /// Tries to connect to the database *x* times, panics after reaching that limit + + /// Creates table(s) for gregory if they don't exist already + pub(crate) async fn create_tables(conn: &mut Box) { + sqlx::query( + "CREATE TABLE IF NOT EXISTS job_logs ( + start_time timestamp, + end_time timestamp, + duration interval GENERATED ALWAYS AS (end_time - start_time) STORED, + exit_code smallint, + job_id text, + revision text, + uuid text, + container_name text GENERATED ALWAYS AS (job_id || '-' || uuid) STORED, + log_path text +); +", + ) + .execute(conn.as_mut()) + .await + .unwrap(); + } +} + +#[test] +pub(crate) fn test_db_vars() { + assert_eq!( + ( + "postgres".to_string(), + "gregory".to_string(), + "pass".to_string() + ), + sql::db_vars() + ) } diff --git a/src/main.rs b/src/main.rs index 9439064..f18d1f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,8 @@ use crate::data::*; use better_commands; use clap::{CommandFactory, Parser}; use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh}; +use logging::sql; +use sqlx::PgConnection; use std::collections::HashMap; use std::fs::create_dir_all; use std::fs::remove_dir_all; @@ -14,6 +16,7 @@ use std::path::Path; use std::process::Command; use std::sync::Arc; use std::sync::Mutex; +use std::time::Instant; use uuid::Uuid; mod cli; @@ -22,7 +25,8 @@ mod errors; mod logging; mod tests; -fn main() { +#[tokio::main] +async fn main() { let cli = Cli::parse(); match cli.command { @@ -44,37 +48,18 @@ fn main() { } }, Commands::Run { config } => { - run(config); + run(config).await; } } } -fn run(config_path: String) { - let config = config_from_file(config_path).unwrap(); // this reads the file to a [`Config`] thing - - let mut jobs: HashMap = HashMap::new(); - - // arranges all the jobs by their job id (e.g. `packages.librewolf.compilation`) - for (package_name, package) in config.clone().packages { - match package.compilation { - Some(tmp) => { - jobs.insert(format!("packages.{}.compilation", package_name), tmp); - } - None => {} - } - - for (job_name, job) in package.packaging { - jobs.insert( - format!("packages.{}.packaging.{}", package_name, job_name), - job, - ); - } - } +async fn run(config_path: String) { + let config = Config::from_file(config_path).unwrap(); // this reads the file to a [`Config`] thing + let state = State::from_config(config.clone()).await; // TODO: improve efficiency of all this logic // TODO: Also clean it up and split it into different functions, especially the job sorter // TODO: figure all this out and stuff and update the comments above this - the dependency map is done though - let dep_map = dependency_map(jobs.clone(), config.clone()); let mut ordered: Vec = Vec::new(); // holds the job ids in order of how they should be run @@ -88,38 +73,36 @@ fn run(config_path: String) { let failed_packages: Vec = Vec::new(); // runs the jobs (will need to be updated after sorting is added) - for (job_id, job) in jobs { - let job_exit_status = run_job(config.clone(), job_id, job); + for (job_id, job) in state.jobs { + let job_exit_status = run_job(&state.conf, job_id, job); println!("{:#?}", job_exit_status); } } -fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus { +fn run_job(conf: &Config, job_id: String, job: Job) -> JobExitStatus { // limit threads to max_threads in the config let mut threads = job.threads; if job.threads > conf.max_threads { threads = conf.max_threads; } - let container_name: String = format!("gregory-{}-{}-{}", job_id, job.revision, Uuid::now_v7()); + let run_id = Uuid::now_v7(); - // do job log setup - let log_path = &format!("{}/logs/{container_name}", conf.data_dir); // can't select fields in the format!() {} thing, have to do this - let log_dir: &Path = Path::new(log_path).parent().unwrap(); - create_dir_all(log_dir).unwrap(); - - let job_logger = Arc::new(Mutex::new( - logging::JobLogger::new(log_path.clone()).unwrap(), - )); + let job_logger = Arc::new(Mutex::new(logging::JobLogger::new( + conf.data_dir.clone(), + job_id.clone(), + job.revision.clone(), + run_id, + ))); // write the script - let script_path = &format!("{}/tmp/{container_name}.sh", conf.data_dir); - let script_dir: &Path = Path::new(script_path).parent().unwrap(); // create dir for the script + let script_path: String = format!("{}/tmp/{}.sh", conf.data_dir, run_id); + let script_dir = Path::new(&script_path).parent().unwrap(); // create dir for the script create_dir_all(script_dir).unwrap(); - write(script_path, job.commands.join("\n")).unwrap(); + write(&script_path, job.commands.join("\n")).unwrap(); // set permissions - *unix specific* - let mut perms = File::open(script_path) + let mut perms = File::open(&script_path) .unwrap() .metadata() .unwrap() @@ -129,7 +112,8 @@ fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus { // run the job let mut cmd_args: Vec = vec![ "run".to_string(), - format!("--name={container_name}"), + "--rm".to_string(), + format!("--name={job_id}-{run_id}"), format!("--cpus={threads}"), format!("--privileged={}", job.privileged), format!("-v={script_path}:/gregory-entrypoint.sh"), @@ -151,18 +135,26 @@ fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus { let cmd_output = better_commands::run_funcs( Command::new("podman").args(cmd_args), { + let start_time = Instant::now(); let logger_clone = Arc::clone(&job_logger); move |stdout_lines| { for line in stdout_lines { - let _ = logger_clone.lock().unwrap().stdout(line.unwrap()); + let _ = logger_clone + .lock() + .unwrap() + .stdout(line.unwrap(), start_time); } } }, { + let start_time = Instant::now(); let logger_clone = Arc::clone(&job_logger); move |stderr_lines| { for line in stderr_lines { - let _ = logger_clone.lock().unwrap().stderr(line.unwrap()); + let _ = logger_clone + .lock() + .unwrap() + .stderr(line.unwrap(), start_time); } } }, @@ -171,12 +163,16 @@ fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus { // remove tmp dir/clean up remove_dir_all(script_dir).unwrap(); + let log_path = job_logger.lock().unwrap().path(); + + // TODO: PUSH IT TO THE DB HERE + return JobExitStatus { - container_name: container_name, + container_name: script_path, duration: cmd_output.clone().duration(), - job: job, + job, exit_code: cmd_output.status_code(), - log_path: log_path.clone(), + log_path, }; } @@ -220,73 +216,6 @@ fn order_jobs(jobs: HashMap, conf: Config) { */ } -/// Returns a hashmap mapping all job ids to what jobs depend on them (recursively) -/// -/// Example output using the example toml: -/// -/// ```json -/// { -/// "packages.some-librewolf-dependency.packaging.fedora": [ -/// "packages.librewolf.compilation", -/// "packages.librewolf.packaging.fedora", -/// ], -/// "packages.some-librewolf-dependency.compilation": [ -/// "packages.librewolf.compilation", -/// "packages.librewolf.packaging.fedora", -/// "packages.some-librewolf-dependency.packaging.fedora", -/// ], -/// "packages.librewolf.compilation": [ -/// "packages.librewolf.packaging.fedora", -/// ], -/// } -/// ``` -fn dependency_map(jobs: HashMap, conf: Config) -> HashMap> { - let mut dep_map: HashMap> = HashMap::new(); // holds job ids and every job they depend on (recursively) - not just specified dependencies, also packaging depending on compilation - - for (job_id, _) in jobs.clone() { - let (_, package_name, _) = jod_id_to_metadata(job_id.clone()); - - for dep_name in conf - .packages - .get(&package_name) - .unwrap() - .dependencies - .clone() - { - let all_deps = recursive_deps_for_package(dep_name.clone(), conf.clone()); - for dep in all_deps { - if !dep_map.contains_key(&dep) { - dep_map.insert(dep.clone(), Vec::new()); - } - dep_map.get_mut(&dep).unwrap().push(job_id.clone()); - } - } - } - - // add compilation jobs when relevant - for (package_name, package) in conf.packages { - if package.compilation.is_some() { - if !dep_map.contains_key(&format!("packages.{package_name}.compilation")) { - dep_map.insert(format!("packages.{package_name}.compilation"), Vec::new()); - } - - for (job_name, _) in package.packaging { - dep_map - .get_mut(&format!("packages.{package_name}.compilation")) - .unwrap() - .push(format!("packages.{package_name}.packaging.{job_name}")); - } - } - } - - // deduplicate dependencies - for (_, deps) in dep_map.iter_mut() { - deps.dedup(); - } - - return dep_map; -} - /// Returns all the dependencies for a package recursively, *not* including the package's own jobs (e.g. compilation) fn recursive_deps_for_package(package_name: String, conf: Config) -> Vec { let mut deps: Vec = Vec::new(); @@ -327,3 +256,139 @@ fn recursive_deps_for_package(package_name: String, conf: Config) -> Vec return deps; } + +struct State { + /// The entire config, from the config file. + conf: Config, + /// A hashmap mapping all job ids to what jobs depend on them (recursively) + /// + /// Using the example config (`gregory.example.toml`): + /// + /// ```json + /// { + /// "packages.some-librewolf-dependency.packaging.fedora": [ + /// "packages.librewolf.compilation", + /// "packages.librewolf.packaging.fedora", + /// ], + /// "packages.some-librewolf-dependency.compilation": [ + /// "packages.librewolf.compilation", + /// "packages.librewolf.packaging.fedora", + /// "packages.some-librewolf-dependency.packaging.fedora", + /// ], + /// "packages.librewolf.compilation": [ + /// "packages.librewolf.packaging.fedora", + /// ], + /// } + /// ``` + dependency_map: HashMap>, + /// A hashmap mapping all job ids to their jobs + jobs: HashMap, + /// The connection to the database + /// + /// Example (from sqlx README, modified) + /// ```ignore + /// sqlx::query("DELETE FROM table").execute(&mut state.conn).await?; + /// ``` + sql: Box, +} + +impl State { + pub(crate) async fn from_file(filename: String) -> State { + let conf = Config::from_file(filename).unwrap(); + return State::from_config(conf).await; + } + + pub(crate) async fn from_config(conf: Config) -> State { + let mut jobs = HashMap::new(); + + for (package_name, package) in conf.clone().packages { + match package.compilation { + Some(tmp) => { + jobs.insert(format!("packages.{}.compilation", package_name), tmp); + } + None => {} + } + + for (job_name, job) in package.packaging { + jobs.insert( + format!("packages.{}.packaging.{}", package_name, job_name), + job, + ); + } + } + + return State { + conf: conf.clone(), + jobs: jobs.clone(), + dependency_map: State::dependency_map(jobs, conf), + sql: logging::sql::start(5).await, + }; + } + + /// Returns a hashmap mapping all job ids to what jobs depend on them (recursively) + /// + /// Example output using the example toml: + /// + /// ```json + /// { + /// "packages.some-librewolf-dependency.packaging.fedora": [ + /// "packages.librewolf.compilation", + /// "packages.librewolf.packaging.fedora", + /// ], + /// "packages.some-librewolf-dependency.compilation": [ + /// "packages.librewolf.compilation", + /// "packages.librewolf.packaging.fedora", + /// "packages.some-librewolf-dependency.packaging.fedora", + /// ], + /// "packages.librewolf.compilation": [ + /// "packages.librewolf.packaging.fedora", + /// ], + /// } + /// ``` + fn dependency_map(jobs: HashMap, conf: Config) -> HashMap> { + let mut dep_map: HashMap> = HashMap::new(); // holds job ids and every job they depend on (recursively) - not just specified dependencies, also packaging depending on compilation + + for (job_id, _) in jobs.clone() { + let (_, package_name, _) = jod_id_to_metadata(job_id.clone()); + + for dep_name in conf + .packages + .get(&package_name) + .unwrap() + .dependencies + .clone() + { + let all_deps = recursive_deps_for_package(dep_name.clone(), conf.clone()); + for dep in all_deps { + if !dep_map.contains_key(&dep) { + dep_map.insert(dep.clone(), Vec::new()); + } + dep_map.get_mut(&dep).unwrap().push(job_id.clone()); + } + } + } + + // add compilation jobs when relevant + for (package_name, package) in conf.packages { + if package.compilation.is_some() { + if !dep_map.contains_key(&format!("packages.{package_name}.compilation")) { + dep_map.insert(format!("packages.{package_name}.compilation"), Vec::new()); + } + + for (job_name, _) in package.packaging { + dep_map + .get_mut(&format!("packages.{package_name}.compilation")) + .unwrap() + .push(format!("packages.{package_name}.packaging.{job_name}")); + } + } + } + + // deduplicate dependencies + for (_, deps) in dep_map.iter_mut() { + deps.dedup(); + } + + return dep_map; + } +} From e6775f19813af999ef2d3de8a02421c7253a277b Mon Sep 17 00:00:00 2001 From: askiiart Date: Thu, 30 Jan 2025 10:58:25 -0600 Subject: [PATCH 5/5] log entries to db --- src/data.rs | 2 ++ src/logging.rs | 41 +++++++++++++++++++---------------------- src/main.rs | 33 ++++++++++++++++++++++++++------- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/src/data.rs b/src/data.rs index fc52ac3..47e7649 100644 --- a/src/data.rs +++ b/src/data.rs @@ -110,6 +110,8 @@ pub(crate) struct JobExitStatus { pub(crate) duration: time::Duration, /// The name of the container this job ran in pub(crate) container_name: String, + /// Uuid + pub(crate) job_uuid: String } // ========================== diff --git a/src/logging.rs b/src/logging.rs index 72d5083..529f3d2 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -5,7 +5,7 @@ use std::env; use std::fs::{create_dir_all, File, OpenOptions}; use std::io::Write; use std::path::Path; -use std::time::Instant; +use std::time::{Instant, SystemTime}; /// Logging for a [`Job`] // TODO: log to postgres instead; maybe i already made a comment todo-ing this idk @@ -75,14 +75,15 @@ impl JobLogger { } pub(crate) mod sql { + use chrono::{DateTime, Utc}; use sqlx::{Connection, PgConnection}; - use std::{env, time::Instant}; + use std::{env, time::SystemTime}; /// Returns a new connection to postgres /// /// *x*: How many times to retry the reconnect - pub(crate) async fn start(x: u16) -> Box { - let mut conn = Box::new(db_connect_with_retries(x).await); + pub(crate) async fn start(x: u16) -> PgConnection { + let mut conn = db_connect_with_retries(x).await; create_tables(&mut conn).await; return conn; } @@ -138,38 +139,34 @@ pub(crate) mod sql { // TODO: when adding logging to postgres directly, update this so it 1) adds the job at the start, 2) logs line-by-line, and 3) adds the end time and exit code at the end of the job pub(crate) async fn log_job( - mut conn: Box, - start_time: Instant, - end_time: Instant, + conn: &mut PgConnection, + start_time: SystemTime, + end_time: SystemTime, exit_code: Option, job_id: String, revision: String, uuid: String, log_path: String, ) { - let start_time = - chrono::DateTime::from_timestamp_millis(start_time.elapsed().as_millis() as i64) - .unwrap() - .format("%+") - .to_string(); - let end_time = - chrono::DateTime::from_timestamp_millis(end_time.elapsed().as_millis() as i64) - .unwrap() - .format("%+") - .to_string(); + let start_time: DateTime = start_time.into(); + let start_time = start_time.format("%+").to_string(); + let end_time: DateTime = end_time.into(); + let end_time = end_time.format("%+").to_string(); let exit_code = match exit_code { Some(code) => code.to_string(), None => "NULL".to_string(), }; - sqlx::query(format!("INSERT INTO job_logs (start_time, end_time, exit_code, job_id, revision, uuid, log_path) - VALUES ('{start_time}', '{end_time}', {exit_code}, '{job_id}', '{revision}', '{uuid}', '{log_path}')); -").as_str()).execute(conn.as_mut()).await.unwrap(); + let query = format!("INSERT INTO job_logs (start_time, end_time, exit_code, job_id, revision, uuid, log_path) VALUES ('{start_time}', '{end_time}', {exit_code}, '{job_id}', '{revision}', '{uuid}', '{log_path}')"); + sqlx::query(query.as_str()) + .execute(conn.as_mut()) + .await + .unwrap(); } /// Tries to connect to the database *x* times, panics after reaching that limit /// Creates table(s) for gregory if they don't exist already - pub(crate) async fn create_tables(conn: &mut Box) { + pub(crate) async fn create_tables(conn: &mut PgConnection) { sqlx::query( "CREATE TABLE IF NOT EXISTS job_logs ( start_time timestamp, @@ -184,7 +181,7 @@ pub(crate) mod sql { ); ", ) - .execute(conn.as_mut()) + .execute(conn) .await .unwrap(); } diff --git a/src/main.rs b/src/main.rs index f18d1f1..1919b5c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,7 @@ use std::process::Command; use std::sync::Arc; use std::sync::Mutex; use std::time::Instant; +use std::time::SystemTime; use uuid::Uuid; mod cli; @@ -71,15 +72,34 @@ async fn run(config_path: String) { // TODO: Add logic to add repo update repos when relevant (see dependencies) here - or maybe do that logic earlier? let failed_packages: Vec = Vec::new(); - + + + let mut pg_connection = sql::start(5).await; + // runs the jobs (will need to be updated after sorting is added) for (job_id, job) in state.jobs { - let job_exit_status = run_job(&state.conf, job_id, job); - println!("{:#?}", job_exit_status); + let start_time = SystemTime::now(); + let job_exit_status = run_job(&state.conf, job_id.clone(), job.clone()); + + // TODO: PUSH IT TO THE DB HERE + sql::log_job( + pg_connection.as_mut(), + start_time, + start_time + job_exit_status.duration, + job_exit_status.exit_code, + job_id, + job.revision, + job_exit_status.job_uuid, + job_exit_status.log_path, + ).await; } } -fn run_job(conf: &Config, job_id: String, job: Job) -> JobExitStatus { +fn run_job( + conf: &Config, + job_id: String, + job: Job, +) -> JobExitStatus { // limit threads to max_threads in the config let mut threads = job.threads; if job.threads > conf.max_threads { @@ -165,14 +185,13 @@ fn run_job(conf: &Config, job_id: String, job: Job) -> JobExitStatus { let log_path = job_logger.lock().unwrap().path(); - // TODO: PUSH IT TO THE DB HERE - return JobExitStatus { container_name: script_path, duration: cmd_output.clone().duration(), job, exit_code: cmd_output.status_code(), log_path, + job_uuid: run_id.to_string(), }; } @@ -289,7 +308,7 @@ struct State { /// ```ignore /// sqlx::query("DELETE FROM table").execute(&mut state.conn).await?; /// ``` - sql: Box, + sql: PgConnection, } impl State {