add db connection - INCOMPLETE
- remove containers after running - change log paths to make more sense (more hierarchical) - add dependency funcs (e.g. dependency_map()) - add postgres connection
This commit is contained in:
askiiart
parent
e56fb94f35
commit
d6b78eb62c
GPG key ID:
6A32977DAF31746A
8 changed files with 562 additions and 143 deletions
163
Cargo.lock
generated
163
Cargo.lock
generated
|
|
@ -29,6 +29,21 @@ version = "1.5.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d67c60c5f10f11c6ee04de72b2dd98bb9d2548cbc314d22a609bfa8bd9e87e8f"
|
checksum = "d67c60c5f10f11c6ee04de72b2dd98bb9d2548cbc314d22a609bfa8bd9e87e8f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "android-tzdata"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "android_system_properties"
|
||||||
|
version = "0.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "anstream"
|
name = "anstream"
|
||||||
version = "0.6.18"
|
version = "0.6.18"
|
||||||
|
|
@ -145,6 +160,12 @@ dependencies = [
|
||||||
"generic-array",
|
"generic-array",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bumpalo"
|
||||||
|
version = "3.17.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "byteorder"
|
name = "byteorder"
|
||||||
version = "1.5.0"
|
version = "1.5.0"
|
||||||
|
|
@ -157,12 +178,35 @@ version = "1.9.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
|
checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cc"
|
||||||
|
version = "1.2.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229"
|
||||||
|
dependencies = [
|
||||||
|
"shlex",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cfg-if"
|
name = "cfg-if"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chrono"
|
||||||
|
version = "0.4.39"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
|
||||||
|
dependencies = [
|
||||||
|
"android-tzdata",
|
||||||
|
"iana-time-zone",
|
||||||
|
"js-sys",
|
||||||
|
"num-traits",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap"
|
name = "clap"
|
||||||
version = "4.5.27"
|
version = "4.5.27"
|
||||||
|
|
@ -233,6 +277,12 @@ version = "0.9.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
|
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "core-foundation-sys"
|
||||||
|
version = "0.8.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cpufeatures"
|
name = "cpufeatures"
|
||||||
version = "0.2.16"
|
version = "0.2.16"
|
||||||
|
|
@ -506,6 +556,7 @@ version = "0.1.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"alphanumeric-sort",
|
"alphanumeric-sort",
|
||||||
"better-commands",
|
"better-commands",
|
||||||
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
"clap_complete",
|
"clap_complete",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
@ -575,6 +626,29 @@ dependencies = [
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone"
|
||||||
|
version = "0.1.61"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
|
||||||
|
dependencies = [
|
||||||
|
"android_system_properties",
|
||||||
|
"core-foundation-sys",
|
||||||
|
"iana-time-zone-haiku",
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone-haiku"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_collections"
|
name = "icu_collections"
|
||||||
version = "1.5.0"
|
version = "1.5.0"
|
||||||
|
|
@ -736,6 +810,16 @@ version = "1.0.14"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
|
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "js-sys"
|
||||||
|
version = "0.3.77"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
|
||||||
|
dependencies = [
|
||||||
|
"once_cell",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lazy_static"
|
name = "lazy_static"
|
||||||
version = "1.5.0"
|
version = "1.5.0"
|
||||||
|
|
@ -1081,6 +1165,12 @@ dependencies = [
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustversion"
|
||||||
|
version = "1.0.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ryu"
|
name = "ryu"
|
||||||
version = "1.0.18"
|
version = "1.0.18"
|
||||||
|
|
@ -1168,6 +1258,12 @@ dependencies = [
|
||||||
"digest",
|
"digest",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "shlex"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "signal-hook-registry"
|
name = "signal-hook-registry"
|
||||||
version = "1.4.2"
|
version = "1.4.2"
|
||||||
|
|
@ -1733,6 +1829,64 @@ version = "0.1.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
|
checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen"
|
||||||
|
version = "0.2.100"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"once_cell",
|
||||||
|
"rustversion",
|
||||||
|
"wasm-bindgen-macro",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-backend"
|
||||||
|
version = "0.2.100"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
|
||||||
|
dependencies = [
|
||||||
|
"bumpalo",
|
||||||
|
"log",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
"wasm-bindgen-shared",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-macro"
|
||||||
|
version = "0.2.100"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
|
||||||
|
dependencies = [
|
||||||
|
"quote",
|
||||||
|
"wasm-bindgen-macro-support",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-macro-support"
|
||||||
|
version = "0.2.100"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
"wasm-bindgen-backend",
|
||||||
|
"wasm-bindgen-shared",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-shared"
|
||||||
|
version = "0.2.100"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "whoami"
|
name = "whoami"
|
||||||
version = "1.5.2"
|
version = "1.5.2"
|
||||||
|
|
@ -1743,6 +1897,15 @@ dependencies = [
|
||||||
"wasite",
|
"wasite",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-core"
|
||||||
|
version = "0.52.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
|
||||||
|
dependencies = [
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-sys"
|
name = "windows-sys"
|
||||||
version = "0.48.0"
|
version = "0.48.0"
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ readme = "README.md"
|
||||||
[dependencies]
|
[dependencies]
|
||||||
alphanumeric-sort = "1.5.3"
|
alphanumeric-sort = "1.5.3"
|
||||||
better-commands = "1.0.2"
|
better-commands = "1.0.2"
|
||||||
|
chrono = "0.4.39"
|
||||||
clap = { version = "4.5.23", features = ["derive"] }
|
clap = { version = "4.5.23", features = ["derive"] }
|
||||||
clap_complete = "4.5.40"
|
clap_complete = "4.5.40"
|
||||||
serde = { version = "1.0.216", features = ["derive"] }
|
serde = { version = "1.0.216", features = ["derive"] }
|
||||||
|
|
|
||||||
31
docs/database.md
Normal file
31
docs/database.md
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
# Database docs
|
||||||
|
|
||||||
|
Gregory's database is described as follows:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE IF NOT EXISTS job_logs (
|
||||||
|
start_time timestamp,
|
||||||
|
end_time timestamp,
|
||||||
|
duration interval GENERATED ALWAYS AS (end_time - start_time) STORED,
|
||||||
|
exit_code smallint,
|
||||||
|
job_id text,
|
||||||
|
revision text,
|
||||||
|
uuid text,
|
||||||
|
container_name text GENERATED ALWAYS AS (job_id || '-' || uuid) STORED,
|
||||||
|
log_path text
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
i.e. it uses the table `job_logs`, containing the following fields:
|
||||||
|
|
||||||
|
| start_time | end_time | duration | exit_code | job_id | revision | uuid | container_name | log_path |
|
||||||
|
| ---------- | -------- | -------- | --------- | ------ | -------- | ---- | -------------- | -------- |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
`duration` and `container_name` don't have to be inserted, as the database generates them, so they're just inserted like this:
|
||||||
|
|
||||||
|
```rs
|
||||||
|
INSERT INTO job_logs (start_time, end_time, exit_code, job_id, revision, uuid, log_path)
|
||||||
|
VALUES ('1970-01-01 10:10:10 idkkkkk', '1970-01-01 10:11:10 idkkkkk', 1, 'packaging.librewolf.compilation', '5', 'blahblahblahblah', './data/logs/packages.librewolf.compilation/5/blahblahblahblah');
|
||||||
|
```
|
||||||
|
|
@ -12,7 +12,7 @@ max-threads = 10
|
||||||
revision = "2"
|
revision = "2"
|
||||||
threads = 6
|
threads = 6
|
||||||
image = "docker.io/library/debian"
|
image = "docker.io/library/debian"
|
||||||
commands = ["echo hi", "echo helloooooooooo"]
|
commands = ["echo hi", "sleep 2.432", "echo helloooooooooo"]
|
||||||
volumes = ["librewolf"]
|
volumes = ["librewolf"]
|
||||||
|
|
||||||
[packages.librewolf.packaging.fedora]
|
[packages.librewolf.packaging.fedora]
|
||||||
|
|
|
||||||
31
src/data.rs
31
src/data.rs
|
|
@ -32,6 +32,22 @@ pub(crate) struct Config {
|
||||||
pub(crate) volumes: HashMap<String, String>,
|
pub(crate) volumes: HashMap<String, String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Config {
|
||||||
|
pub(crate) fn from_file(filename: String) -> Result<Config, Error> {
|
||||||
|
match fs::read_to_string(filename) {
|
||||||
|
Ok(raw_data) => match toml::from_str(raw_data.as_str()) {
|
||||||
|
Ok(conf) => return Ok(conf),
|
||||||
|
Err(e) => {
|
||||||
|
return Err(Error::DeserError(e));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(e) => {
|
||||||
|
return Err(Error::IOError(e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Holds the data for a job
|
/// Holds the data for a job
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
pub(crate) struct Job {
|
pub(crate) struct Job {
|
||||||
|
|
@ -89,7 +105,6 @@ pub(crate) struct JobExitStatus {
|
||||||
/// Where the log is
|
/// Where the log is
|
||||||
///
|
///
|
||||||
/// TEMPORARY
|
/// TEMPORARY
|
||||||
/// TODO: Have main() handle logs and writing them to the database, not doing it in run_job()
|
|
||||||
pub(crate) log_path: String,
|
pub(crate) log_path: String,
|
||||||
/// How long it took to run the job
|
/// How long it took to run the job
|
||||||
pub(crate) duration: time::Duration,
|
pub(crate) duration: time::Duration,
|
||||||
|
|
@ -97,20 +112,6 @@ pub(crate) struct JobExitStatus {
|
||||||
pub(crate) container_name: String,
|
pub(crate) container_name: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn config_from_file(filename: String) -> Result<Config, Error> {
|
|
||||||
match fs::read_to_string(filename) {
|
|
||||||
Ok(raw_data) => match toml::from_str(raw_data.as_str()) {
|
|
||||||
Ok(conf) => return Ok(conf),
|
|
||||||
Err(e) => {
|
|
||||||
return Err(Error::DeserError(e));
|
|
||||||
}
|
|
||||||
},
|
|
||||||
Err(e) => {
|
|
||||||
return Err(Error::IOError(e));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==========================
|
// ==========================
|
||||||
// === ===
|
// === ===
|
||||||
// === ↓ DEFAULTS ↓ ===
|
// === ↓ DEFAULTS ↓ ===
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,4 @@ pub enum Error {
|
||||||
IOError(#[from] std::io::Error),
|
IOError(#[from] std::io::Error),
|
||||||
#[error("error while deserializing TOML: {0}")]
|
#[error("error while deserializing TOML: {0}")]
|
||||||
DeserError(#[from] toml::de::Error),
|
DeserError(#[from] toml::de::Error),
|
||||||
#[error("Error connecting to database: {0}")]
|
|
||||||
DbConnectionError(String),
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
184
src/logging.rs
184
src/logging.rs
|
|
@ -1,29 +1,51 @@
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::errors::Error;
|
use crate::errors::Error;
|
||||||
use std::fs::{File, OpenOptions};
|
use std::env;
|
||||||
|
use std::fs::{create_dir_all, File, OpenOptions};
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
/// Logging for a [`Job`]
|
/// Logging for a [`Job`]
|
||||||
|
// TODO: log to postgres instead; maybe i already made a comment todo-ing this idk
|
||||||
pub(crate) struct JobLogger {
|
pub(crate) struct JobLogger {
|
||||||
log_file: File,
|
log_file: File,
|
||||||
|
path: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl JobLogger {
|
impl JobLogger {
|
||||||
pub(crate) fn new(path: String) -> Result<JobLogger, Error> {
|
pub(crate) fn new(
|
||||||
match OpenOptions::new().create_new(true).append(true).open(path) {
|
data_dir: String,
|
||||||
Ok(f) => return Ok(JobLogger { log_file: f }),
|
job_id: String,
|
||||||
Err(e) => {
|
revision: String,
|
||||||
return Err(Error::IOError(e));
|
run_id: Uuid,
|
||||||
}
|
) -> JobLogger {
|
||||||
}
|
// get path and create the dir.
|
||||||
|
let log_path = format!("{data_dir}/logs/{job_id}/{revision}/{run_id}");
|
||||||
|
let log_dir = Path::new(&log_path).parent().unwrap();
|
||||||
|
create_dir_all(log_dir).unwrap();
|
||||||
|
|
||||||
|
return JobLogger {
|
||||||
|
log_file: OpenOptions::new()
|
||||||
|
.create_new(true)
|
||||||
|
.append(true)
|
||||||
|
.open(&log_path)
|
||||||
|
.unwrap(),
|
||||||
|
path: log_path,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Log something printed to stdout
|
/// Log something printed to stdout
|
||||||
///
|
///
|
||||||
/// Fun gregory lore: I originally typo'd this as "Strign" and the linter didn't catch it for some reason
|
/// Fun gregory lore: I originally typo'd this as "Strign" and the linter didn't catch it for some reason
|
||||||
pub(crate) fn stdout(&mut self, text: String) -> Result<(), Error> {
|
pub(crate) fn stdout(&mut self, text: String, start_time: Instant) -> Result<(), Error> {
|
||||||
match writeln!(&mut self.log_file, "[stdout] {}", text) {
|
match writeln!(
|
||||||
|
&mut self.log_file,
|
||||||
|
"[{:.3}] [stdout] {}",
|
||||||
|
start_time.elapsed().as_millis() as f64 / 1000.0,
|
||||||
|
text
|
||||||
|
) {
|
||||||
Ok(_) => return Ok(()),
|
Ok(_) => return Ok(()),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
return Err(Error::IOError(e));
|
return Err(Error::IOError(e));
|
||||||
|
|
@ -32,12 +54,150 @@ impl JobLogger {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Log something printed to stderr
|
/// Log something printed to stderr
|
||||||
pub(crate) fn stderr(&mut self, text: String) -> Result<(), Error> {
|
pub(crate) fn stderr(&mut self, text: String, start_time: Instant) -> Result<(), Error> {
|
||||||
match writeln!(&mut self.log_file, "[stderr] {}", text) {
|
match writeln!(
|
||||||
|
&mut self.log_file,
|
||||||
|
"[{}] [stderr] {}",
|
||||||
|
start_time.elapsed().as_millis() / 1000,
|
||||||
|
text
|
||||||
|
) {
|
||||||
Ok(_) => return Ok(()),
|
Ok(_) => return Ok(()),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
return Err(Error::IOError(e));
|
return Err(Error::IOError(e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the path the job's output was logged to
|
||||||
|
pub(crate) fn path(&self) -> String {
|
||||||
|
return self.path.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) mod sql {
|
||||||
|
use sqlx::{Connection, PgConnection};
|
||||||
|
use std::{env, time::Instant};
|
||||||
|
|
||||||
|
/// Returns a new connection to postgres
|
||||||
|
///
|
||||||
|
/// *x*: How many times to retry the reconnect
|
||||||
|
pub(crate) async fn start(x: u16) -> Box<PgConnection> {
|
||||||
|
let mut conn = Box::new(db_connect_with_retries(x).await);
|
||||||
|
create_tables(&mut conn).await;
|
||||||
|
return conn;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the database environment variables
|
||||||
|
///
|
||||||
|
/// Format: (address, username, password)
|
||||||
|
pub(crate) fn db_vars() -> (String, String, String) {
|
||||||
|
let db_address: String = match env::var("GREGORY_DB_ADDRESS") {
|
||||||
|
Ok(address) => address,
|
||||||
|
Err(_) => {
|
||||||
|
panic!("Environment variable `GREGORY_DB_ADDRESS` not set")
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let db_user: String = match env::var("GREGORY_DB_USER") {
|
||||||
|
Ok(user) => user,
|
||||||
|
Err(_) => {
|
||||||
|
panic!("Environment variable `GREGORY_DB_USER` not set")
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let db_pass: String = match env::var("GREGORY_DB_PASSWORD") {
|
||||||
|
Ok(pass) => pass,
|
||||||
|
Err(_) => {
|
||||||
|
panic!("Environment variable `GREGORY_DB_PASSWORD` not set")
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (db_address, db_user, db_pass);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the connection to the database
|
||||||
|
pub(crate) async fn db_connection() -> Result<PgConnection, sqlx::Error> {
|
||||||
|
let (db_address, db_user, db_pass) = db_vars();
|
||||||
|
let uri = format!("postgres://{db_user}:{db_pass}@{db_address}/gregory");
|
||||||
|
return PgConnection::connect(uri.as_str()).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn db_connect_with_retries(x: u16) -> PgConnection {
|
||||||
|
let mut conn = db_connection().await;
|
||||||
|
if conn.is_ok() {
|
||||||
|
return conn.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
for _ in 0..x {
|
||||||
|
conn = db_connection().await;
|
||||||
|
if conn.is_ok() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return conn.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: when adding logging to postgres directly, update this so it 1) adds the job at the start, 2) logs line-by-line, and 3) adds the end time and exit code at the end of the job
|
||||||
|
pub(crate) async fn log_job(
|
||||||
|
mut conn: Box<PgConnection>,
|
||||||
|
start_time: Instant,
|
||||||
|
end_time: Instant,
|
||||||
|
exit_code: Option<i32>,
|
||||||
|
job_id: String,
|
||||||
|
revision: String,
|
||||||
|
uuid: String,
|
||||||
|
log_path: String,
|
||||||
|
) {
|
||||||
|
let start_time =
|
||||||
|
chrono::DateTime::from_timestamp_millis(start_time.elapsed().as_millis() as i64)
|
||||||
|
.unwrap()
|
||||||
|
.format("%+")
|
||||||
|
.to_string();
|
||||||
|
let end_time =
|
||||||
|
chrono::DateTime::from_timestamp_millis(end_time.elapsed().as_millis() as i64)
|
||||||
|
.unwrap()
|
||||||
|
.format("%+")
|
||||||
|
.to_string();
|
||||||
|
let exit_code = match exit_code {
|
||||||
|
Some(code) => code.to_string(),
|
||||||
|
None => "NULL".to_string(),
|
||||||
|
};
|
||||||
|
sqlx::query(format!("INSERT INTO job_logs (start_time, end_time, exit_code, job_id, revision, uuid, log_path)
|
||||||
|
VALUES ('{start_time}', '{end_time}', {exit_code}, '{job_id}', '{revision}', '{uuid}', '{log_path}'));
|
||||||
|
").as_str()).execute(conn.as_mut()).await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tries to connect to the database *x* times, panics after reaching that limit
|
||||||
|
|
||||||
|
/// Creates table(s) for gregory if they don't exist already
|
||||||
|
pub(crate) async fn create_tables(conn: &mut Box<PgConnection>) {
|
||||||
|
sqlx::query(
|
||||||
|
"CREATE TABLE IF NOT EXISTS job_logs (
|
||||||
|
start_time timestamp,
|
||||||
|
end_time timestamp,
|
||||||
|
duration interval GENERATED ALWAYS AS (end_time - start_time) STORED,
|
||||||
|
exit_code smallint,
|
||||||
|
job_id text,
|
||||||
|
revision text,
|
||||||
|
uuid text,
|
||||||
|
container_name text GENERATED ALWAYS AS (job_id || '-' || uuid) STORED,
|
||||||
|
log_path text
|
||||||
|
);
|
||||||
|
",
|
||||||
|
)
|
||||||
|
.execute(conn.as_mut())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
pub(crate) fn test_db_vars() {
|
||||||
|
assert_eq!(
|
||||||
|
(
|
||||||
|
"postgres".to_string(),
|
||||||
|
"gregory".to_string(),
|
||||||
|
"pass".to_string()
|
||||||
|
),
|
||||||
|
sql::db_vars()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
291
src/main.rs
291
src/main.rs
|
|
@ -3,6 +3,8 @@ use crate::data::*;
|
||||||
use better_commands;
|
use better_commands;
|
||||||
use clap::{CommandFactory, Parser};
|
use clap::{CommandFactory, Parser};
|
||||||
use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh};
|
use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh};
|
||||||
|
use logging::sql;
|
||||||
|
use sqlx::PgConnection;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::fs::create_dir_all;
|
use std::fs::create_dir_all;
|
||||||
use std::fs::remove_dir_all;
|
use std::fs::remove_dir_all;
|
||||||
|
|
@ -14,6 +16,7 @@ use std::path::Path;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
|
use std::time::Instant;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
mod cli;
|
mod cli;
|
||||||
|
|
@ -22,7 +25,8 @@ mod errors;
|
||||||
mod logging;
|
mod logging;
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|
||||||
fn main() {
|
#[tokio::main]
|
||||||
|
async fn main() {
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
|
|
||||||
match cli.command {
|
match cli.command {
|
||||||
|
|
@ -44,37 +48,18 @@ fn main() {
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
Commands::Run { config } => {
|
Commands::Run { config } => {
|
||||||
run(config);
|
run(config).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run(config_path: String) {
|
async fn run(config_path: String) {
|
||||||
let config = config_from_file(config_path).unwrap(); // this reads the file to a [`Config`] thing
|
let config = Config::from_file(config_path).unwrap(); // this reads the file to a [`Config`] thing
|
||||||
|
let state = State::from_config(config.clone()).await;
|
||||||
let mut jobs: HashMap<String, Job> = HashMap::new();
|
|
||||||
|
|
||||||
// arranges all the jobs by their job id (e.g. `packages.librewolf.compilation`)
|
|
||||||
for (package_name, package) in config.clone().packages {
|
|
||||||
match package.compilation {
|
|
||||||
Some(tmp) => {
|
|
||||||
jobs.insert(format!("packages.{}.compilation", package_name), tmp);
|
|
||||||
}
|
|
||||||
None => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (job_name, job) in package.packaging {
|
|
||||||
jobs.insert(
|
|
||||||
format!("packages.{}.packaging.{}", package_name, job_name),
|
|
||||||
job,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: improve efficiency of all this logic
|
// TODO: improve efficiency of all this logic
|
||||||
// TODO: Also clean it up and split it into different functions, especially the job sorter
|
// TODO: Also clean it up and split it into different functions, especially the job sorter
|
||||||
// TODO: figure all this out and stuff and update the comments above this - the dependency map is done though
|
// TODO: figure all this out and stuff and update the comments above this - the dependency map is done though
|
||||||
let dep_map = dependency_map(jobs.clone(), config.clone());
|
|
||||||
|
|
||||||
let mut ordered: Vec<String> = Vec::new(); // holds the job ids in order of how they should be run
|
let mut ordered: Vec<String> = Vec::new(); // holds the job ids in order of how they should be run
|
||||||
|
|
||||||
|
|
@ -88,38 +73,36 @@ fn run(config_path: String) {
|
||||||
let failed_packages: Vec<String> = Vec::new();
|
let failed_packages: Vec<String> = Vec::new();
|
||||||
|
|
||||||
// runs the jobs (will need to be updated after sorting is added)
|
// runs the jobs (will need to be updated after sorting is added)
|
||||||
for (job_id, job) in jobs {
|
for (job_id, job) in state.jobs {
|
||||||
let job_exit_status = run_job(config.clone(), job_id, job);
|
let job_exit_status = run_job(&state.conf, job_id, job);
|
||||||
println!("{:#?}", job_exit_status);
|
println!("{:#?}", job_exit_status);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus {
|
fn run_job(conf: &Config, job_id: String, job: Job) -> JobExitStatus {
|
||||||
// limit threads to max_threads in the config
|
// limit threads to max_threads in the config
|
||||||
let mut threads = job.threads;
|
let mut threads = job.threads;
|
||||||
if job.threads > conf.max_threads {
|
if job.threads > conf.max_threads {
|
||||||
threads = conf.max_threads;
|
threads = conf.max_threads;
|
||||||
}
|
}
|
||||||
|
|
||||||
let container_name: String = format!("gregory-{}-{}-{}", job_id, job.revision, Uuid::now_v7());
|
let run_id = Uuid::now_v7();
|
||||||
|
|
||||||
// do job log setup
|
let job_logger = Arc::new(Mutex::new(logging::JobLogger::new(
|
||||||
let log_path = &format!("{}/logs/{container_name}", conf.data_dir); // can't select fields in the format!() {} thing, have to do this
|
conf.data_dir.clone(),
|
||||||
let log_dir: &Path = Path::new(log_path).parent().unwrap();
|
job_id.clone(),
|
||||||
create_dir_all(log_dir).unwrap();
|
job.revision.clone(),
|
||||||
|
run_id,
|
||||||
let job_logger = Arc::new(Mutex::new(
|
)));
|
||||||
logging::JobLogger::new(log_path.clone()).unwrap(),
|
|
||||||
));
|
|
||||||
|
|
||||||
// write the script
|
// write the script
|
||||||
let script_path = &format!("{}/tmp/{container_name}.sh", conf.data_dir);
|
let script_path: String = format!("{}/tmp/{}.sh", conf.data_dir, run_id);
|
||||||
let script_dir: &Path = Path::new(script_path).parent().unwrap(); // create dir for the script
|
let script_dir = Path::new(&script_path).parent().unwrap(); // create dir for the script
|
||||||
create_dir_all(script_dir).unwrap();
|
create_dir_all(script_dir).unwrap();
|
||||||
write(script_path, job.commands.join("\n")).unwrap();
|
write(&script_path, job.commands.join("\n")).unwrap();
|
||||||
|
|
||||||
// set permissions - *unix specific*
|
// set permissions - *unix specific*
|
||||||
let mut perms = File::open(script_path)
|
let mut perms = File::open(&script_path)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.metadata()
|
.metadata()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
@ -129,7 +112,8 @@ fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus {
|
||||||
// run the job
|
// run the job
|
||||||
let mut cmd_args: Vec<String> = vec![
|
let mut cmd_args: Vec<String> = vec![
|
||||||
"run".to_string(),
|
"run".to_string(),
|
||||||
format!("--name={container_name}"),
|
"--rm".to_string(),
|
||||||
|
format!("--name={job_id}-{run_id}"),
|
||||||
format!("--cpus={threads}"),
|
format!("--cpus={threads}"),
|
||||||
format!("--privileged={}", job.privileged),
|
format!("--privileged={}", job.privileged),
|
||||||
format!("-v={script_path}:/gregory-entrypoint.sh"),
|
format!("-v={script_path}:/gregory-entrypoint.sh"),
|
||||||
|
|
@ -151,18 +135,26 @@ fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus {
|
||||||
let cmd_output = better_commands::run_funcs(
|
let cmd_output = better_commands::run_funcs(
|
||||||
Command::new("podman").args(cmd_args),
|
Command::new("podman").args(cmd_args),
|
||||||
{
|
{
|
||||||
|
let start_time = Instant::now();
|
||||||
let logger_clone = Arc::clone(&job_logger);
|
let logger_clone = Arc::clone(&job_logger);
|
||||||
move |stdout_lines| {
|
move |stdout_lines| {
|
||||||
for line in stdout_lines {
|
for line in stdout_lines {
|
||||||
let _ = logger_clone.lock().unwrap().stdout(line.unwrap());
|
let _ = logger_clone
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.stdout(line.unwrap(), start_time);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
let start_time = Instant::now();
|
||||||
let logger_clone = Arc::clone(&job_logger);
|
let logger_clone = Arc::clone(&job_logger);
|
||||||
move |stderr_lines| {
|
move |stderr_lines| {
|
||||||
for line in stderr_lines {
|
for line in stderr_lines {
|
||||||
let _ = logger_clone.lock().unwrap().stderr(line.unwrap());
|
let _ = logger_clone
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.stderr(line.unwrap(), start_time);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -171,12 +163,16 @@ fn run_job(conf: Config, job_id: String, job: Job) -> JobExitStatus {
|
||||||
// remove tmp dir/clean up
|
// remove tmp dir/clean up
|
||||||
remove_dir_all(script_dir).unwrap();
|
remove_dir_all(script_dir).unwrap();
|
||||||
|
|
||||||
|
let log_path = job_logger.lock().unwrap().path();
|
||||||
|
|
||||||
|
// TODO: PUSH IT TO THE DB HERE
|
||||||
|
|
||||||
return JobExitStatus {
|
return JobExitStatus {
|
||||||
container_name: container_name,
|
container_name: script_path,
|
||||||
duration: cmd_output.clone().duration(),
|
duration: cmd_output.clone().duration(),
|
||||||
job: job,
|
job,
|
||||||
exit_code: cmd_output.status_code(),
|
exit_code: cmd_output.status_code(),
|
||||||
log_path: log_path.clone(),
|
log_path,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -220,73 +216,6 @@ fn order_jobs(jobs: HashMap<String, Job>, conf: Config) {
|
||||||
*/
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns a hashmap mapping all job ids to what jobs depend on them (recursively)
|
|
||||||
///
|
|
||||||
/// Example output using the example toml:
|
|
||||||
///
|
|
||||||
/// ```json
|
|
||||||
/// {
|
|
||||||
/// "packages.some-librewolf-dependency.packaging.fedora": [
|
|
||||||
/// "packages.librewolf.compilation",
|
|
||||||
/// "packages.librewolf.packaging.fedora",
|
|
||||||
/// ],
|
|
||||||
/// "packages.some-librewolf-dependency.compilation": [
|
|
||||||
/// "packages.librewolf.compilation",
|
|
||||||
/// "packages.librewolf.packaging.fedora",
|
|
||||||
/// "packages.some-librewolf-dependency.packaging.fedora",
|
|
||||||
/// ],
|
|
||||||
/// "packages.librewolf.compilation": [
|
|
||||||
/// "packages.librewolf.packaging.fedora",
|
|
||||||
/// ],
|
|
||||||
/// }
|
|
||||||
/// ```
|
|
||||||
fn dependency_map(jobs: HashMap<String, Job>, conf: Config) -> HashMap<String, Vec<String>> {
|
|
||||||
let mut dep_map: HashMap<String, Vec<String>> = HashMap::new(); // holds job ids and every job they depend on (recursively) - not just specified dependencies, also packaging depending on compilation
|
|
||||||
|
|
||||||
for (job_id, _) in jobs.clone() {
|
|
||||||
let (_, package_name, _) = jod_id_to_metadata(job_id.clone());
|
|
||||||
|
|
||||||
for dep_name in conf
|
|
||||||
.packages
|
|
||||||
.get(&package_name)
|
|
||||||
.unwrap()
|
|
||||||
.dependencies
|
|
||||||
.clone()
|
|
||||||
{
|
|
||||||
let all_deps = recursive_deps_for_package(dep_name.clone(), conf.clone());
|
|
||||||
for dep in all_deps {
|
|
||||||
if !dep_map.contains_key(&dep) {
|
|
||||||
dep_map.insert(dep.clone(), Vec::new());
|
|
||||||
}
|
|
||||||
dep_map.get_mut(&dep).unwrap().push(job_id.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// add compilation jobs when relevant
|
|
||||||
for (package_name, package) in conf.packages {
|
|
||||||
if package.compilation.is_some() {
|
|
||||||
if !dep_map.contains_key(&format!("packages.{package_name}.compilation")) {
|
|
||||||
dep_map.insert(format!("packages.{package_name}.compilation"), Vec::new());
|
|
||||||
}
|
|
||||||
|
|
||||||
for (job_name, _) in package.packaging {
|
|
||||||
dep_map
|
|
||||||
.get_mut(&format!("packages.{package_name}.compilation"))
|
|
||||||
.unwrap()
|
|
||||||
.push(format!("packages.{package_name}.packaging.{job_name}"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// deduplicate dependencies
|
|
||||||
for (_, deps) in dep_map.iter_mut() {
|
|
||||||
deps.dedup();
|
|
||||||
}
|
|
||||||
|
|
||||||
return dep_map;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns all the dependencies for a package recursively, *not* including the package's own jobs (e.g. compilation)
|
/// Returns all the dependencies for a package recursively, *not* including the package's own jobs (e.g. compilation)
|
||||||
fn recursive_deps_for_package(package_name: String, conf: Config) -> Vec<String> {
|
fn recursive_deps_for_package(package_name: String, conf: Config) -> Vec<String> {
|
||||||
let mut deps: Vec<String> = Vec::new();
|
let mut deps: Vec<String> = Vec::new();
|
||||||
|
|
@ -327,3 +256,139 @@ fn recursive_deps_for_package(package_name: String, conf: Config) -> Vec<String>
|
||||||
|
|
||||||
return deps;
|
return deps;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct State {
|
||||||
|
/// The entire config, from the config file.
|
||||||
|
conf: Config,
|
||||||
|
/// A hashmap mapping all job ids to what jobs depend on them (recursively)
|
||||||
|
///
|
||||||
|
/// Using the example config (`gregory.example.toml`):
|
||||||
|
///
|
||||||
|
/// ```json
|
||||||
|
/// {
|
||||||
|
/// "packages.some-librewolf-dependency.packaging.fedora": [
|
||||||
|
/// "packages.librewolf.compilation",
|
||||||
|
/// "packages.librewolf.packaging.fedora",
|
||||||
|
/// ],
|
||||||
|
/// "packages.some-librewolf-dependency.compilation": [
|
||||||
|
/// "packages.librewolf.compilation",
|
||||||
|
/// "packages.librewolf.packaging.fedora",
|
||||||
|
/// "packages.some-librewolf-dependency.packaging.fedora",
|
||||||
|
/// ],
|
||||||
|
/// "packages.librewolf.compilation": [
|
||||||
|
/// "packages.librewolf.packaging.fedora",
|
||||||
|
/// ],
|
||||||
|
/// }
|
||||||
|
/// ```
|
||||||
|
dependency_map: HashMap<String, Vec<String>>,
|
||||||
|
/// A hashmap mapping all job ids to their jobs
|
||||||
|
jobs: HashMap<String, Job>,
|
||||||
|
/// The connection to the database
|
||||||
|
///
|
||||||
|
/// Example (from sqlx README, modified)
|
||||||
|
/// ```ignore
|
||||||
|
/// sqlx::query("DELETE FROM table").execute(&mut state.conn).await?;
|
||||||
|
/// ```
|
||||||
|
sql: Box<PgConnection>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl State {
|
||||||
|
pub(crate) async fn from_file(filename: String) -> State {
|
||||||
|
let conf = Config::from_file(filename).unwrap();
|
||||||
|
return State::from_config(conf).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn from_config(conf: Config) -> State {
|
||||||
|
let mut jobs = HashMap::new();
|
||||||
|
|
||||||
|
for (package_name, package) in conf.clone().packages {
|
||||||
|
match package.compilation {
|
||||||
|
Some(tmp) => {
|
||||||
|
jobs.insert(format!("packages.{}.compilation", package_name), tmp);
|
||||||
|
}
|
||||||
|
None => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (job_name, job) in package.packaging {
|
||||||
|
jobs.insert(
|
||||||
|
format!("packages.{}.packaging.{}", package_name, job_name),
|
||||||
|
job,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return State {
|
||||||
|
conf: conf.clone(),
|
||||||
|
jobs: jobs.clone(),
|
||||||
|
dependency_map: State::dependency_map(jobs, conf),
|
||||||
|
sql: logging::sql::start(5).await,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a hashmap mapping all job ids to what jobs depend on them (recursively)
|
||||||
|
///
|
||||||
|
/// Example output using the example toml:
|
||||||
|
///
|
||||||
|
/// ```json
|
||||||
|
/// {
|
||||||
|
/// "packages.some-librewolf-dependency.packaging.fedora": [
|
||||||
|
/// "packages.librewolf.compilation",
|
||||||
|
/// "packages.librewolf.packaging.fedora",
|
||||||
|
/// ],
|
||||||
|
/// "packages.some-librewolf-dependency.compilation": [
|
||||||
|
/// "packages.librewolf.compilation",
|
||||||
|
/// "packages.librewolf.packaging.fedora",
|
||||||
|
/// "packages.some-librewolf-dependency.packaging.fedora",
|
||||||
|
/// ],
|
||||||
|
/// "packages.librewolf.compilation": [
|
||||||
|
/// "packages.librewolf.packaging.fedora",
|
||||||
|
/// ],
|
||||||
|
/// }
|
||||||
|
/// ```
|
||||||
|
fn dependency_map(jobs: HashMap<String, Job>, conf: Config) -> HashMap<String, Vec<String>> {
|
||||||
|
let mut dep_map: HashMap<String, Vec<String>> = HashMap::new(); // holds job ids and every job they depend on (recursively) - not just specified dependencies, also packaging depending on compilation
|
||||||
|
|
||||||
|
for (job_id, _) in jobs.clone() {
|
||||||
|
let (_, package_name, _) = jod_id_to_metadata(job_id.clone());
|
||||||
|
|
||||||
|
for dep_name in conf
|
||||||
|
.packages
|
||||||
|
.get(&package_name)
|
||||||
|
.unwrap()
|
||||||
|
.dependencies
|
||||||
|
.clone()
|
||||||
|
{
|
||||||
|
let all_deps = recursive_deps_for_package(dep_name.clone(), conf.clone());
|
||||||
|
for dep in all_deps {
|
||||||
|
if !dep_map.contains_key(&dep) {
|
||||||
|
dep_map.insert(dep.clone(), Vec::new());
|
||||||
|
}
|
||||||
|
dep_map.get_mut(&dep).unwrap().push(job_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// add compilation jobs when relevant
|
||||||
|
for (package_name, package) in conf.packages {
|
||||||
|
if package.compilation.is_some() {
|
||||||
|
if !dep_map.contains_key(&format!("packages.{package_name}.compilation")) {
|
||||||
|
dep_map.insert(format!("packages.{package_name}.compilation"), Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (job_name, _) in package.packaging {
|
||||||
|
dep_map
|
||||||
|
.get_mut(&format!("packages.{package_name}.compilation"))
|
||||||
|
.unwrap()
|
||||||
|
.push(format!("packages.{package_name}.packaging.{job_name}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// deduplicate dependencies
|
||||||
|
for (_, deps) in dep_map.iter_mut() {
|
||||||
|
deps.dedup();
|
||||||
|
}
|
||||||
|
|
||||||
|
return dep_map;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue