functional prototype
update docs and structs, add actual logic, now it actually works
This commit is contained in:
askiiart
parent
acf4c9e84d
commit
df0cf3b93d
GPG key ID:
EA85979611654C30
10 changed files with 322 additions and 24 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1 +1,2 @@
|
|||
/target
|
||||
/data
|
||||
49
.vscode/launch.json
vendored
Normal file
49
.vscode/launch.json
vendored
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug executable 'gregory'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"build",
|
||||
"--bin=gregory",
|
||||
"--package=gregory",
|
||||
],
|
||||
"filter": {
|
||||
"name": "gregory",
|
||||
"kind": "bin"
|
||||
}
|
||||
},
|
||||
"args": [
|
||||
"run",
|
||||
"-c",
|
||||
"gregory.example.yml"
|
||||
],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in executable 'gregory'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--bin=gregory",
|
||||
"--package=gregory"
|
||||
],
|
||||
"filter": {
|
||||
"name": "gregory",
|
||||
"kind": "bin"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
}
|
||||
]
|
||||
}
|
||||
108
Cargo.lock
generated
108
Cargo.lock
generated
|
|
@ -63,6 +63,18 @@ version = "1.0.95"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.5.23"
|
||||
|
|
@ -124,15 +136,27 @@ version = "1.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gregory"
|
||||
version = "0.1.0"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"alphanumeric-sort",
|
||||
"clap",
|
||||
"clap_complete",
|
||||
"serde",
|
||||
"serde_yml",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -169,6 +193,12 @@ version = "1.0.14"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.169"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
|
||||
|
||||
[[package]]
|
||||
name = "libyml"
|
||||
version = "0.0.5"
|
||||
|
|
@ -185,6 +215,15 @@ version = "2.7.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.92"
|
||||
|
|
@ -203,6 +242,36 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.18"
|
||||
|
|
@ -273,12 +342,28 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
|
|
@ -351,3 +436,24 @@ name = "windows_x86_64_msvc"
|
|||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
[package]
|
||||
name = "gregory"
|
||||
version = "0.1.0"
|
||||
version = "0.1.1"
|
||||
edition = "2021"
|
||||
license = "GPL-3.0-only"
|
||||
|
||||
[dependencies]
|
||||
alphanumeric-sort = "1.5.3"
|
||||
|
|
@ -9,6 +10,7 @@ clap = { version = "4.5.23", features = ["derive"] }
|
|||
clap_complete = "4.5.40"
|
||||
serde = { version = "1.0.216", features = ["derive"] }
|
||||
serde_yml = "0.0.12"
|
||||
uuid = { version = "1.11.0", features = ["v7", "fast-rng"] }
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
This is Gregory. Gregory controls repos. Gregory keeps track of updating repos, trying to be simple and elegant, but enough.
|
||||
|
||||
## THIS IS A PROTOTYPE
|
||||
|
||||
## Documentation
|
||||
|
||||
Go look at [`docs/`](/docs/)
|
||||
|
|
|
|||
3
docs/behind-the-scenes/commands.md
Normal file
3
docs/behind-the-scenes/commands.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
# How commands are run
|
||||
|
||||
I was unable to find a way to directly run *multiple* commands via Docker/Podman. Instead of doing that, greg puts all the commands in a temporary script, mounts it inside, and then run it with
|
||||
|
|
@ -14,6 +14,8 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr
|
|||
- Default is CPU's threads - 2
|
||||
- `max-jobs` (integer): The maximum number of jobs to be run at once
|
||||
- Default is 1
|
||||
- `data-dir` (string): The path to put data for job logs and stuff
|
||||
- **Temporary**, will be removed once SQL database support is added
|
||||
|
||||
**Multithreading notes (IMPORTANT)**: Gregory will first run compilation jobs, then packaging jobs for whatever programs are done, then run the `update-repo` for whichever distros are finished. For this reason, the distro names listed under `packaging` and `update-repo` *must* match.
|
||||
|
||||
|
|
@ -21,16 +23,23 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr
|
|||
|
||||
## Job config
|
||||
|
||||
- `id` (string): An ID to identify the job, such as the compilation of a program **(highly recommended)**
|
||||
- Default is `-1` for unassigned
|
||||
- If you just want to run stuff, you don't need this, but it's *highly* recommended as it allows you to filter your logs.
|
||||
- `revision` (string): A revision id for the job, such as a version number for a compilation script
|
||||
- Default is `1`
|
||||
- `threads` (integer): The maximum number of vCPUs/threads to dedicate to a job; this can be a fractional number
|
||||
- Set this as less than or equal to the max number of threads the thing you're running will use
|
||||
- See `--cpus` in the [`podman run` docs](https://docs.podman.io/en/latest/markdown/podman-run.1.html#cpus)
|
||||
- *Root may be required for this argument*
|
||||
- If not specified, it will fall back to `max-threads`
|
||||
- `image` (string): The Docker image to run the job in *(required)*
|
||||
- `commands` (sequence): The commands to run *(required)*
|
||||
- `image` (string): The Docker image to run the job in **(required)**
|
||||
- `commands` (sequence): The commands to run **(required)**
|
||||
- TODO: Add command file/bash script instead
|
||||
- `volumes` (sequence): Names of volumes as defined in [`volumes` (top level)](#volumes)
|
||||
- `privileged` (bool): Whether the job's container should be privileged
|
||||
- `shell` (string): The shell to run the commands in
|
||||
- Default: `/bin/sh`
|
||||
|
||||
## Packages (`packages`)
|
||||
|
||||
|
|
|
|||
83
src/data.rs
83
src/data.rs
|
|
@ -1,6 +1,7 @@
|
|||
//! Data structs. used by gregory and stuff for handling them
|
||||
|
||||
use serde::Deserialize;
|
||||
use std::time;
|
||||
use std::{collections::HashMap, fs, thread};
|
||||
|
||||
/// The config for gregory
|
||||
|
|
@ -12,57 +13,88 @@ pub(crate) struct Config {
|
|||
/// - 1: Warning
|
||||
/// - 2: Info
|
||||
/// - 3: Debug
|
||||
#[serde(default = "log_level", rename = "log-level")] // the rename lets it use `log-level` instead in the yaml file - this is not an alias, `log_level` in the yaml will *not* work
|
||||
log_level: u8,
|
||||
#[serde(default = "log_level", rename = "log-level")]
|
||||
// the rename lets it use `log-level` instead in the yaml file - this is not an alias, `log_level` in the yaml will *not* work
|
||||
pub(crate) log_level: u8,
|
||||
/// Maximum number of jobs to run simultaneously
|
||||
#[serde(default = "max_jobs", rename = "max-jobs")]
|
||||
max_jobs: u32,
|
||||
pub(crate) max_jobs: u32,
|
||||
/// Maximum number of threads to use
|
||||
#[serde(default = "max_threads", rename = "max-threads")]
|
||||
max_threads: u32,
|
||||
pub(crate) max_threads: u32,
|
||||
#[serde(default = "data", rename = "data-dir")]
|
||||
pub(crate) data_dir: String,
|
||||
/// Holds the packages, including their compilation and packaging
|
||||
///
|
||||
/// Format: `{ "librewolf": Package { compilation, packaging } }`
|
||||
///
|
||||
/// See [`Package`] for details
|
||||
packages: HashMap<String, Package>,
|
||||
pub(crate) packages: HashMap<String, Package>,
|
||||
/// The jobs for updating the repo, organized by distro/repo name
|
||||
#[serde(rename = "update-repo")]
|
||||
update_repo: HashMap<String, Job>,
|
||||
pub(crate) update_repo: HashMap<String, Job>,
|
||||
/// All volumes, organized like this:
|
||||
///
|
||||
/// Format: `{ "librewolf": "./data/librewolf:/librewolf" }` - like Docker/Podman formatting
|
||||
#[serde(default = "volumes")]
|
||||
volumes: HashMap<String, String>,
|
||||
pub(crate) volumes: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub(crate) struct Job {
|
||||
/// An ID to identify the job, such as the compilation of a program
|
||||
#[serde(default = "id")]
|
||||
pub(crate) id: String,
|
||||
#[serde(default = "revision")]
|
||||
pub(crate) revision: String,
|
||||
/// How many threads to limit this job to; recommended to set it to the max threads the job will use
|
||||
///
|
||||
/// If `threads` isn't specified, it will fall back to `max_threads` (from [`Config`]); the same behavior applies if `threads` is greater than `max_threads`
|
||||
#[serde(default = "job_threads")]
|
||||
threads: u32,
|
||||
/// The OCi image to run it in
|
||||
pub(crate) threads: u32,
|
||||
/// The OCI image to run it in
|
||||
///
|
||||
/// For example, `docker.io/library/debian:latest`
|
||||
image: String,
|
||||
pub(crate) image: String,
|
||||
/// The commands to run in the job
|
||||
commands: Vec<String>,
|
||||
volumes: Option<Vec<String>>,
|
||||
/// Whether the job should be privileged
|
||||
pub(crate) commands: Vec<String>,
|
||||
/// A list of all volumes given their name - see [`Config`] -> `volumes`
|
||||
pub(crate) volumes: Option<Vec<String>>,
|
||||
/// Whether the job W be privileged
|
||||
///
|
||||
/// Defauolt: false
|
||||
#[serde(default = "privileged")]
|
||||
privileged: bool,
|
||||
pub(crate) privileged: bool,
|
||||
#[serde(default = "shell")]
|
||||
pub(crate) shell: String,
|
||||
}
|
||||
|
||||
/// Holds the data for a certain package's config
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub(crate) struct Package {
|
||||
/// The compilation [`Job`] - optional
|
||||
compilation: Option<Job>,
|
||||
pub(crate) compilation: Option<Job>,
|
||||
/// The packaging [`Job`]s, organized by the distro/repo name
|
||||
packaging: HashMap<String, Job>,
|
||||
pub(crate) packaging: HashMap<String, Job>,
|
||||
}
|
||||
|
||||
/// The exit status and stuff for a [`Job`]
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct JobExitStatus {
|
||||
pub(crate) job: Job,
|
||||
/// The [`Job`] this status is from
|
||||
///
|
||||
/// This is stored as a u16 rather than a u8 so that 65535 can be returned if there is no exit code rather than doing an Option or something, which I fear will probably come back to haunt me, but whatever
|
||||
pub(crate) exit_code: u16,
|
||||
/// Where the log is
|
||||
///
|
||||
/// TEMPORARY
|
||||
/// TODO: Have main() handle logs and writing them to the database, not doing it in run_job()
|
||||
pub(crate) log_path: String,
|
||||
/// How long it took to run the job
|
||||
pub(crate) duration: time::Duration,
|
||||
/// The name of the container this job ran in
|
||||
pub(crate) container_name: String,
|
||||
}
|
||||
|
||||
pub(crate) fn config_from_file(filename: String) -> Config {
|
||||
|
|
@ -114,3 +146,22 @@ pub(crate) fn job_threads() -> u32 {
|
|||
pub(crate) fn privileged() -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Default (`/bin/sh`) for which shell to use
|
||||
pub(crate) fn shell() -> String {
|
||||
return "/bin/sh".to_string();
|
||||
}
|
||||
|
||||
/// Default id (`-1`)
|
||||
pub(crate) fn id() -> String {
|
||||
return "-1".to_string();
|
||||
}
|
||||
|
||||
/// Default revision (`1`)
|
||||
pub(crate) fn revision() -> String {
|
||||
return "1".to_string();
|
||||
}
|
||||
|
||||
pub(crate) fn data() -> String {
|
||||
return "./data".to_string();
|
||||
}
|
||||
|
|
|
|||
83
src/main.rs
83
src/main.rs
|
|
@ -3,7 +3,16 @@ use crate::data::*;
|
|||
use alphanumeric_sort::sort_str_slice;
|
||||
use clap::{CommandFactory, Parser};
|
||||
use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh};
|
||||
use std::fs;
|
||||
use std::fs::create_dir_all;
|
||||
use std::fs::write;
|
||||
use std::fs::File;
|
||||
use std::io::stdout;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::time::Instant;
|
||||
use uuid::Uuid;
|
||||
|
||||
mod cli;
|
||||
mod data;
|
||||
|
|
@ -40,9 +49,75 @@ fn main() {
|
|||
fn run(config_path: String) {
|
||||
let config = config_from_file(config_path);
|
||||
|
||||
println!("{:?}", config);
|
||||
println!("{:#?}", config);
|
||||
|
||||
let mut jobs: Vec<Job> = Vec::new();
|
||||
|
||||
for (_, package) in config.clone().packages {
|
||||
match package.compilation {
|
||||
Some(tmp) => {
|
||||
jobs.push(tmp);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
|
||||
for (_, job) in package.packaging {
|
||||
jobs.push(job);
|
||||
}
|
||||
}
|
||||
|
||||
for (_, job) in config.clone().update_repo {
|
||||
jobs.push(job);
|
||||
}
|
||||
|
||||
for job in jobs {
|
||||
println!("{:#?}", run_job(config.clone(), job));
|
||||
}
|
||||
}
|
||||
|
||||
fn run_job(max_threads: u32, job: Job) {
|
||||
|
||||
}
|
||||
fn run_job(conf: Config, job: Job) -> JobExitStatus {
|
||||
// limit threads to max_threads in the config
|
||||
let mut threads: u32 = job.threads;
|
||||
if job.threads > conf.max_threads {
|
||||
threads = conf.max_threads;
|
||||
}
|
||||
|
||||
let container_name: String = format!("gregory-{}-{}-{}", job.id, job.revision, Uuid::now_v7());
|
||||
|
||||
let log_path = &format!("{}/logs/{container_name}.txt", conf.data_dir); // can't select fields in the format!() {} thing, have to do this
|
||||
let log_dir: &Path = Path::new(log_path);
|
||||
create_dir_all(log_dir.parent().unwrap()).unwrap();
|
||||
write(log_path, job.commands.join("\n")).unwrap();
|
||||
|
||||
// set permissions - *unix specific*
|
||||
let mut perms = File::open(log_path)
|
||||
.unwrap()
|
||||
.metadata()
|
||||
.unwrap()
|
||||
.permissions();
|
||||
PermissionsExt::set_mode(&mut perms, 0o755);
|
||||
|
||||
let now = Instant::now();
|
||||
let cmd_args: Vec<String> = vec![
|
||||
"run".to_string(),
|
||||
format!("--name={container_name}"),
|
||||
format!("--cpus={threads}"),
|
||||
format!("--privileged={}", job.privileged),
|
||||
format!("-v={log_path}:/gregory-entrypoint.sh"),
|
||||
format!("--entrypoint=['{}', '/gregory-entrypoint.sh']", &job.shell),
|
||||
job.clone().image
|
||||
];
|
||||
let cmd_output = Command::new("podman").args(cmd_args).output().unwrap();
|
||||
let elapsed = now.elapsed();
|
||||
|
||||
|
||||
println!("{:?}", cmd_output);
|
||||
|
||||
return JobExitStatus {
|
||||
container_name: container_name,
|
||||
duration: elapsed,
|
||||
job: job,
|
||||
exit_code: cmd_output.status.code().ok_or_else(|| 65535).unwrap() as u16,
|
||||
log_path: log_path.clone(),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue