switch to toml

2024-12-29 23:10:29 -06:00 · 2024-12-29 23:10:29 -06:00 · e6b521262d
commit e6b521262d
parent df0cf3b93d
12 changed files with 218 additions and 160 deletions
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@ -22,7 +22,7 @@
            "args": [
                "run",
                "-c",
-                "gregory.example.yml"
+                "gregory.example.toml"
            ],
            "cwd": "${workspaceFolder}"
        },
--- a/Cargo.lock
+++ b/Cargo.lock
@ -57,12 +57,6 @@ dependencies = [
 "windows-sys",
 ]

-[[package]]
-name = "anyhow"
-version = "1.0.95"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
-
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@ -155,7 +149,7 @@ dependencies = [
 "clap",
 "clap_complete",
 "serde",
- "serde_yml",
+ "toml",
 "uuid",
 ]

@ -187,28 +181,12 @@ version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"

-[[package]]
-name = "itoa"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
-
 [[package]]
 name = "libc"
 version = "0.2.169"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"

-[[package]]
-name = "libyml"
-version = "0.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3302702afa434ffa30847a83305f0a69d6abd74293b6554c18ec85c7ef30c980"
-dependencies = [
- "anyhow",
- "version_check",
-]
-
 [[package]]
 name = "memchr"
 version = "2.7.4"
@ -235,9 +213,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
+checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
 dependencies = [
 "proc-macro2",
 ]
@ -272,26 +250,20 @@ dependencies = [
 "getrandom",
 ]

-[[package]]
-name = "ryu"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
-
 [[package]]
 name = "serde"
-version = "1.0.216"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
+checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.216"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
+checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
 dependencies = [
 "proc-macro2",
 "quote",
@ -299,18 +271,12 @@ dependencies = [
 ]

 [[package]]
-name = "serde_yml"
-version = "0.0.12"
+name = "serde_spanned"
+version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
 dependencies = [
- "indexmap",
- "itoa",
- "libyml",
- "memchr",
- "ryu",
 "serde",
- "version_check",
 ]

 [[package]]
@ -321,15 +287,49 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"

 [[package]]
 name = "syn"
-version = "2.0.91"
+version = "2.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035"
+checksum = "9c786062daee0d6db1132800e623df74274a0a87322d8e183338e01b3d98d058"
 dependencies = [
 "proc-macro2",
 "quote",
 "unicode-ident",
 ]

+[[package]]
+name = "toml"
+version = "0.8.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.14"
@ -352,12 +352,6 @@ dependencies = [
 "rand",
 ]

-[[package]]
-name = "version_check"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@ -437,6 +431,15 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

+[[package]]
+name = "winnow"
+version = "0.6.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.7.35"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -3,13 +3,15 @@ name = "gregory"
 version = "0.1.1"
 edition = "2021"
 license = "GPL-3.0-only"
+repository = "https://git.askiiart.net/askiiart/gregory"
+readme = "README.md"

 [dependencies]
 alphanumeric-sort = "1.5.3"
 clap = { version = "4.5.23", features = ["derive"] }
 clap_complete = "4.5.40"
 serde = { version = "1.0.216", features = ["derive"] }
-serde_yml = "0.0.12"
+toml = "0.8.19"
 uuid = { version = "1.11.0", features = ["v7", "fast-rng"] }

 [profile.release]
--- a/README.md
+++ b/README.md
@ -13,14 +13,14 @@ Go look at [`docs/`](/docs/)
 - Add sources (similar to `sources` in a PKGBUILD)?
 - Add support for loading scripts rather than listing commands
 - Add multithreading
- Add better/custom grouping for when to run jobs (dependency system?)
- Add dependency system (automatic detection?)
+- Add better/custom grouping for when to run jobs (dependency system? (automatic detection?))
 - Add hook system
 - Add SQL database (maybe using `sqlx`?)
+  - Log stderr too ()

 ## Other stuff

- The formatting for the config file (`gregory.yml`) was heavily inspired by Drone's config.
+- The formatting for the config file (`gregory.toml`) was heavily inspired by Drone's config.
 - Why the name?
  - I was thinking to go with something dark and foreboding, since this is a program to control *everything* about a repo - it's the high command. But I couldn't think of anything and thought just naming it some lame random name instead would be way funnier. Hence, Gregory.
 - Gregory is a program, so it uses it/its pronouns. It also doesn't mind whether you capitalize its name or not, "gregory" or "Gregory" are fine, you can even shorten it if you want.
--- a/docs/cli-arguments.md
+++ b/docs/cli-arguments.md
@ -10,7 +10,7 @@ Usage: gregory run [OPTIONS]

 **Options:**

- `-c`, `--config`: Path to the config file; default: `gregory.yml`
+- `-c`, `--config`: Path to the config file; default: `gregory.toml`
 <!-- - `-d`, `--daemonize`: Whether to daemonize the program - not yet supported -->

 ## Generate shell completions `gen-completion`
--- a/docs/config-reference.md
+++ b/docs/config-reference.md
@ -1,7 +1,14 @@
 # Config Reference

- Default config location: `gregory.yml`
- Example: see [`gregory.example.yml`](/gregory.example.yml)
+- Default config location: `gregory.toml`
+- **Example**: see [`gregory.example.toml`](/gregory.example.toml)
+
+It's recommended to edit the config file using [Even Better TOML](https://marketplace.visualstudio.com/items?itemName=tamasfe.even-better-toml) for VS Code with the following options:
+
+```json
+    "evenBetterToml.formatter.indentTables": true,
+    "evenBetterToml.formatter.indentString": "  "
+```

 Note: This primarily uses LibreWolf and Fedora as examples of packages and distros. Also note that rather than separating by what distro, you can instead use those field to define which repo.

@ -34,9 +41,9 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr
  - *Root may be required for this argument*
  - If not specified, it will fall back to `max-threads`
 - `image` (string): The Docker image to run the job in **(required)**
- `commands` (sequence): The commands to run **(required)**
-  - TODO: Add command file/bash script instead
- `volumes` (sequence): Names of volumes as defined in [`volumes` (top level)](#volumes)
+- `commands` (array): The commands to run **(required)**
+  - Note than you can use single-quote strings instead for string literals - see [TOML docs](https://github.com/toml-lang/toml/blob/main/toml.md#string) for details
+- `volumes` (array): Names of volumes as defined in [`volumes` (top level)](#volumes)
 - `privileged` (bool): Whether the job's container should be privileged
 - `shell` (string): The shell to run the commands in
  - Default: `/bin/sh`
@ -45,19 +52,27 @@ Note: This primarily uses LibreWolf and Fedora as examples of packages and distr

 Example:

-```yml
-packages:
-  librewolf:
-    compilation:
-      image: 'debian'
-      commands:
-        - './mach build'
-    
-    packaging:
-      fedora:
-        image: 'lesbi-oops-i-mean/debian'
-        commands:
-          - './lesbiab package thingy'
+```toml
+[packages]
+
+  [packages.librewolf]
+
+    [packages.librewolf.compilation]
+    id = "1"
+    revision = "2"
+    threads = 8
+    image = "docker.io/library/debian"
+    commands = ["echo hi", "echo helloooooooooo"]
+    volumes = ["librewolf"]
+
+    [packages.librewolf.packaging.fedora]
+    threads = 8
+    image = "docker.io/library/fedora"
+    commands = [
+      "echo did you ever hear the tragedy of darth plageuis the wise?",
+      "echo it\\'s not a story the jedi would tell you",
+    ]
+    volumes = ["librewolf"]
 ```

 ### Compilation (optional)
@ -66,17 +81,18 @@ Defines the compilation of a program, if applicable. Stuff like Python scripts c

 It's defined in this format:

-```yml
-packages:
-  pkgname:
-    compilation:
-      image: 'fedora'
-      commands:
-        - 'echo hi'
+```toml
+[packages]

-  other-package:
-    compilation:
-      job-details-go-here:
+  [packages.librewolf]
+
+    [packages.librewolf.compilation]
+    id = "1"
+    revision = "2"
+    threads = 8
+    image = "docker.io/library/debian"
+    commands = ["echo hi", "echo helloooooooooo"]
+    volumes = ["librewolf"]
 ```

 ### Packaging
@ -85,19 +101,22 @@ Defines the packaging of a program into stuff like `.deb` or `.rpm` files.

 Example:

-```yml
-packages:
-  pkgname:
-    packaging:
-      distro-name:
-        image: 'fedora'
-        commands:
-          - 'echo hi'
-  
-  other-package:
-    packaging:
-      distro-name:
-        job-details-go-here:
+```toml
+[packages.librewolf.packaging.fedora]
+threads = 8
+image = "docker.io/library/fedora"
+commands = [
+  "echo did you ever hear the tragedy of darth plageuis the wise?",
+  "echo it\\'s not a story the jedi would tell you",
+]
+volumes = ["librewolf"]
+
+[packages.librewolf.packaging.debian]
+threads = 4
+image = "docker.io/library/debian"
+commands = [
+  "echo hiiiiiii"
+]
 ```

 Replace `distro-name` with the name of a distro, like `fedora` or `debian`
@ -108,19 +127,21 @@ Defines how to update a repo.

 Example:

-```yml
-update-repo:
-  distro-name:
-    image: 'fedora'
-    command:
-      - 'echo hi'
+```toml
+[update-repo]
+
+  [update-repo.fedora]
+  threads = 4
+  image = 'docker.io/library/fedora'
+  commands = ["echo hai"]
+  volumes = ["librewolf"]
 ```

 ## Volumes

 Lists a volume in Docker/Podman's volume format, to be used in [job configs](#job-config)

-```yml
-volumes:
-  librewolf: './local/path:/path/in/container'
+```toml
+[volumes]
+librewolf = "./local/path:/path-in-container"
 ```
--- a/gregory.example.toml
+++ b/gregory.example.toml
@ -0,0 +1,35 @@
+max-jobs = 4
+max-threads = 10
+log-level = 0
+
+[packages]
+
+  [packages.librewolf]
+
+    [packages.librewolf.compilation]
+    id = "1"
+    revision = "2"
+    threads = 8
+    image = "docker.io/library/debian"
+    commands = ["echo hi", "echo helloooooooooo"]
+    volumes = ["librewolf"]
+
+    [packages.librewolf.packaging.fedora]
+    threads = 8
+    image = "docker.io/library/fedora"
+    commands = [
+      "echo did you ever hear the tragedy of darth plageuis the wise?",
+      "echo it\\'s not a story the jedi would tell you",
+    ]
+    volumes = ["librewolf"]
+
+[update-repo]
+
+  [update-repo.fedora]
+  threads = 4
+  image = 'docker.io/library/fedora'
+  commands = ["echo hai"]
+  volumes = ["librewolf"]
+
+[volumes]
+librewolf = "./data/librewolf:/librewolf"
--- a/gregory.example.yml
+++ b/gregory.example.yml
@ -1,37 +0,0 @@
-max-jobs: 4
-max-threads: 10
-log-level: 0
-
-packages:
-  librewolf:
-    compilation:
-      id: 1
-      revision: 2
-      threads: 8
-      image: 'docker.io/library/debian'
-      commands:
-        - 'echo hi'
-        - 'echo helloooooooooo'
-      volumes:
-        - 'librewolf'
-    packaging:
-      fedora:
-        threads: 8
-        image: 'docker.io/library/fedora'
-        commands:
-          - 'echo did you ever hear the tragedy of darth plageuis the wise?'
-          - "echo it's not a story the jedi would tell you"
-        volumes:
-          - 'librewolf'
-
-update-repo:
-  fedora:
-    threads: 4
-    image: 'docker.io/library/fedora'
-    commands:
-      - 'echo hai'
-    volumes:
-      - 'librewolf'
-
-volumes:
-  librewolf: './data/librewolf:/librewolf'
--- a/src/cli.rs
+++ b/src/cli.rs
@ -19,7 +19,7 @@ pub enum Commands {
    ///Runs it
    Run {
        ///Path to the config file
-        #[arg(short, long, default_value = "gregory.yml")]
+        #[arg(short, long, default_value = "gregory.toml")]
        config: String,
        /* Not yet supported
        #[arg(short, long)]
--- a/src/data.rs
+++ b/src/data.rs
@ -14,7 +14,7 @@ pub(crate) struct Config {
    /// - 2: Info
    /// - 3: Debug
    #[serde(default = "log_level", rename = "log-level")]
-    // the rename lets it use `log-level` instead in the yaml file - this is not an alias, `log_level` in the yaml will *not* work
+    // the rename lets it use `log-level` instead in the toml file - this is not an alias, `log_level` in the toml will *not* work
    pub(crate) log_level: u8,
    /// Maximum number of jobs to run simultaneously
    #[serde(default = "max_jobs", rename = "max-jobs")]
@ -98,8 +98,8 @@ pub(crate) struct JobExitStatus {
 }

 pub(crate) fn config_from_file(filename: String) -> Config {
-    let yaml: Config = serde_yml::from_str(fs::read_to_string(filename).unwrap().as_str()).unwrap();
-    return yaml;
+    let toml: Config = toml::from_str(fs::read_to_string(filename).unwrap().as_str()).unwrap();
+    return toml;
 }

 // ==========================
--- a/src/main.rs
+++ b/src/main.rs
@ -5,6 +5,7 @@ use clap::{CommandFactory, Parser};
 use clap_complete::aot::{generate, Bash, Elvish, Fish, PowerShell, Zsh};
 use std::fs;
 use std::fs::create_dir_all;
+use std::fs::remove_dir_all;
 use std::fs::write;
 use std::fs::File;
 use std::io::stdout;
@ -84,13 +85,30 @@ fn run_job(conf: Config, job: Job) -> JobExitStatus {

    let container_name: String = format!("gregory-{}-{}-{}", job.id, job.revision, Uuid::now_v7());

-    let log_path = &format!("{}/logs/{container_name}.txt", conf.data_dir); // can't select fields in the format!() {} thing, have to do this
-    let log_dir: &Path = Path::new(log_path);
-    create_dir_all(log_dir.parent().unwrap()).unwrap();
-    write(log_path, job.commands.join("\n")).unwrap();
+    // create log path
+    let log_path = &format!("{}/logs/{container_name}", conf.data_dir); // can't select fields in the format!() {} thing, have to do this
+    let log_dir: &Path = Path::new(log_path).parent().unwrap();
+    create_dir_all(log_dir).unwrap();
+
+    // write the script
+    let script_path = &format!("{}/tmp/{container_name}.sh", conf.data_dir); // can't select fields in the format!() {} thing, have to do this
+                                                                             // create dir for the script
+    let script_dir: &Path = Path::new(script_path).parent().unwrap();
+    create_dir_all(script_dir).unwrap();
+    write(
+        script_path,
+        job.commands
+            //.iter()
+            //.map(|item| {
+            //    // TODO: FIGURE OUT HOW TO HANDLE IT ESCAPING IT OR WHATEVER AAAAAAAAAAAAA
+            //})
+            //.collect::<Vec<String>>()
+            .join("\n"),
+    )
+    .unwrap();

    // set permissions - *unix specific*
-    let mut perms = File::open(log_path)
+    let mut perms = File::open(script_path)
        .unwrap()
        .metadata()
        .unwrap()
@ -103,13 +121,22 @@ fn run_job(conf: Config, job: Job) -> JobExitStatus {
        format!("--name={container_name}"),
        format!("--cpus={threads}"),
        format!("--privileged={}", job.privileged),
-        format!("-v={log_path}:/gregory-entrypoint.sh"),
-        format!("--entrypoint=['{}', '/gregory-entrypoint.sh']", &job.shell),
-        job.clone().image
+        format!("-v={script_path}:/gregory-entrypoint.sh"),
+        format!(
+            "--entrypoint=[\"{}\", \"/gregory-entrypoint.sh\"]",
+            &job.shell
+        ),
+        job.clone().image,
    ];
+    println!("{:?}", cmd_args);
    let cmd_output = Command::new("podman").args(cmd_args).output().unwrap();
    let elapsed = now.elapsed();
+    // remove tmp dir
+    remove_dir_all(script_dir).unwrap();

+    // write logs - TEMPORARY
+    write(log_path, &cmd_output.stdout).unwrap();
+    write(format!("{log_path}.err"), &cmd_output.stderr).unwrap();

    println!("{:?}", cmd_output);

--- a/src/tests.rs
+++ b/src/tests.rs
@ -1,11 +1,18 @@
+use crate::data;
+#[cfg(test)]
 use alphanumeric_sort::sort_str_slice;
-use crate::data::*;

 #[test]
 fn test_config() {
    // It's a pain to make the config manually so I'm just doing this lol
-    let conf = "Config { log_level: 0, max_jobs: 4, max_threads: 10, packages: {\"librewolf\": Package { compilation: Some(Job { threads: 8, image: \"docker.io/library/debian\", commands: [\"cd ~/librewolf\", \"./mach build\"], volumes: Some([\"librewolf\"]), privileged: false }), packaging: {\"fedora\": Job { threads: 8, image: \"docker.io/library/fedora\", commands: [\"git clone http://example.com/librewolf-fedora-packaging.git && cd librewolf-fedora-packaging/\", \"do-rpm-stuff-idk\"], volumes: Some([\"librewolf\"]), privileged: false }} }}, update_repo: {\"fedora\": Job { threads: 4, image: \"docker.io/library/fedora\", commands: [\"idkkkkk\"], volumes: Some([\"librewolf\"]), privileged: false }}, volumes: {\"librewolf\": \"./data/librewolf:/librewolf\"} }";
-    assert_eq!(format!("{:?}", config_from_file("gregory.example.yml".to_string())), conf);
+    let conf = "Config { log_level: 0, max_jobs: 4, max_threads: 10, data_dir: \"./data\", packages: {\"librewolf\": Package { compilation: Some(Job { id: \"1\", revision: \"2\", threads: 8, image: \"docker.io/library/debian\", commands: [\"echo hi\", \"echo helloooooooooo\"], volumes: Some([\"librewolf\"]), privileged: false, shell: \"/bin/sh\" }), packaging: {\"fedora\": Job { id: \"-1\", revision: \"1\", threads: 8, image: \"docker.io/library/fedora\", commands: [\"echo did you ever hear the tragedy of darth plageuis the wise?\", \"echo it\\\\'s not a story the jedi would tell you\"], volumes: Some([\"librewolf\"]), privileged: false, shell: \"/bin/sh\" }} }}, update_repo: {\"fedora\": Job { id: \"-1\", revision: \"1\", threads: 4, image: \"docker.io/library/fedora\", commands: [\"echo hai\"], volumes: Some([\"librewolf\"]), privileged: false, shell: \"/bin/sh\" }}, volumes: {\"librewolf\": \"./data/librewolf:/librewolf\"} }";
+    assert_eq!(
+        format!(
+            "{:?}",
+            data::config_from_file("gregory.example.toml".to_string())
+        ),
+        conf
+    );
 }

 #[test]