commit 82252c8a5a142b986f00149002b8e428a0a35a90 Author: askiiart Date: Sun Oct 15 22:14:31 2023 -0500 Initial commit - mergerfs + snapraid setup only diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c13a587 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.venv/ +site/ \ No newline at end of file diff --git a/docs/blog/index.md b/docs/blog/index.md new file mode 100644 index 0000000..c58f16c --- /dev/null +++ b/docs/blog/index.md @@ -0,0 +1,2 @@ +# Blog + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..37e8e29 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,3 @@ +# Wiki + +Wiki and stuff. \ No newline at end of file diff --git a/docs/mergerfs-snapraid-setup.md b/docs/mergerfs-snapraid-setup.md new file mode 100644 index 0000000..833089d --- /dev/null +++ b/docs/mergerfs-snapraid-setup.md @@ -0,0 +1,99 @@ +# mergerfs + SnapRAID setup + +## mergerfs + +`/etc/fstab`: + +```txt +# Data drives +UUID=2eeb4386-e26e-4340-9747-74fb3d18dd57 /mnt/disk1 xfs defaults 0 0 +UUID=e030fa2f-a9b4-4788-8d10-05cf5031f9c0 /mnt/disk2 xfs defaults 0 0 +UUID=f9a79cef-3959-4aa8-a9c9-af54660cb755 /mnt/disk3 xfs defaults 0 0 +UUID=c7283793-37c6-4b01-95e1-cca26af0da8f /mnt/cache btrfs defaults 0 0 +UUID=8d8cb485-eb23-475b-a9c1-03b8309cdbaa /mnt/parity xfs defaults 0 0 + +# mergerfs +/mnt/disk1:/mnt/disk2:/mnt/disk3:/mnt/cache /mnt/user fuse.mergerfs defaults,allow_other,use_ino,category.create=lfs,moveonenospc=true,minfreespace=512G,cache.files=partial,dropcacheonclose=true 0 0 +/mnt/disk1:/mnt/disk2:/mnt/disk3 /mnt/hdds fuse.mergerfs defaults,allow_other,use_ino,category.create=mfs,moveonenospc=true,minfreespace=25M,cache.files=partial,dropcacheonclose=true 0 0``` +``` + +Command equivalent to fstab mergerfs mounts: + +```txt +mergerfs -o defaults,allow_other,use_ino,category.create=lfs,moveonenospc=true,minfreespace=512G,cache.files=partial,dropcacheonclose=true /mnt/disk1:/mnt/disk2:/mnt/disk3:/mnt/cache /mnt/user +mergerfs -o defaults,allow_other,use_ino,category.create=mfs,moveonenospc=true,minfreespace=25M,cache.files=partial,dropcacheonclose=true /mnt/disk1:/mnt/disk2:/mnt/disk3 /mnt/hdds +``` + +## Caching and mover + +### Topology + +- `/mnt/user` - Combined mergerfs pool +- `/mnt/hdds` - HDD-only mergerfs pool +- `/mnt/cache` - cache mount point +- `/mnt/disk*` - Each HDD, starting from 1. `/mnt/disk1`, `/mnt/disk2`, etc. + +### Mover script + +Stolen from [here](https://raw.githubusercontent.com/trapexit/mergerfs/latest-release/tools/mergerfs.time-based-mover) + +It moves anything that hasn't been access in more than 3 days. + +Requires `rsync` - install with `apt update && apt install rsync` + +```sh +#!/usr/bin/env sh + +if [ $# != 3 ]; then + echo "usage: $0 " + exit 1 +fi + +CACHE="${1}" +BACKING="${2}" +N=${3} + +find "${CACHE}" -type f -atime +${N} -printf '%P\n' | \ + rsync --files-from=- -axqHAXWES --preallocate --remove-source-files "${CACHE}/" "${BACKING}/" +``` + +Located at `/root/mergerfs.time-based-mover.sh`, runs daily at 4 AM: + +```cron +0 4 * * * /root/mergerfs.time-based-mover.sh /mnt/cache /mnt/hdds 3 +``` + +## SnapRAID + +### `/etc/snapraid.conf` + +The list of files is stored on multiple disks in `snapraid.content`, and the parity is just one file (`snapraid.parity`) + +```txt +parity /mnt/parity/snapraid.parity +content /var/snapraid/snapraid.content +content /mnt/disk1/snapraid.content +content /mnt/disk2/snapraid.content +content /mnt/disk3/snapraid.content +data d1 /mnt/disk1/ +data d2 /mnt/disk2/ +data d3 /mnt/disk3/ +``` + +### Syncing + +Run daily at 2 AM (`/etc/cron.d/snapraid-sync`): + +```cron +* 02 * * * /root/snapraid-sync.sh +``` + +`/root/snapraid-sync.sh`: + +```sh +#!/usr/bin/env sh +CONTAINERS=$(docker ps -q) +docker stop $CONTAINERS +snapraid sync +docker start $CONTAINERS +``` diff --git a/images/askiiart.gif b/images/askiiart.gif new file mode 100644 index 0000000..45e3cfb Binary files /dev/null and b/images/askiiart.gif differ diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..9bc9b10 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,85 @@ +site_name: Wiki +site_url: https://wiki.askiiart.net +repo_url: https://git.askiiart.net/askiiart/wiki +repo_name: askiiart/wiki + +theme: + name: material + custom_dir: overrides + language: en + favicon: images/askiiart.gif + icon: + repo: fontawesome/brands/git-alt + logo: fontawesome/regular/folder-open + palette: + # Palette toggle for light mode + - scheme: default + primary: blue + accent: teal + toggle: + icon: material/brightness-7 + name: Switch to dark mode + # Palette toggle for dark mode + - scheme: slate + accent: teal + primary: deep orange + toggle: + icon: material/brightness-4 + name: Switch to light mode + font: + code: Jet Brains Mono + features: + - navigation.instant + - navigation.instant.prefetch + - navigation.sections # prevents collapsible sections + - navigation.indexes # section index pages + #- navigation.expand #expands sections by default + #- navigation.tabs # section tabs along the top + - toc.follow + - toc.integrate # moves toc to the left + # insiders only + - navigation.path # breadcrumbs at the top of each page + - search.suggest +plugins: + - blog: + categories: false + archive: false + #- git-revision-date + - search +markdown_extensions: + - admonition # enables coloured blocks mid article + - attr_list # improves image handling + - md_in_html # grids (insiders) + - pymdownx.details # enables collapsible admonitions + - footnotes + - toc: + permalink: true + toc_depth: 4 + - abbr # enables glossary + - pymdownx.snippets: # enables glossary + auto_append: + - overrides/glossary.md + - pymdownx.keys # display keyboard commands nicely ++ctrl+alt+del++ + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg + - pymdownx.caret + - pymdownx.mark + - pymdownx.tilde +copyright: 2020 - 2023 © Alex Kretzschmar - Licensed under GPLv3 +extra_css: + - extra.css +extra: + social: + - icon: 'fontawesome/brands/github' + link: 'https://github.com/askiiart' + name: askiiart - GitHub + - icon: 'fontawesome/brands/git-alt' + link: 'https://git.askiiart.net/askiiart' + name: askiiart - self-hosted Gitea + - icon: 'fontawesome/brands/mastodon' + link: 'https://fedi.askiiart.net/askiiart' + name: askiiart - Fedi diff --git a/overrides/404.html b/overrides/404.html new file mode 100644 index 0000000..a01ed7b --- /dev/null +++ b/overrides/404.html @@ -0,0 +1,9 @@ +{% extends "main.html" %} + + +{% block content %} +

404 - Not found

+
+

You will be redirected to the main home page automatically shortly.

+ +{% endblock %} \ No newline at end of file diff --git a/overrides/glossary.md b/overrides/glossary.md new file mode 100644 index 0000000..a80b20d --- /dev/null +++ b/overrides/glossary.md @@ -0,0 +1,14 @@ +*[fstab]: The file that lists what to mount on boot, and where to mount it. +*[snapraid.conf]: The SnapRAID config file +*[mergerfs]: Combines multiple directories into one - see the mergerfs docs or Perfect Media Server wiki for more details. +*[snapraid]: A parity-based backup solution - see the SnapRAID docs or Perfect Media Server wiki for more details +*[SnapRAID]: A parity-based backup solution - see the SnapRAID docs or Perfect Media Server wiki for more details +*[PMS]: Perfect Media Server +*[Plex]: A proprietary media server software +*[Jellyfin]: A FOSS media server software +*[Docker]: A popular containerization platform +*[Podman]: A FOSS drop-in replacement for Docker +*[\/mnt\/user]: Combined mergerfs pool +*[\/mnt\/hdds]: HDD-only mergerfs pool +*[\/mnt\/cache]: cache mount point +*[\/mnt\/disk\*]: Each HDD, starting from 1. `/mnt/disk1`, `/mnt/disk2`, etc. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d81a2cc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +mkdocs +mkdocs-material +mkdocs-material-extensions \ No newline at end of file