Rework immutability and overlays

.gitignore

@@ -2,3 +2,5 @@
 /blend-settings/build/
 /blend-settings/dist/
 /blend-settings/package-lock.json
+*.o
+/overlayfs-tools/overlayfs-tools

blend-settings/src/index.html

@@ -18,7 +18,7 @@
         Containers</button>
       <button class="btn btn-outline-light shadow-none d-none" id="android-button" onclick="page('android')">Android
         Apps</button>
-      <button class="btn btn-outline-light shadow-none" id="system-button" onclick="page('system')">System</button>
+      <!-- <button class="btn btn-outline-light shadow-none" id="system-button" onclick="page('system')">System</button> -->
     </div>
   </div>
 

blend-system

@@ -70,51 +70,39 @@ def info(msg):
 def error(err):
     print (colors.bold + colors.fg.red + '>> e: ' + colors.reset + colors.bold + err + colors.reset)
 
-def load_prev_state():
-    subprocess.call(['rm', '-rf', '/blend/overlay/current'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    subprocess.call(['mkdir', '-p', '/blend/overlay/current/usr'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    subprocess.call(['tar', '-xvpzf', f'/blend/states/state{current_state()}.tar.gz', '-C', '/blend/overlay/current/usr', '--numeric-owner'],
-                        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    subprocess.call(['rm', '-f', f'/blend/states/state{current_state()}.tar.gz'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-
 ### END
 
 def current_state():
     _state = -1
-    for s in os.listdir('/blend/states'):
+    for s in os.listdir('/.states'):
-        if re.match(r'^state([0-9]+)\.tar\.gz$', s):
+        if re.match(r'^state([0-9]+)\.squashfs$', s):
             if int(s[5:-7]) > _state:
                 _state = int(s[5:-7])
     return _state
 
 def save_state():
-    subprocess.call(['mkdir', '-p', '/blend/states'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    subprocess.call(['mkdir', '-p', '/.states'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
     state = current_state() + 1
 
-    subprocess.call(r"find /blend/states/ -type f -not -name 'state" + str(state - 1) + ".tar.gz' -print0 | xargs -0 -I {} rm {}", stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
+    subprocess.call(['bash', '-c', 'rm -f /.states/*.tmp'])
-    subprocess.call(['tar', '-C', '/blend/overlay/current/usr', '-cpzf', f'/blend/states/state{state}.tar.gz', '.'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+    if subprocess.call(['mksquashfs', '/usr', f'/.states/state{state}.squashfs.tmp', '-no-compression'], stdout=sys.stdout, stderr=sys.stderr) == 0:
+        subprocess.call(['rm', '-rf', 'add-squashfs'], cwd='/tmp')
+        subprocess.call(['mkdir', '-p', 'add-squashfs'], cwd='/tmp')
+        subprocess.call(['cp', '-a', '/var/lib', 'add-squashfs/varlib'], cwd='/tmp')
+        if subprocess.call(['mksquashfs', 'add-squashfs', f'/.states/state{state}.squashfs.tmp', '-no-compression'], cwd='/tmp') == 0:
+            subprocess.call(['mv', f'/.states/state{state}.squashfs.tmp', f'/.states/state{state}.squashfs'])
+        else:
+            error('state creation failed')
+            exit(1)
+    else:
+        error('state creation failed')
+        exit(1)
+
     info(f'saved state {state}')
 
-def autosave_state():
-    while True:
-        if not os.path.isfile('/blend/states/.disable_states'):
-            save_state()
-        time.sleep(12*60*60) # XXX: make this configurable
-
-def toggle_states():
-    if os.path.isfile('/blend/states/.disable_states'):
-        os.remove('/blend/states/.disable_states')
-        info('enabled saving states automatically (every 6 hours; this will be configurable in future releases)')
-    else:
-        subprocess.call(['touch', '/blend/states/.disable_states'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        info('disabled saving states automatically')
-
 def rollback():
-    if current_state() == -1:
+    info("Rollback hasn't been implemented yet.")
-        error('no states present')
-        exit(1)
-    subprocess.call(['touch', '/blend/states/.load_prev_state'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    info(f'will rollback to the previous state on the next boot')
 
 description = f'''
 {colors.bold}{colors.fg.purple}Usage:{colors.reset}
@@ -126,7 +114,6 @@ description = f'''
   {colors.bold}help{colors.reset}                  Show this help message and exit.
   {colors.bold}version{colors.reset}               Show version information and exit.
   {colors.bold}save-state{colors.reset}            Save the current state (backup).
-  {colors.bold}toggle-states{colors.reset}         Enable/disable automatic state creation (you can still manually save states).
   {colors.bold}rollback{colors.reset}              Rollback to previous state.
 
 {colors.bold}{colors.fg.purple}options for commands{colors.reset}:
@@ -142,8 +129,6 @@ parser = argparse.ArgumentParser(description=description, usage=argparse.SUPPRES
 command_map = { 'help': 'help',
                 'version': 'version',
                 'save-state': save_state,
-                'toggle-states': toggle_states,
-                'autosave-state': autosave_state,
                 'rollback': rollback }
 parser.add_argument('command', choices=command_map.keys(), help=argparse.SUPPRESS)
 parser.add_argument('-v', '--version', action='version', version=f'%(prog)s {__version}', help=argparse.SUPPRESS)

blend-system.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Save system state
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/blend-system autosave-state
-User=root
-
-[Install]
-WantedBy=multi-user.target

blend.hook

@@ -1,30 +1,56 @@
 #!/bin/bash
 
 run_latehook() {
-    if [[ -f /new_root/blend/states/.load_prev_state ]] && compgen -G "/new_root/blend/states/state+([0-9]).tar.gz" >/dev/null; then
+    echo
-        rm -rf /new_root/blend/overlay/current
+
-        mkdir -p /new_root/blend/overlay/current/usr
+    if [[ "$abort_staging" == true ]]; then
-        c=0
+        echo '[ BLEND ] Not applying system changes made in previous boot.'
-        for i in $(compgen -G "/new_root/blend/states/state*.tar.gz"); do
+        rm -rf '/new_root/.upperdir' '/new_root/.workdir'; mkdir -p '/new_root/.upperdir' '/new_root/.workdir'
-            n="${i:19:-7}"
-            if [[ "$n" -gt "$c" ]]; then
-                c="$n"
-            fi
-        done
-        tar -xvpzf "/new_root/blend/states/state${c}.tar.gz" -C "/new_root/blend/overlay/current/usr" --numeric-owner &>/dev/null
-        rm -f "/new_root/blend/states/state${c}.tar.gz" "/new_root/blend/states/.load_prev_state"
     fi
 
-    mkdir -p /new_root/blend/overlay/current/usr/bin \
+    if [[ -d "/new_root/blend/overlay/current" ]]; then
-             /new_root/blend/overlay/current/usr/sbin \
+        echo '[ BLEND ] Detected old version of overlay implementation, switching.'
-             /new_root/blend/overlay/current/usr/share
+        rm -rf /new_root/.upperdir /new_root/.workdir
+        mv /new_root/blend/overlay/current/usr /new_root/.upperdir
+        rm -rf /new_root/blend
+    fi
 
-    mkdir -p /new_root/usr/bin \
+    # Broken attempt at getting rollback and snapshots working.
-             /new_root/usr/sbin \
+    #
-             /new_root/usr/share
+    # if [[ -L "/new_root/.states/rollback.squashfs" ]] && [[ -e "/new_root/.states/rollback.squashfs" ]]; then
-    rm -rf /new_root/blend/overlay/workdir_1 /new_root/blend/overlay/workdir_2 /new_root/blend/overlay/workdir_3
+    #     echo -n '[ BLEND ] Rolling back to selected state. Do __not__ power off or reboot.'
-    mkdir -p /new_root/blend/overlay/workdir_1 /new_root/blend/overlay/workdir_2 /new_root/blend/overlay/workdir_3
+    #     echo
-    mount -t overlay overlay -o 'lowerdir=/new_root/usr/bin,upperdir=/new_root/blend/overlay/current/usr/bin,workdir=/new_root/blend/overlay/workdir_1' /new_root/usr/bin -o index=off
+    #     cd /new_root
-    mount -t overlay overlay -o 'lowerdir=/new_root/usr/sbin,upperdir=/new_root/blend/overlay/current/usr/sbin,workdir=/new_root/blend/overlay/workdir_2' /new_root/usr/sbin -o index=off
+    #     unsquashfs /new_root/.states/rollback.squashfs && (
-    mount -t overlay overlay -o 'lowerdir=/new_root/usr/share,upperdir=/new_root/blend/overlay/current/usr/share,workdir=/new_root/blend/overlay/workdir_3' /new_root/usr/share -o index=off
+    #         for i in bin include lib lib32 share src; do
+    #             rm -rf rm -rf /new_root/.workdir/"$i" rm -rf /new_root/.upperdir/"$i" /new_root/usr/"$i"
+    #             mv squashfs-root/"$i" /new_root/usr
+    #         done
+    #         rm -rf /new_root/.workdir/varlib /new_root/.upperdir/varlib /new_root/var/lib
+    #         mkdir -p /new_root/var/lib
+    #         mv squashfs-root/varlib /new_root/var/varlib
+    #         echo ' - SUCCESS '
+    #         echo
+    #     ); cd ..
+    # fi
+
+    for i in bin include lib lib32 share src; do
+        echo -n "[ BLEND ] Setting up /usr/${i} overlay (applying changes)."
+        rm -rf /new_root/.workdir/"$i"
+        mkdir -p /new_root/.upperdir/"$i" /new_root/.workdir/"$i" /new_root/usr/"$i" /new_root/tmp
+        cd /new_root/tmp; overlayfs-tools merge -l /new_root/usr/$i -u /new_root/.upperdir/$i &>/dev/null; chmod 755 ./overlay-tools-*; ./overlay-tools-* &>/dev/null; rm -f ./overlay-tools-*; cd /
+        mkdir -p /new_root/.upperdir/"$i"
+        mount -t overlay overlay -o 'lowerdir=/new_root/usr/'$i',upperdir=/new_root/.upperdir/'$i',workdir=/new_root/.workdir/'$i /new_root/usr/"$i" -o index=off
+        echo " - SUCCESS"
+    done
+
+    echo
+    echo -n "[ BLEND ] Setting up /var/lib overlay (applying changes)."
+    rm -rf /new_root/.workdir/varlib
+    mkdir -p /new_root/.upperdir/varlib /new_root/.workdir/varlib /new_root/var/lib /new_root/tmp
+    cd /new_root/tmp; overlayfs-tools merge -l /new_root/var/lib -u /new_root/.upperdir/varlib &>/dev/null; chmod 755 ./overlay-tools-*; ./overlay-tools-* &>/dev/null; rm -f ./overlay-tools-*; cd /
+    mkdir -p /new_root/.upperdir/varlib
+    mount -t overlay overlay -o 'lowerdir=/new_root/var/lib,upperdir=/new_root/.upperdir/varlib,workdir=/new_root/.workdir/varlib' /new_root/var/lib -o index=off
+    echo ' - SUCCESS'
+    echo
 }

blend.install

@@ -5,12 +5,13 @@ build() {
     add_module overlay
     add_binary bash
     add_binary tar
+    add_binary overlayfs-tools
     add_runscript
 }
 
 help() {
     cat <<HELPEOF
-This provides a support for mounting the blend /usr overlay. No
+This provides a support for mounting the blend overlays. No
 configuration is needed.
 HELPEOF
 }

overlayfs-tools/README.md

@@ -0,0 +1,42 @@
+overlayfs-tools
+========
+
+[OverlayFS](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) is the union filesystem provided by Linux kernel.
+
+This program comes provides three tools:
+- **vacuum** - remove duplicated files in `upperdir` where `copy_up` is done but the file is not actually modified (see the sentence "the `copy_up` may turn out to be unnecessary" in the [Linux documentation](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt)). This may reduce the size of `upperdir` without changing `lowerdir` or `overlay`.
+- **diff** - show the list of actually changed files (the difference between `overlay` and `lowerdir`). A file with its type changed (i.e. from symbolic link to regular file) will shown as deleted then added, rather than modified. Similarly, for a opaque directory in `upperdir`, the corresponding directory in `lowerdir` (if exists) will be shown as entirely deleted, and a new directory with the same name added. File permission/owner changes will be simply shown as modified.
+- **merge** - merge down the changes from `upperdir` to `lowerdir`. Unlike [aubrsync](http://aufs.sourceforge.net/aufs2/brsync/README.txt) for AuFS which bypasses the union filesystem mechanism, overlayfs-utils emulates the OverlayFS logic, which will be far more efficient. After this operation, `upperdir` will be empty and `lowerdir` will be the same as original `overlay`.
+- **deref** - copy changes from `upperdir` to `uppernew` while unfolding redirect directories and metacopy regular files, so that new upperdir is compatible with legacy overlayfs driver.
+
+For safety reasons, vacuum and merge will not actually modify the filesystem, but generate a shell script to do the changes instead.
+
+Prerequisite and Building
+--------
+
+    $ make
+
+Example usage
+--------
+
+    # ./overlay diff -l /lower -u /upper
+
+See `./overlay --help` for more.
+
+Why sudo
+--------
+
+As [Linux documentation](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) said,
+
+> A directory is made opaque by setting the xattr "trusted.overlay.opaque" to "y".
+
+However, only users with `CAP_SYS_ADMIN` can read `trusted.*` extended attributes.
+
+Warnings / limitations
+--------
+
+- Only works for regular files and directories. Do not use it on OverlayFS with device files, socket files, etc..
+- Hard links may be broken (i.e. resulting in duplicated independent files).
+- File owner, group and permission bits will be preserved. File timestamps, attributes and extended attributes might be lost.
+- This program only works for OverlayFS with only one lower layer.
+- It is recommended to have the OverlayFS unmounted before running this program.

overlayfs-tools/logic.c

@@ -0,0 +1,581 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <linux/limits.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/xattr.h>
+#include <fts.h>
+#include <libgen.h>
+#include "logic.h"
+#include "sh.h"
+
+// exactly the same as in linux/fs.h
+#define WHITEOUT_DEV 0
+
+// exact the same as in fs/overlayfs/overlayfs.h
+const char *ovl_opaque_xattr = "trusted.overlay.opaque";
+const char *ovl_redirect_xattr = "trusted.overlay.redirect";
+const char *ovl_metacopy_xattr = "trusted.overlay.metacopy";
+
+#define MIN(X,Y) ((X) < (Y) ? (X) : (Y))
+
+#define TRAILING_SLASH(ftype) (((ftype) == S_IFDIR) ? "/" : "")
+
+static inline mode_t file_type(const struct stat *status) {
+    return status->st_mode & S_IFMT;
+}
+
+const char *ftype_name(mode_t type) {
+    switch (type) {
+    case S_IFDIR:
+        return "directory";
+    case S_IFREG:
+        return "regular file";
+    case S_IFLNK:
+        return "symbolic link";
+    default:
+        return "special file";
+    }
+}
+
+const char *ftype_name_plural(mode_t type) {
+    switch (type) {
+    case S_IFDIR:
+        return "Directories";
+    case S_IFREG:
+        return "Files";
+    case S_IFLNK:
+        return "Symbolic links";
+    default:
+        return "Special files";
+    }
+}
+
+static inline bool is_whiteout(const struct stat *status) {
+    return (file_type(status) == S_IFCHR) && (status->st_rdev == WHITEOUT_DEV);
+}
+
+static inline mode_t permission_bits(const struct stat *status) { // not used yet. I haven't decided how to treat permission bit changes
+    return status->st_mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISVTX);
+}
+
+int is_opaque(const char *path, bool *output) {
+    char val;
+    ssize_t res = getxattr(path, ovl_opaque_xattr, &val, 1);
+    if ((res < 0) && (errno != ENODATA)) {
+        return -1;
+    }
+    *output = (res == 1 && val == 'y');
+    return 0;
+}
+
+int is_redirect(const char *path, bool *output) {
+    ssize_t res = getxattr(path, ovl_redirect_xattr, NULL, 0);
+    if ((res < 0) && (errno != ENODATA)) {
+        fprintf(stderr, "File %s redirect xattr can not be read.\n", path);
+        return -1;
+    }
+    *output = (res > 0);
+    return 0;
+}
+
+int is_metacopy(const char *path, bool *output) {
+    ssize_t res = getxattr(path, ovl_metacopy_xattr, NULL, 0);
+    if ((res < 0) && (errno != ENODATA)) {
+        fprintf(stderr, "File %s metacopy xattr can not be read.\n", path);
+        return -1;
+    }
+    *output = (res >= 0);
+    return 0;
+}
+
+// Treat redirect as opaque dir because it hides the tree in lower_path
+// and we do not support following to redirected lower path
+int is_opaquedir(const char *path, bool *output) {
+    bool opaque, redirect;
+    if (is_opaque(path, &opaque) < 0) { return -1; }
+    if (is_redirect(path, &redirect) < 0) { return -1; }
+    *output = opaque || redirect;
+    return 0;
+}
+
+bool permission_identical(const struct stat *lower_status, const struct stat *upper_status) {
+    return (permission_bits(lower_status) == permission_bits(upper_status)) && (lower_status->st_uid == upper_status->st_uid) && (lower_status->st_gid == upper_status->st_gid);
+}
+
+int read_chunk(int fd, char *buf, int len) {
+    ssize_t ret;
+    ssize_t remain = len;
+    while (remain > 0 && (ret = read(fd, buf, remain)) != 0) {
+        if (ret == -1) {
+            if (errno == EINTR) {
+                continue;
+            }
+            return -1;
+        }
+        remain -= ret;
+        buf += ret;
+    }
+    return len - remain;
+}
+
+int regular_file_identical(const char *lower_path, const struct stat *lower_status, const char *upper_path, const struct stat *upper_status, bool *output) {
+    size_t blksize = (size_t) MIN(lower_status->st_blksize, upper_status->st_blksize);
+    if (lower_status->st_size != upper_status->st_size) { // different sizes
+        *output = false;
+        return 0;
+    }
+    bool metacopy, redirect;
+    if (is_metacopy(upper_path, &metacopy) < 0) { return -1; }
+    if (is_redirect(upper_path, &redirect) < 0) { return -1; }
+    if (metacopy) {
+	    // metacopy means data is indentical, but redirect means it is not identical to lower_path
+	    *output = !redirect;
+	    return 0;
+    }
+    char lower_buffer[blksize];
+    char upper_buffer[blksize];
+    int lower_file = open(lower_path, O_RDONLY);
+    int upper_file = open(upper_path, O_RDONLY);
+    if (lower_file < 0) {
+        fprintf(stderr, "File %s can not be read for content.\n", lower_path);
+        return -1;
+    }
+    if (upper_file < 0) {
+        fprintf(stderr, "File %s can not be read for content.\n", upper_path);
+        return -1;
+    }
+    ssize_t read_lower; ssize_t read_upper;
+    do { // we can assume one will not reach EOF earlier than the other, as the file sizes are checked to be the same earlier
+        read_lower = read_chunk(lower_file, lower_buffer, blksize);
+        read_upper = read_chunk(upper_file, upper_buffer, blksize);
+        if (read_lower < 0) {
+            fprintf(stderr, "Error occured when reading file %s.\n", lower_path);
+            return -1;
+        }
+        if (read_upper < 0) {
+            fprintf(stderr, "Error occured when reading file %s.\n", upper_path);
+            return -1;
+        }
+        if (read_upper != read_lower) { // this should not happen as we've checked the sizes
+            fprintf(stderr, "Unexpected size difference: %s.\n", upper_path);
+            return -1;
+        }
+        if (memcmp(lower_buffer, upper_buffer, read_upper)) { *output = false; return 0; } // the output is by default false, but we still set it for ease of reading
+    } while (read_lower || read_upper);
+    *output = true; // now we can say they are identical
+    if (close(lower_file) || close(upper_file)) { return -1; }
+    return 0;
+}
+
+int symbolic_link_identical(const char *lower_path, const char *upper_path, bool *output) {
+    char lower_buffer[PATH_MAX];
+    char upper_buffer[PATH_MAX];
+    ssize_t lower_len = readlink(lower_path, lower_buffer, PATH_MAX);
+    ssize_t upper_len = readlink(upper_path, upper_buffer, PATH_MAX);
+    if (lower_len < 0 || lower_len == PATH_MAX) {
+        fprintf(stderr, "Symbolic link %s cannot be resolved.\n", lower_path);
+        return -1;
+    }
+    if (upper_len < 0 || upper_len == PATH_MAX) {
+        fprintf(stderr, "Symbolic link %s cannot be resolved.\n", upper_path);
+        return -1;
+    }
+    lower_buffer[lower_len] = '\0';
+    upper_buffer[upper_len] = '\0';
+    *output = (strcmp(lower_buffer, upper_buffer) == 0);
+    return 0;
+}
+
+static int vacuum_d(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool opaque;
+    if (is_opaquedir(upper_path, &opaque) < 0) { return -1; }
+    if (opaque) { // TODO: sometimes removing opaque directory (and combine with lower directory) might be better
+        *fts_instr = FTS_SKIP;
+    }
+    return 0;
+}
+
+static int vacuum_dp(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    if (lower_status == NULL) { return 0; } // lower does not exist
+    if (file_type(lower_status) != S_IFDIR) { return 0; }
+    if (!permission_identical(lower_status, upper_status)) { return 0; }
+    bool opaque;
+    if (is_opaquedir(upper_path, &opaque) < 0) {
+        return -1;
+    }
+    if (opaque) { return 0; }
+    // this directory might be empty if all children are deleted in previous commands. but we simply don't test whether it's that case
+    return command(script_stream, "rmdir --ignore-fail-on-non-empty %U", upper_path);
+}
+
+static int vacuum_f(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    if (lower_status == NULL) { return 0; } // lower does not exist
+    if (file_type(lower_status) != S_IFREG) { return 0; }
+    if (!permission_identical(lower_status, upper_status)) { return 0; }
+    bool identical;
+    if (regular_file_identical(lower_path, lower_status, upper_path, upper_status, &identical) < 0) {
+        return -1;
+    }
+    if (!identical) { return 0; }
+    return command(script_stream, "rm %U", upper_path);
+}
+
+static int vacuum_sl(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    if (lower_status == NULL) { return 0; } // lower does not exist
+    if (file_type(lower_status) != S_IFLNK) { return 0; }
+    if (!permission_identical(lower_status, upper_status)) { return 0; }
+    bool identical;
+    if (symbolic_link_identical(lower_path, upper_path, &identical) < 0) {
+        return -1;
+    }
+    if (!identical) { return 0; }
+    return command(script_stream, "rm %U", upper_path);
+}
+
+void print_only_in(const char *path) {
+    char *dirc = strdup(path);
+    char *basec = strdup(path);
+    char *dname = dirname(dirc);
+    char *bname = basename(basec);
+    printf("Only in %s: %s\n", dname, bname);
+    free(dirc);
+    free(basec);
+}
+
+void print_removed(const char *lower_path, const size_t lower_root_len, mode_t lower_type) {
+    if (brief) {
+	print_only_in(lower_path);
+    } else {
+        printf("Removed: %s%s\n", &lower_path[lower_root_len], TRAILING_SLASH(lower_type));
+    }
+}
+
+void print_added(const char *lower_path, const size_t lower_root_len, const char *upper_path, mode_t upper_type) {
+    if (brief) {
+	print_only_in(upper_path);
+    } else {
+        printf("Added: %s%s\n", &lower_path[lower_root_len], TRAILING_SLASH(upper_type));
+    }
+}
+
+void print_replaced(const char *lower_path, const size_t lower_root_len, mode_t lower_type, const char *upper_path, mode_t upper_type) {
+    if (brief) {
+	printf("File %s is a %s while file %s is a %s\n", lower_path, ftype_name(lower_type), upper_path, ftype_name(upper_type));
+    } else {
+        if (lower_type != S_IFDIR) { // dir removed already printed by list_deleted_files()
+            print_removed(lower_path, lower_root_len, lower_type);
+        }
+        print_added(lower_path, lower_root_len, upper_path, upper_type);
+    }
+}
+
+void print_modified(const char *lower_path, const size_t lower_root_len, mode_t lower_type, const char *upper_path, bool identical) {
+    if (brief) {
+        if (!identical) { // brief format does not print permission difference
+	    printf("%s %s and %s differ\n", ftype_name_plural(lower_type), lower_path, upper_path);
+        }
+    } else {
+        printf("Modified: %s%s\n", &lower_path[lower_root_len], TRAILING_SLASH(lower_type));
+    }
+}
+
+int list_deleted_files(const char *lower_path, size_t lower_root_len, mode_t upper_type) { // This WORKS with files and itself is listed. However, prefixs are WRONG!
+    // brief format needs to print only first level deleted children under opaque dir
+    bool children = (brief && (upper_type == S_IFDIR));
+    if (!verbose && !children) {
+        if (!brief || upper_type == S_IFCHR) { // dir replaced already printed by print_replaced()
+            print_removed(lower_path, lower_root_len, S_IFDIR);
+        }
+        return 0;
+    }
+    FTSENT *cur;
+    char *paths[2] = {(char *) lower_path, NULL };
+    FTS *ftsp = fts_open(paths, FTS_NOCHDIR | FTS_PHYSICAL, NULL);
+    if (ftsp == NULL) { return -1; }
+    int return_val = 0;
+    while (((cur = fts_read(ftsp)) != NULL) && (return_val == 0)) {
+        switch (cur->fts_info) {
+            case FTS_D:
+                // brief format does not need to print deleted grand children under opaque dir
+                if (children && cur->fts_level > 0) {
+                    print_removed(cur->fts_path, lower_root_len, S_IFDIR);
+                    fts_set(ftsp, cur, FTS_SKIP);
+                }
+                break; // do nothing
+            case FTS_DP:
+                // brief format does not need to print deleted dir under opaque dir itself
+                if (!children) {
+                    print_removed(cur->fts_path, lower_root_len, S_IFDIR);
+                }
+                break;
+            case FTS_F:
+                print_removed(cur->fts_path, lower_root_len, S_IFREG);
+                break;
+            case FTS_SL:
+                print_removed(cur->fts_path, lower_root_len, S_IFLNK);
+                break;
+            case FTS_DEFAULT:
+                fprintf(stderr, "File %s is a special file (device or pipe). We cannot handle that.\n", cur->fts_path);
+                return_val = -1;
+                break;
+            default:
+                fprintf(stderr, "Error occured when opening %s.\n", cur->fts_path);
+                return_val = -1;
+        }
+    }
+    if (errno) { return_val = -1; } // if no error happened, fts_read will "sets the external variable errno to 0" according to the documentation
+    return fts_close(ftsp) || return_val;
+}
+
+static int diff_d(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool opaque = false;
+    bool lower_exist = (lower_status != NULL);
+    if (lower_exist) {
+        if (file_type(lower_status) == S_IFDIR) {
+            if (is_opaquedir(upper_path, &opaque) < 0) { return -1; }
+            if (opaque) {
+                if (list_deleted_files(lower_path, lower_root_len, S_IFDIR) < 0) { return -1; }
+            } else {
+                if (!permission_identical(lower_status, upper_status)) {
+                    print_modified(lower_path, lower_root_len, S_IFDIR, upper_path, true);
+                }
+                return 0; // children must be recursed, and directory itself does not need to be printed
+            }
+        } else { // other types of files
+            print_replaced(lower_path, lower_root_len, file_type(lower_status), upper_path, S_IFDIR);
+        }
+    }
+    if (!(verbose || (brief && opaque))) { // brief format needs to print children of opaque dir
+        *fts_instr = FTS_SKIP;
+    }
+    if (!lower_exist || (!brief && opaque)) { // brief format does not need to print opaque dir itself
+        print_added(lower_path, lower_root_len, upper_path, S_IFDIR);
+    }
+    return 0;
+}
+
+static int diff_f(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool identical;
+    if (lower_status != NULL) {
+        switch (file_type(lower_status)) {
+            case S_IFREG:
+                if (regular_file_identical(lower_path, lower_status, upper_path, upper_status, &identical) < 0) {
+                    return -1;
+                }
+                if (!(identical && permission_identical(lower_status, upper_status))) {
+                    print_modified(lower_path, lower_root_len, S_IFREG, upper_path, identical);
+                }
+                return 0;
+            case S_IFDIR:
+                if (list_deleted_files(lower_path, lower_root_len, S_IFREG) < 0) { return -1; }
+                /* fallthrough */
+            case S_IFLNK:
+                print_replaced(lower_path, lower_root_len, file_type(lower_status), upper_path, S_IFREG);
+                return 0;
+            default:
+                fprintf(stderr, "File %s is a special file (device or pipe). We cannot handle that.\n", lower_path);
+                return -1;
+        }
+    }
+    print_added(lower_path, lower_root_len, upper_path, S_IFREG);
+    return 0;
+}
+
+static int diff_sl(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool identical;
+    if (lower_status != NULL) {
+        switch (file_type(lower_status)) {
+            case S_IFDIR:
+                if (list_deleted_files(lower_path, lower_root_len, S_IFLNK) < 0) { return -1; }
+                /* fallthrough */
+            case S_IFREG:
+                print_replaced(lower_path, lower_root_len, file_type(lower_status), upper_path, S_IFLNK);
+                return 0;
+            case S_IFLNK:
+                if (symbolic_link_identical(lower_path, upper_path, &identical) < 0) {
+                    return -1;
+                }
+                if (!(identical && permission_identical(lower_status, upper_status))) {
+                    print_modified(lower_path, lower_root_len, S_IFLNK, upper_path, identical);
+                }
+                return 0;
+            default:
+                fprintf(stderr, "File %s is a special file (device or pipe). We cannot handle that.\n", lower_path);
+                return -1;
+        }
+    }
+    print_added(lower_path, lower_root_len, upper_path, S_IFLNK);
+    return 0;
+}
+
+static int diff_whiteout(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    if (lower_status != NULL) {
+        if (file_type(lower_status) == S_IFDIR) {
+            if (list_deleted_files(lower_path, lower_root_len, S_IFCHR) < 0) { return -1; }
+        } else {
+            print_removed(lower_path, lower_root_len, file_type(lower_status));
+        }
+    } // else: whiteouting a nonexistent file? must be an error. but we ignore that :)
+    return 0;
+}
+
+static int merge_d(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool redirect;
+    if (is_redirect(upper_path, &redirect) < 0) { return -1; }
+    // merging redirects is not supported, we must abort merge so redirected lower (under whiteout) won't be deleted
+    // upper_path may be hiding the directory in lower_path, but there may be another redirect upper pointing at it
+    if (redirect) {
+        fprintf(stderr, "Found redirect on %s. Merging redirect is not supported - Abort.\n", upper_path);
+        return -1;
+    }
+    if (lower_status != NULL) {
+        if (file_type(lower_status) == S_IFDIR) {
+            bool opaque = false;
+            if (is_opaquedir(upper_path, &opaque) < 0) { return -1; }
+            if (opaque) {
+                if (command(script_stream, "rm -r %L", lower_path) < 0) { return -1; };
+            } else {
+                if (!permission_identical(lower_status, upper_status)) {
+                    command(script_stream, "chmod --reference %U %L", upper_path, lower_path);
+                }
+                return 0; // children must be recursed, and directory itself does not need to be printed
+            }
+        } else {
+            command(script_stream, "rm %L", lower_path);
+        }
+    }
+    *fts_instr = FTS_SKIP;
+    return command(script_stream, "mv -T %U %L", upper_path, lower_path);
+}
+
+static int merge_dp(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    if (lower_status != NULL) {
+        if (file_type(lower_status) == S_IFDIR) {
+            bool opaque = false;
+            if (is_opaquedir(upper_path, &opaque) < 0) { return -1; }
+            if (!opaque) { // delete the directory: it should be empty already
+                return command(script_stream, "rmdir %U", upper_path);
+            }
+        }
+    }
+    return 0;
+}
+
+static int merge_f(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool metacopy, redirect;
+    if (is_metacopy(upper_path, &metacopy) < 0) { return -1; }
+    if (is_redirect(upper_path, &redirect) < 0) { return -1; }
+    // merging metacopy is not supported, we must abort merge so lower data won't be deleted
+    if (metacopy || redirect) {
+        fprintf(stderr, "Found metacopy/redirect on %s. Merging metacopy/redirect is not supported - Abort.\n", upper_path);
+        return -1;
+    }
+    return command(script_stream, "rm -rf %L", lower_path) || command(script_stream, "mv -T %U %L", upper_path, lower_path);
+}
+
+static int merge_sl(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    return command(script_stream, "rm -rf %L", lower_path) || command(script_stream, "mv -T %U %L", upper_path, lower_path);
+}
+
+static int merge_whiteout(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    return command(script_stream, "rm -r %L", lower_path) || command(script_stream, "rm %U", upper_path);
+}
+
+typedef int (*TRAVERSE_CALLBACK)(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr);
+
+int traverse(const char *lower_root, const char *upper_root, FILE* script_stream, TRAVERSE_CALLBACK callback_d, TRAVERSE_CALLBACK callback_dp, TRAVERSE_CALLBACK callback_f, TRAVERSE_CALLBACK callback_sl, TRAVERSE_CALLBACK callback_whiteout) { // returns 0 on success
+    FTSENT *cur;
+    char *paths[2] = {(char *) upper_root, NULL };
+    char lower_path[PATH_MAX];
+    strcpy(lower_path, lower_root);
+    size_t upper_root_len = strlen(upper_root);
+    size_t lower_root_len = strlen(lower_root);
+    FTS *ftsp = fts_open(paths, FTS_NOCHDIR | FTS_PHYSICAL, NULL);
+    if (ftsp == NULL) { return -1; }
+    int return_val = 0;
+    while ((return_val == 0) && ((cur = fts_read(ftsp)) != NULL)) {
+        TRAVERSE_CALLBACK callback = NULL;
+        switch (cur->fts_info) {
+            case FTS_D:
+                callback = callback_d;
+                break;
+            case FTS_DP:
+                callback = callback_dp;
+                break;
+            case FTS_F:
+                callback = callback_f;
+                break;
+            case FTS_SL:
+                callback = callback_sl;
+                break;
+            case FTS_DEFAULT:
+                if (is_whiteout(cur->fts_statp)) {
+                    callback = callback_whiteout;
+                } else {
+                    return_val = -1;
+                    fprintf(stderr, "File %s is a special file (device or pipe). We cannot handle that.\n", cur->fts_path);
+                }
+                break;
+            default:
+                return_val = -1;
+                fprintf(stderr, "Error occured when opening %s.\n", cur->fts_path);
+        }
+        if (callback != NULL) {
+            int fts_instr = 0;
+            struct stat lower_status;
+            bool lower_exist = true;
+            strcpy(&lower_path[lower_root_len], &(cur->fts_path[upper_root_len]));
+            if (lstat(lower_path, &lower_status) != 0) {
+                if (errno == ENOENT || errno == ENOTDIR) { // the corresponding lower file (or its ancestor) does not exist at all
+                    lower_exist = false;
+                } else { // stat failed for some unknown reason
+                    fprintf(stderr, "Failed to stat %s.\n", lower_path);
+                    return_val = -1;
+                    break; // do not call callback in this case
+                }
+            }
+            return_val = callback(lower_path, cur->fts_path, lower_root_len, lower_exist ? &lower_status : NULL, cur->fts_statp, script_stream, &fts_instr); // return_val must previously be 0
+            if (fts_instr) {
+                fts_set(ftsp, cur, fts_instr);
+            }
+        }
+    }
+    if (errno) { return_val = -1; } // if no error happened, fts_read will "sets the external variable errno to 0" according to the documentation
+    return fts_close(ftsp) || return_val;
+}
+
+static int deref_d(const char *mnt_path, const char* upper_path, const size_t mnt_root_len, const struct stat *mnt_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool redirect;
+    if (is_redirect(upper_path, &redirect) < 0) { return -1; }
+    if (!redirect) { return 0; }
+    *fts_instr = FTS_SKIP;
+    return command(script_stream, "rm -rf %U", upper_path) || command(script_stream, "cp -a %M %U", mnt_path, upper_path);
+}
+
+static int deref_f(const char *mnt_path, const char* upper_path, const size_t mnt_root_len, const struct stat *mnt_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool metacopy;
+    if (is_metacopy(upper_path, &metacopy) < 0) { return -1; }
+    if (!metacopy) { return 0; }
+    return command(script_stream, "rm -r %U", upper_path) || command(script_stream, "cp -a %M %U", mnt_path, upper_path);
+}
+
+int vacuum(const char* lowerdir, const char* upperdir, FILE* script_stream) {
+    return traverse(lowerdir, upperdir, script_stream, vacuum_d, vacuum_dp, vacuum_f, vacuum_sl, NULL);
+}
+
+int diff(const char* lowerdir, const char* upperdir) {
+    return traverse(lowerdir, upperdir, NULL, diff_d, NULL, diff_f, diff_sl, diff_whiteout);
+}
+
+int merge(const char* lowerdir, const char* upperdir, FILE* script_stream) {
+    return traverse(lowerdir, upperdir, script_stream, merge_d, merge_dp, merge_f, merge_sl, merge_whiteout);
+}
+
+int deref(const char* mountdir, const char* upperdir, FILE* script_stream) {
+    return traverse(mountdir, upperdir, script_stream, deref_d, NULL, deref_f, NULL, NULL);
+}

overlayfs-tools/logic.h

@@ -0,0 +1,37 @@
+/*
+ * logic.h / logic.c
+ *
+ * the logic for the three feature functions
+ */
+
+#ifndef OVERLAYFS_TOOLS_LOGIC_H
+#define OVERLAYFS_TOOLS_LOGIC_H
+
+#include <stdbool.h>
+
+extern bool verbose;
+extern bool brief;
+
+/*
+ * feature function. will take very long time to complete. returns 0 on success
+ */
+int vacuum(const char* lowerdir, const char* upperdir, FILE* script_stream);
+
+/*
+ * feature function. will take very long time to complete. returns 0 on success
+ */
+int diff(const char* lowerdir, const char* upperdir);
+
+/*
+ * feature function. will take very long time to complete. returns 0 on success
+ */
+int merge(const char* lowerdir, const char* upperdir, FILE* script_stream);
+
+/*
+ * Unfold metacopy and redirect upper.
+ *
+ * mountdir is required and lowerdir is irrelevant.
+ */
+int deref(const char* mountdir, const char* upperdir, FILE* script_stream);
+
+#endif //OVERLAYFS_TOOLS_LOGIC_H

overlayfs-tools/main.c

@@ -0,0 +1,266 @@
+/*
+ * main.c
+ *
+ * the command line user interface
+ */
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+#include <linux/limits.h>
+#include <stdbool.h>
+#include <sys/stat.h>
+#include <sys/xattr.h>
+#include <errno.h>
+#ifndef _SYS_STAT_H
+  #include <linux/stat.h>
+#endif
+#include "logic.h"
+#include "sh.h"
+
+#define STRING_BUFFER_SIZE PATH_MAX * 2
+
+// currently, brief and verbose are mutually exclusive
+bool verbose;
+bool brief;
+bool yes;
+
+void print_help(const char *program) {
+    printf("Usage: %s command options\n", program);
+    puts("");
+    puts("Commands:");
+    puts("  vacuum - remove duplicated files in upperdir where copy_up is done but the file is not actually modified");
+    puts("  diff   - show the list of actually changed files");
+    puts("  merge  - merge all changes from upperdir to lowerdir, and clear upperdir");
+    puts("  deref  - copy changes from upperdir to a new upperdir unfolding redirect and metacopy");
+    puts("");
+    puts("Options:");
+    puts("  -l, --lowerdir=LOWERDIR    the lowerdir of OverlayFS (required)");
+    puts("  -u, --upperdir=UPPERDIR    the upperdir of OverlayFS (required)");
+    puts("  -m, --mountdir=MOUNTDIR    the mountdir of OverlayFS (optional)");
+    puts("  -L, --lowernew=LOWERNEW    the lowerdir of new OverlayFS (optional)");
+    puts("  -U, --uppernew=UPPERNEW    the upperdir of new OverlayFS (optional)");
+    puts("  -y  --yes                  don't prompt if OverlayFS is still mounted (optional)");
+    puts("  -v, --verbose              with diff action only: when a directory only exists in one version, still list every file of the directory");
+    puts("  -b, --brief                with diff action only: conform to output of diff --brief --recursive --no-dereference");
+    puts("  -h, --help                 show this help text");
+    puts("");
+    puts("See https://github.com/kmxz/overlayfs-tools/ for warnings and more information.");
+}
+
+bool starts_with(const char *haystack, const char* needle) {
+    return strncmp(needle, haystack, strlen(needle)) == 0;
+}
+
+bool is_mounted(const char *lower, const char *upper) {
+    FILE *f = fopen("/proc/mounts", "r");
+    if (!f) {
+        fprintf(stderr, "Cannot read /proc/mounts to test whether OverlayFS is mounted.\n");
+        return true;
+    }
+    char buf[STRING_BUFFER_SIZE];
+    while (fgets(buf, STRING_BUFFER_SIZE, f)) {
+        if (!starts_with(buf, "overlay")) {
+            continue;
+        }
+        if (strlen(buf) == STRING_BUFFER_SIZE) {
+            fprintf(stderr, "OverlayFS line in /proc/mounts is too long.\n");
+            return true;
+        }
+        char *m_lower = strstr(buf, "lowerdir=");
+        char *m_upper = strstr(buf, "upperdir=");
+        if (m_lower == NULL || m_upper == NULL) {
+            fprintf(stderr, "Cannot extract information from OverlayFS line in /proc/mounts.\n");
+            return true;
+        }
+        m_lower = &(m_lower[strlen("lowerdir=")]);
+        m_upper = &(m_upper[strlen("upperdir=")]);
+        if (!(strncmp(lower, m_lower, strlen(lower)) && strncmp(upper, m_upper, strlen(upper)))) {
+            printf("The OverlayFS involved is still mounted.\n");
+            return true;
+        }
+    }
+    return false;
+}
+
+bool check_mounted(const char *lower, const char *upper) {
+    if (is_mounted(lower, upper) && !yes) {
+        printf("It is strongly recommended to unmount OverlayFS first. Still continue (not recommended)?: \n");
+        int r = getchar();
+        if (r != 'Y' && r != 'y') {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool directory_exists(const char *path) {
+    struct stat sb;
+    if (lstat(path, &sb) != 0) { return false; }
+    return (sb.st_mode & S_IFMT) == S_IFDIR;
+}
+
+bool directory_create(const char *name, const char *path) {
+    if (mkdir(path, 0755) == 0 || errno == EEXIST) { return true; }
+    fprintf(stderr, "%s directory '%s' does not exist and cannot be created.\n", name, path);
+    exit(EXIT_FAILURE);
+}
+
+bool real_check_xattr_trusted(const char *tmp_path, int tmp_file) {
+    int ret = fsetxattr(tmp_file, "trusted.overlay.test", "naive", 5, 0);
+    close(tmp_file);
+    if (ret) { return false; }
+    char verify_buffer[10];
+    if (getxattr(tmp_path, "trusted.overlay.test", verify_buffer, 10) != 5) { return false; }
+    return !strncmp(verify_buffer, "naive", 5);
+}
+
+bool check_xattr_trusted(const char *upper) {
+    char tmp_path[PATH_MAX];
+    strcpy(tmp_path, upper);
+    strcat(tmp_path, "/.xattr_test_XXXXXX.tmp");
+    int tmp_file = mkstemps(tmp_path, 4);
+    if (tmp_file < 0) { return false; }
+    bool ret = real_check_xattr_trusted(tmp_path, tmp_file);
+    unlink(tmp_path);
+    return ret;
+}
+
+int main(int argc, char *argv[]) {
+
+    char *lower = NULL;
+    char *upper = NULL;
+    char *dir, *mnt = NULL;
+
+    static struct option long_options[] = {
+        { "lowerdir", required_argument, 0, 'l' },
+        { "upperdir", required_argument, 0, 'u' },
+        { "mountdir", required_argument, 0, 'm' },
+        { "lowernew", required_argument, 0, 'L' },
+        { "uppernew", required_argument, 0, 'U' },
+        { "yes",      no_argument      , 0, 'y' },
+        { "help",     no_argument      , 0, 'h' },
+        { "verbose",  no_argument      , 0, 'v' },
+        { "brief",    no_argument      , 0, 'b' },
+        { 0,          0,                 0,  0  }
+    };
+
+    int opt = 0;
+    int long_index = 0;
+    while ((opt = getopt_long_only(argc, argv, "l:u:m:L:U:yhvb", long_options, &long_index)) != -1) {
+        switch (opt) {
+            case 'l':
+                lower = realpath(optarg, NULL);
+                if (lower) { vars[LOWERDIR] = lower; }
+                break;
+            case 'u':
+                upper = realpath(optarg, NULL);
+                if (upper) { vars[UPPERDIR] = upper; }
+                break;
+            case 'm':
+                mnt = realpath(optarg, NULL);
+                if (mnt) { vars[MOUNTDIR] = mnt; }
+                break;
+            case 'L':
+                directory_create("New lowerdir", optarg);
+                dir = realpath(optarg, NULL);
+                if (dir) { vars[LOWERNEW] = dir; }
+                break;
+            case 'U':
+                directory_create("New upperdir", optarg);
+                dir = realpath(optarg, NULL);
+                if (dir) { vars[UPPERNEW] = dir; }
+                break;
+            case 'y':
+                yes = true;
+                break;
+            case 'h':
+                print_help(argv[0]);
+                return EXIT_SUCCESS;
+            case 'v':
+                verbose = true;
+                brief = false;
+                break;
+            case 'b':
+                verbose = false;
+                brief = true;
+                break;
+            default:
+                fprintf(stderr, "Option %c is not supported.\n", opt);
+                goto see_help;
+        }
+    }
+
+    if (!lower) {
+        fprintf(stderr, "Lower directory not specified.\n");
+        goto see_help;
+    }
+    if (!directory_exists(lower)) {
+        fprintf(stderr, "Lower directory cannot be opened.\n");
+        goto see_help;
+    }
+    if (!upper) {
+        fprintf(stderr, "Upper directory not specified.\n");
+        goto see_help;
+    }
+    if (!directory_exists(upper)) {
+        fprintf(stderr, "Upper directory cannot be opened.\n");
+        goto see_help;
+    }
+    if (!check_xattr_trusted(upper)) {
+        fprintf(stderr, "The program cannot write trusted.* xattr. Try run again as root.\n");
+        return EXIT_FAILURE;
+    }
+    // Relax check for mounted overlay if we are not going to modify lowerdir/upperdir
+    if ((!vars[LOWERNEW] || !vars[UPPERNEW]) && check_mounted(lower, upper)) {
+        return EXIT_FAILURE;
+    }
+
+    if (optind == argc - 1) {
+        int out;
+        char filename_template[] = "overlay-tools-XXXXXX.sh";
+        FILE *script = NULL;
+        if (strcmp(argv[optind], "diff") == 0) {
+            out = diff(lower, upper);
+        } else if (strcmp(argv[optind], "vacuum") == 0) {
+            script = create_shell_script(filename_template);
+            if (script == NULL) { fprintf(stderr, "Script file cannot be created.\n"); return EXIT_FAILURE; }
+            out = vacuum(lower, upper, script);
+        } else if (strcmp(argv[optind], "merge") == 0) {
+            script = create_shell_script(filename_template);
+            if (script == NULL) { fprintf(stderr, "Script file cannot be created.\n"); return EXIT_FAILURE; }
+            out = merge(lower, upper, script);
+        } else if (strcmp(argv[optind], "deref") == 0) {
+            if (!mnt || !vars[UPPERNEW]) { fprintf(stderr, "'deref' command requires --uppernew and --mountdir.\n"); return EXIT_FAILURE; }
+            if (!directory_exists(mnt)) {
+                fprintf(stderr, "OverlayFS mount directory cannot be opened.\n");
+                goto see_help;
+            }
+            script = create_shell_script(filename_template);
+            if (script == NULL) { fprintf(stderr, "Script file cannot be created.\n"); return EXIT_FAILURE; }
+            out = deref(mnt, upper, script);
+        } else {
+            fprintf(stderr, "Action not supported.\n");
+            goto see_help;
+        }
+        if (script != NULL) {
+            printf("The script %s is created. Run the script to do the actual work please. Remember to run it when the OverlayFS is not mounted.\n", filename_template);
+            fclose(script);
+        }
+        if (out) {
+            fprintf(stderr, "Action aborted due to fatal error.\n");
+            return EXIT_FAILURE;
+        }
+        return EXIT_SUCCESS;
+    }
+
+    fprintf(stderr, "Please specify one action.\n");
+
+see_help:
+    fprintf(stderr, "Try '%s --help' for more information.\n", argv[0]);
+    return EXIT_FAILURE;
+
+}

overlayfs-tools/makefile

@@ -0,0 +1,23 @@
+CC = gcc
+CFLAGS = -Wall -std=c99
+LDFLAGS = -lm
+ifneq (,$(wildcard /etc/alpine-release))
+  LDFLAGS += -lfts
+endif
+
+all: overlayfs-tools
+
+overlayfs-tools: main.o logic.o sh.o
+	$(CC) main.o logic.o sh.o -o overlayfs-tools $(LDFLAGS)
+
+main.o: main.c logic.h
+	$(CC) $(CFLAGS) -c main.c
+
+logic.o: logic.c logic.h sh.h
+	$(CC) $(CFLAGS) -c logic.c
+
+sh.o: sh.c sh.h
+	$(CC) $(CFLAGS) -c sh.c
+
+clean:
+	$(RM) main.o logic.o sh.o overlayfs-tools

overlayfs-tools/sh.c

@@ -0,0 +1,98 @@
+#define _GNU_SOURCE
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <time.h>
+#include "sh.h"
+
+char * vars[NUM_VARS];
+const char * var_names[NUM_VARS] = {
+    "LOWERDIR",
+    "UPPERDIR",
+    "MOUNTDIR",
+    "LOWERNEW",
+    "UPPERNEW",
+};
+
+int quote(const char *filename, FILE *output);
+
+FILE* create_shell_script(char *tmp_path_buffer) {
+    int tmp_file = mkstemps(tmp_path_buffer, 3); // the 3 is for suffix length (".sh")
+    if (tmp_file < 0) { return NULL; }
+    fchmod(tmp_file, S_IRWXU); // chmod to 0700
+    FILE* f = fdopen(tmp_file, "w");
+    if (f == NULL) { return NULL; }
+    fprintf(f, "#!/usr/bin/env bash\n");
+    fprintf(f, "set -x\n");
+    time_t rawtime;
+    time (&rawtime);
+    fprintf(f, "# This shell script is generated by overlayfs-tools on %s\n", ctime (&rawtime));
+    for (int i=0; i < NUM_VARS; i++) {
+        if (vars[i]) {
+            fprintf(f, "%s=", var_names[i]);
+            if (quote(vars[i], f) < 0) { return NULL; }
+            if (fputc('\n', f) == EOF) { return NULL; }
+	}
+    }
+    // Non-empty *NEW vars make a backup copy and override *DIR vars
+    if (vars[LOWERNEW]) {
+        fprintf(f, "rm -rf \"$LOWERNEW\"\n");
+        fprintf(f, "cp -a \"$LOWERDIR\" \"$LOWERNEW\"\n");
+        fprintf(f, "LOWERDIR=");
+        if (quote(vars[LOWERNEW], f) < 0) { return NULL; }
+        if (fputc('\n', f) == EOF) { return NULL; }
+    }
+    if (vars[UPPERNEW]) {
+        fprintf(f, "rm -rf \"$UPPERNEW\"\n");
+        fprintf(f, "cp -a \"$UPPERDIR\" \"$UPPERNEW\"\n");
+        fprintf(f, "UPPERDIR=");
+        if (quote(vars[UPPERNEW], f) < 0) { return NULL; }
+        if (fputc('\n', f) == EOF) { return NULL; }
+    }
+    return f;
+}
+
+int quote(const char *filename, FILE *output) {
+    if (fputc('\'', output) == EOF) { return -1; }
+    for (const char *s = filename; *s != '\0'; s++) {
+        if (*s == '\'') {
+            if (fprintf(output, "'\"'\"'") < 0) { return -1; }
+        } else {
+            if (fputc(*s, output) == EOF) { return -1; }
+        }
+    }
+    if (fputc('\'', output) == EOF) { return -1; }
+    return 0;
+}
+
+int substitue(char what, const char *filename, FILE *output) {
+    int i;
+    for (i=0; i < NUM_VARS; i++)
+        if (vars[i] && var_names[i][0] == what)
+	    break;
+    if (i == NUM_VARS) { return -1; }
+    // filename prefix must match the var value
+    int prefix = strlen(vars[i]);
+    if (strncmp(filename, vars[i], prefix)) { return -1; }
+    filename += prefix;
+    fprintf(output, "\"$%s\"", var_names[i]);
+    return quote(filename, output);
+}
+
+int command(FILE *output, const char *command_format, ...) {
+    va_list arg;
+    va_start(arg, command_format);
+    for (size_t i = 0; command_format[i] != '\0'; i++) {
+        if (command_format[i] == '%') {
+            const char *s = va_arg(arg, char *);
+            if (substitue(command_format[++i], s, output) < 0) { return -1; }
+        } else {
+            if (fputc(command_format[i], output) == EOF) { return -1; }
+        }
+    }
+    va_end(arg);
+    if (fputc('\n', output) == EOF) { return -1; }
+    return 0;
+}

overlayfs-tools/sh.h

@@ -0,0 +1,20 @@
+#ifndef OVERLAYFS_TOOLS_SH_H
+#define OVERLAYFS_TOOLS_SH_H
+
+enum {
+    LOWERDIR,
+    UPPERDIR,
+    MOUNTDIR,
+    LOWERNEW,
+    UPPERNEW,
+    NUM_VARS
+};
+
+extern const char *var_names[NUM_VARS];
+extern char *vars[NUM_VARS];
+
+FILE* create_shell_script(char *tmp_path_buffer);
+
+int command(FILE *output, const char *command_format, ...);
+
+#endif //OVERLAYFS_TOOLS_SH_H