borg create support

README.md

@@ -105,33 +105,49 @@ or
 
     $ ./find.sh ~/projects/ | ./find2flat.py - | ./unflatten.py - | ncdu -f -
 
+### Borg export
+
+`borg.sh` allows you to turn the `borg create --dry-run --list` output into an
+ncdu-compatible JSON format. This allows you to preview the directories which
+borg would back up using ncdu's interactive, discoverable front-end. Unlike the
+`find` export however, borg and the python script must be ran as the same user
+on the same machine for this to work.
+
+    $ ./borg.sh /path/to/borg/repo::{now} /path/to/back/up/ > borg-flat-export.json
+    $ ./unflatten.py borg-flat-export.json > borg-export.json
+    $ ncdu -f borg-export.json
+
+or
+
+    $ ./borg.sh /path/to/borg/repo::{now} /path/to/back/up/ | ./unflatten.py - | ncdu -f -
+
 ## Graph of tools
 
 
                          .------------.
-         .---------------| filesystem |
-         |               '------------'
-         |                      |
-         |                      | ncdu -o / ncdu-export
-         |                      v
-         |                  .------.         .---------.
-         | find.sh          | ncdu | ncdu -f |  ncdu   |
-         |                  | JSON |-------->| preview |
-         |                  '------'         '---------'
-         |                    |  ^
-         |         flatten.py |  | unflatten.py
-         v                    v  |
+      .----.-------------| filesystem |
+      |    |             '------------'
+      |    |                    |
+      |    |                    | ncdu -o / ncdu-export
+      |    |                    v
+      |    |                .------.         .---------.
+      |    | find.sh        | ncdu | ncdu -f |  ncdu   |
+      |    |                | JSON |-------->| preview |
+      |    |                '------'         '---------'
+      |    |                  |  ^
+      |    |       flatten.py |  | unflatten.py
+      |    v                  v  |
     .--------.              .------.
     |  find  | find2flat.py | flat |<---. jq filtering
     | output |------------->| JSON |----'
     '--------'              '------'
-                                |
-                                | jq
-                                v
-                          .-----------.        .---------.
-                          |    tar    | tar -T |   tar   |
-                          | file list |------->| archive |
-                          '-----------'        '---------'
+      |                       ^  |
+      |                       |  | jq
+      v                       |  v
+    .---------.               |  .-----------.        .---------.
+    | borg.sh |---------------'  |    tar    | tar -T |   tar   |
+    | output  |                  | file list |------->| archive |
+    '---------'                  '-----------'        '---------'
 
 
 

borg.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Runs `borg create` with the required flags and pipe stderr to borg2flat
+exec borg create --dry-run --list "$@" 2>&1 | $(dirname "$0")/borg2flat.py /dev/stdin

borg2flat.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import json
+import stat
+import sys
+
+def getFileInfo(path, root, is_excluded = False):
+    stats = os.lstat(path)
+    dirname = os.path.dirname(path)
+
+    return {
+        "name" : os.path.basename(path),
+        "asize" : stats.st_size,
+        "dsize" : stats.st_blocks * 512,
+        "ino" : stats.st_ino,
+        "mtime" : int(stats.st_mtime),
+        "type" : "dir" if os.path.isdir(path) and not is_excluded else "file",
+        "dirs" : f"{root}/{dirname}" if dirname else root,
+    } | ({
+        "excluded" : "pattern",
+    } if is_excluded else { })
+
+p = argparse.ArgumentParser()
+p.add_argument("--root", default="<root>", help="root directory name")
+p.add_argument("file", type=argparse.FileType("r"), help="borg export filename")
+args = p.parse_args()
+
+args.root = args.root.rstrip("/")
+
+for line in args.file:
+    # FIXME For now we map errors to exclusions because flatten does not
+    # support adding "read_error": true
+    exclusion_letters = [ "x", "E" ]
+    inclusion_letters = [ "-" ]
+    # Borg prints errors to the same output as the dry-run output i.e.
+    # inaccessible: dir_open: [Errno 13] Permission denied: 'inaccessible'
+    # The best determinator for this is that they typically don't have a space
+    # as the second character and don't start with a -
+    # FIXME ask upstream for a better solution
+    if line[1] != " " or line[0] not in exclusion_letters + inclusion_letters:
+        print(f"ERROR: Not a legal borg dry-run line: \"{line.rstrip('\n')}\"", file=sys.stderr)
+        continue
+
+    filename = line[2:].rstrip("\n")
+    excluded = line[0] in exclusion_letters
+    print(json.dumps(getFileInfo(filename, args.root, excluded), ensure_ascii = False))