From 2159b800f19884a5d44c5190b11992c2ecdb55d8 Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 11:33:35 +0200 Subject: [PATCH 01/12] borg2flat: init --- borg2flat.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 borg2flat.py diff --git a/borg2flat.py b/borg2flat.py new file mode 100755 index 0000000..ef6ce93 --- /dev/null +++ b/borg2flat.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +import os +import argparse +import json +import stat + +def getFileInfo(path): + stats = os.lstat(path) + dirname = os.path.dirname(path) + + return { + "name" : os.path.basename(path), + "asize" : stats.st_size, + "dsize" : stats.st_blocks * 512, + "ino" : stats.st_ino, + "mtime" : int(stats.st_mtime), + "type" : "dir" if os.path.isdir(path) else "file", + "dirs" : f"/{dirname}" if dirname else "", + } + +parser = argparse.ArgumentParser() +parser.add_argument('filename') +args = parser.parse_args() + +print(json.dumps(getFileInfo(args.filename))) From 2181b7c76b0749ba8cee08a037ffcd1674761874 Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 11:42:53 +0200 Subject: [PATCH 02/12] borg2flat: add root argument --- borg2flat.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/borg2flat.py b/borg2flat.py index ef6ce93..d714008 100755 --- a/borg2flat.py +++ b/borg2flat.py @@ -5,7 +5,7 @@ import json import stat -def getFileInfo(path): +def getFileInfo(path, root): stats = os.lstat(path) dirname = os.path.dirname(path) @@ -16,11 +16,14 @@ def getFileInfo(path): "ino" : stats.st_ino, "mtime" : int(stats.st_mtime), "type" : "dir" if os.path.isdir(path) else "file", - "dirs" : f"/{dirname}" if dirname else "", + "dirs" : f"{root}/{dirname}" if dirname else root, } -parser = argparse.ArgumentParser() -parser.add_argument('filename') -args = parser.parse_args() +p = argparse.ArgumentParser() +p.add_argument("--root", default="", help="root directory name") +p.add_argument("filename", help="find export filename") +args = p.parse_args() -print(json.dumps(getFileInfo(args.filename))) +args.root = args.root.rstrip("/") + +print(json.dumps(getFileInfo(args.filename, args.root))) From 2c0e8d8e46d4b22e5a760499e91d8d86447f6fe9 Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 11:49:55 +0200 Subject: [PATCH 03/12] borg2flat: process a file with lines --- borg2flat.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/borg2flat.py b/borg2flat.py index d714008..f2e9ac3 100755 --- a/borg2flat.py +++ b/borg2flat.py @@ -21,9 +21,10 @@ def getFileInfo(path, root): p = argparse.ArgumentParser() p.add_argument("--root", default="", help="root directory name") -p.add_argument("filename", help="find export filename") +p.add_argument("file", type=argparse.FileType("r"), help="find export filename") args = p.parse_args() args.root = args.root.rstrip("/") -print(json.dumps(getFileInfo(args.filename, args.root))) +for line in args.file: + print(json.dumps(getFileInfo(line[:-1], args.root))) From afe8e7092f178db9b389301ac657b7e9d3956e21 Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 11:52:17 +0200 Subject: [PATCH 04/12] borg2flat: use rstrip("\n") --- borg2flat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg2flat.py b/borg2flat.py index f2e9ac3..ea8bbc9 100755 --- a/borg2flat.py +++ b/borg2flat.py @@ -27,4 +27,5 @@ def getFileInfo(path, root): args.root = args.root.rstrip("/") for line in args.file: - print(json.dumps(getFileInfo(line[:-1], args.root))) + filename = line.rstrip("\n") + print(json.dumps(getFileInfo(filename, args.root))) From a233f96d97f8aa4cd30428eb3a57e29d016cc783 Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 12:04:38 +0200 Subject: [PATCH 05/12] borg2flat: add is_excluded parameter --- borg2flat.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/borg2flat.py b/borg2flat.py index ea8bbc9..cd32e36 100755 --- a/borg2flat.py +++ b/borg2flat.py @@ -5,7 +5,7 @@ import json import stat -def getFileInfo(path, root): +def getFileInfo(path, root, is_excluded = False): stats = os.lstat(path) dirname = os.path.dirname(path) @@ -17,7 +17,9 @@ def getFileInfo(path, root): "mtime" : int(stats.st_mtime), "type" : "dir" if os.path.isdir(path) else "file", "dirs" : f"{root}/{dirname}" if dirname else root, - } + } | ({ + "excluded" : "pattern", + } if is_excluded else { }) p = argparse.ArgumentParser() p.add_argument("--root", default="", help="root directory name") From 0e8047d0425fdbf0035a2ad849be1319780e342f Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 12:55:38 +0200 Subject: [PATCH 06/12] borg2flat: mark excluded dirs as files ncdu does this too --- borg2flat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg2flat.py b/borg2flat.py index cd32e36..6d4d260 100755 --- a/borg2flat.py +++ b/borg2flat.py @@ -15,7 +15,7 @@ def getFileInfo(path, root, is_excluded = False): "dsize" : stats.st_blocks * 512, "ino" : stats.st_ino, "mtime" : int(stats.st_mtime), - "type" : "dir" if os.path.isdir(path) else "file", + "type" : "dir" if os.path.isdir(path) and not is_excluded else "file", "dirs" : f"{root}/{dirname}" if dirname else root, } | ({ "excluded" : "pattern", From e6656345682143c23c405a44ff1c528836b679cb Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 13:00:54 +0200 Subject: [PATCH 07/12] borg2flat: process borg dry-run input --- borg2flat.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/borg2flat.py b/borg2flat.py index 6d4d260..14dfa28 100755 --- a/borg2flat.py +++ b/borg2flat.py @@ -4,6 +4,7 @@ import argparse import json import stat +import sys def getFileInfo(path, root, is_excluded = False): stats = os.lstat(path) @@ -29,5 +30,19 @@ def getFileInfo(path, root, is_excluded = False): args.root = args.root.rstrip("/") for line in args.file: - filename = line.rstrip("\n") - print(json.dumps(getFileInfo(filename, args.root))) + # FIXME For now we map errors to exclusions because flatten does not + # support adding "read_error": true + exclusion_letters = [ "x", "E" ] + inclusion_letters = [ "-" ] + # Borg prints errors to the same output as the dry-run output i.e. + # inaccessible: dir_open: [Errno 13] Permission denied: 'inaccessible' + # The best determinator for this is that they typically don't have a space + # as the second character and don't start with a - + # FIXME ask upstream for a better solution + if line[1] != " " or line[0] not in exclusion_letters + inclusion_letters: + print(f"ERROR: Not a legal borg dry-run line: \"{line.rstrip('\n')}\"", file=sys.stderr) + continue + + filename = line[2:].rstrip("\n") + excluded = line[0] in exclusion_letters + print(json.dumps(getFileInfo(filename, args.root, excluded))) From 969b7a27b4d78619c7242ae2eaec4ffa91897cfa Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 13:21:27 +0200 Subject: [PATCH 08/12] borg.sh: init --- borg.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 borg.sh diff --git a/borg.sh b/borg.sh new file mode 100755 index 0000000..944b086 --- /dev/null +++ b/borg.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# Runs `borg create` with the required flags and pipe stderr to borg2flat +exec borg create --dry-run --list "$@" 2>&1 | $(dirname "$0")/borg2flat.py /dev/stdin From 022a032171ad2614be88ae03de6ceb305e7fb37a Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 13:31:27 +0200 Subject: [PATCH 09/12] README: document borg export --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 43f3e6d..2ab45e3 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,22 @@ or $ ./find.sh ~/projects/ | ./find2flat.py - | ./unflatten.py - | ncdu -f - +### Borg export + +`borg.sh` allows you to turn the `borg create --dry-run --list` output into an +ncdu-compatible JSON format. This allows you to preview the directories which +borg would back up using ncdu's interactive, discoverable front-end. Unlike the +`find` export however, borg and the python script must be ran as the same user +on the same machine for this to work. + + $ ./borg.sh /path/to/borg/repo::{now} /path/to/back/up/ > borg-flat-export.json + $ ./unflatten.py borg-flat-export.json > borg-export.json + $ ncdu -f borg-export.json + +or + + $ ./borg.sh /path/to/borg/repo::{now} /path/to/back/up/ | ./unflatten.py - | ncdu -f - + ## Graph of tools From 1a6ee8ad30c68b17b8716b294f4f2c19a69df1c9 Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 13:40:26 +0200 Subject: [PATCH 10/12] README: include borg.sh in graph --- README.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2ab45e3..734c912 100644 --- a/README.md +++ b/README.md @@ -125,29 +125,29 @@ or .------------. - .---------------| filesystem | - | '------------' - | | - | | ncdu -o / ncdu-export - | v - | .------. .---------. - | find.sh | ncdu | ncdu -f | ncdu | - | | JSON |-------->| preview | - | '------' '---------' - | | ^ - | flatten.py | | unflatten.py - v v | + .----.-------------| filesystem | + | | '------------' + | | | + | | | ncdu -o / ncdu-export + | | v + | | .------. .---------. + | | find.sh | ncdu | ncdu -f | ncdu | + | | | JSON |-------->| preview | + | | '------' '---------' + | | | ^ + | | flatten.py | | unflatten.py + | v v | .--------. .------. | find | find2flat.py | flat |<---. jq filtering | output |------------->| JSON |----' '--------' '------' - | - | jq - v - .-----------. .---------. - | tar | tar -T | tar | - | file list |------->| archive | - '-----------' '---------' + | ^ | + | | | jq + v | v + .---------. | .-----------. .---------. + | borg.sh |---------------' | tar | tar -T | tar | + | output | | file list |------->| archive | + '---------' '-----------' '---------' From 39bdeddc2d6f8983dc3ee19b44601383f91de8ba Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 14:23:48 +0200 Subject: [PATCH 11/12] borg2flat: correct arg help string --- borg2flat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg2flat.py b/borg2flat.py index 14dfa28..55700da 100755 --- a/borg2flat.py +++ b/borg2flat.py @@ -24,7 +24,7 @@ def getFileInfo(path, root, is_excluded = False): p = argparse.ArgumentParser() p.add_argument("--root", default="", help="root directory name") -p.add_argument("file", type=argparse.FileType("r"), help="find export filename") +p.add_argument("file", type=argparse.FileType("r"), help="borg export filename") args = p.parse_args() args.root = args.root.rstrip("/") From 58eb7614d88608530f949b2c13da4efc70b7e685 Mon Sep 17 00:00:00 2001 From: Atemu Date: Thu, 26 Sep 2024 14:37:21 +0200 Subject: [PATCH 12/12] borg2flat: don't ensure ascii in JSON dump This mitigates https://github.com/wodny/ncdu-export/issues/5 for at least this part of the pipeline. unflatten.py's processing still makes ncdu crash. --- borg2flat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg2flat.py b/borg2flat.py index 55700da..69a75ea 100755 --- a/borg2flat.py +++ b/borg2flat.py @@ -45,4 +45,4 @@ def getFileInfo(path, root, is_excluded = False): filename = line[2:].rstrip("\n") excluded = line[0] in exclusion_letters - print(json.dumps(getFileInfo(filename, args.root, excluded))) + print(json.dumps(getFileInfo(filename, args.root, excluded), ensure_ascii = False))