From 726e6a2ed5b271e13b116f896442987bacfb390b Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Sun, 19 Jul 2020 22:17:40 +0200 Subject: [PATCH 1/8] new release cycle --- CHANGES.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 30a41a4..a260b8f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Changes +## 0.0.4 (2020-XX-XX) + +- (TBD) + ## 0.0.3 (2020-07-19) - FEATURE: Added wizard GUI for backup tasks (`snap`, `backup`, `cleanup`) diff --git a/setup.py b/setup.py index 45d2fc6..df9c1d1 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ # Package version -__version__ = "0.0.3" +__version__ = "0.0.4" # List all versions of Python which are supported python_minor_min = 6 From 15dbc73314c2d740d973af6daef79df09bad259e Mon Sep 17 00:00:00 2001 From: Akhuna Date: Mon, 20 Jul 2020 19:58:40 +0200 Subject: [PATCH 2/8] Uebersetzungen deutsch und englisch --- src/abgleich/share/translations.yaml | 108 +++++++++++++++++---------- 1 file changed, 69 insertions(+), 39 deletions(-) diff --git a/src/abgleich/share/translations.yaml b/src/abgleich/share/translations.yaml index ae0b469..5de5a56 100644 --- a/src/abgleich/share/translations.yaml +++ b/src/abgleich/share/translations.yaml @@ -5,73 +5,103 @@ Cancel: Close: de: Schließen Collect Backup Transactions: - de: Sicherungstransaktionen sammeln + de_SE: Sicherungstransaktionen sammeln + de: Snapshots vergleichen + en: Compare snapshots Collect Cleanup Transactions: - de: Säuberungstransaktionen sammeln + de_SE: Säuberungstransaktionen sammeln + de: Snapshots ermitteln + en: Detect snapshots Collect Snapshot Transactions: - de: Schnappschuss-Transaktionen sammeln + de_SE: Schnappschuss-Transaktionen sammeln + de: Änderungen ermitteln + en: Detect changes Collecting backup transactions ...: - de: Sicherungstransaktionen sammeln ... + de_SE: Sicherungstransaktionen sammeln … + de: Snapshots des Quell- und Ziel-Systems vergleichen ... + en: Compare snapshots on source and target sides ... Collecting cleanup transactions ...: - de: Säuberungstransaktionen sammeln ... + de_SE: Säuberungstransaktionen sammeln … + de: Nicht mehr erforderliche Snapshots ermitteln ... + en: Detect obsolete snapshots ... Collecting snapshot transactions ...: - de: Schnappschuss-Transaktionen sammeln ... + de_SE: Schnappschuss-Transaktionen sammeln … + de: Änderungen im Quell-System ermitteln … + en: Detect changes on source side ... Creating snapshots ...: - de: Schnappschüsse erstellen ... + de_SE: Schnappschüsse erstellen … + de: Snapshots im Quell-System anlegen … + en: Create snapshots on source side ... Do you want to continue?: de: Möchten Sie fortfahren? Execute Backup Transactions: - de: Sicherungstransaktionen ausführen + de_SE: Sicherungstransaktionen ausführen + de: Snapshots vom Quell-System auf das Ziel-System übertragen … + en: Transfer snapshots from source to target side ... Execute Cleanup Transactions: - de: Säuberungstransaktionen ausführen + de_SE: Säuberungstransaktionen ausführen + de: Snapshots entfernen + en: Delete snapshots Execute Snapshot Transactions: - de: Schnappschuss-Transaktionen ausführen + de_SE: Schnappschuss-Transaktionen ausführen + de: Snapshots anlegen + en: Create snapshots +OK: {} FAILED: de: FEHLGESCHLAGEN -NAME: {} -OK: {} Old snapshots removed.: - de: Alte Schnappschüsse entfernt. -REFER: - de: VERWEIST + de_SE: Alte Schnappschüsse entfernt. + de: Alte Snapshots erfolgreich entfernt! Removing old snapshots ...: - de: Entferne alte Schnappschüsse ... + de_SE: Entferne alte Schnappschüsse … + de: Entferne alte Snapshots Snapshots created.: - de: Schnappschüsse erstellt. + de_SE: Schnappschüsse erstellt. + de: Snapshots erfolgreich angelegt! Snapshots transferred.: - de: Schnappschüsse gesichert. + de_SE: Schnappschüsse gesichert. + de: Snapshots erfolgreich übertragen! Transferring snapshots ...: - de: Sichere Schnappschüsse ... -USED: - de: BELEGT + de_SE: Sichere Schnappschüsse … + de: Übertrage Snapshots. abgleich wizard: de: Abgleich Assistent ancestor_name: - de: Vorfahr + de_SE: Vorfahr + de: Vorhergehender Snapshot + en: Name of ancestor cleanup_snapshot: - de: Säuberung -compressratio: - de: Kompressionsrate + de_SE: Säuberung + de: Zu löschender Snapshot + en: Obsolete snapshot dataset_subname: - de: Datensatz-Namensfragment + de_SE: Datensatz-Namensfragment + de: Name des Datensatzes + en: Name of dataset nothing to do: de: nichts zu tun -snapshot: - de: Schnappschuss -snapshot_name: - de: Schnappschuss-Name -snapshot_subparent: - de: Datensatz-Namensfragment -source: - de: Quelle -target: - de: Ziel transaction: - de: Transaktion + de_SE: Transaktion + de: Aktion transfer_snapshot: - de: Sicherung + de_SE: Sicherung + de: Neuer Datensatz & Snapshot + en: New dataset & snapshot transfer_snapshot_incremental: - de: Inkrementelle Sicherung + de_SE: Inkrementelle Sicherung + de: Differenz zweier Snapshots + en: Difference between snapshots +snapshot: + de_SE: Schnappschuss + de: Snapshot +snapshot_name: + de_SE: Schnappschuss-Name + de: Snapshot-Name + en: Name of snapshot +snapshot_subparent: + de_SE: Datensatz-Namensfragment + de: Name des Datensatzes + en: Name of dataset type: de: Typ written: From 0afd3a4a923d15e74b545edb2f9ac9c098d27296 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Mon, 20 Jul 2020 20:22:13 +0200 Subject: [PATCH 3/8] organized translations --- src/abgleich/share/translations.yaml | 65 +++++++++++++++++----------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/abgleich/share/translations.yaml b/src/abgleich/share/translations.yaml index 5de5a56..60f04d0 100644 --- a/src/abgleich/share/translations.yaml +++ b/src/abgleich/share/translations.yaml @@ -10,34 +10,34 @@ Collect Backup Transactions: en: Compare snapshots Collect Cleanup Transactions: de_SE: Säuberungstransaktionen sammeln - de: Snapshots ermitteln - en: Detect snapshots + de: Alte Snapshots ermitteln + en: Detect obsolete snapshots Collect Snapshot Transactions: de_SE: Schnappschuss-Transaktionen sammeln de: Änderungen ermitteln en: Detect changes Collecting backup transactions ...: - de_SE: Sicherungstransaktionen sammeln … + de_SE: Sicherungstransaktionen sammeln ... de: Snapshots des Quell- und Ziel-Systems vergleichen ... en: Compare snapshots on source and target sides ... Collecting cleanup transactions ...: - de_SE: Säuberungstransaktionen sammeln … + de_SE: Säuberungstransaktionen sammeln ... de: Nicht mehr erforderliche Snapshots ermitteln ... en: Detect obsolete snapshots ... Collecting snapshot transactions ...: - de_SE: Schnappschuss-Transaktionen sammeln … - de: Änderungen im Quell-System ermitteln … + de_SE: Schnappschuss-Transaktionen sammeln ... + de: Änderungen im Quell-System ermitteln ... en: Detect changes on source side ... Creating snapshots ...: - de_SE: Schnappschüsse erstellen … - de: Snapshots im Quell-System anlegen … + de_SE: Schnappschüsse erstellen ... + de: Snapshots im Quell-System anlegen ... en: Create snapshots on source side ... Do you want to continue?: de: Möchten Sie fortfahren? Execute Backup Transactions: de_SE: Sicherungstransaktionen ausführen - de: Snapshots vom Quell-System auf das Ziel-System übertragen … - en: Transfer snapshots from source to target side ... + de: Snapshots übertragen + en: Transfer snapshots Execute Cleanup Transactions: de_SE: Säuberungstransaktionen ausführen de: Snapshots entfernen @@ -46,15 +46,18 @@ Execute Snapshot Transactions: de_SE: Schnappschuss-Transaktionen ausführen de: Snapshots anlegen en: Create snapshots -OK: {} FAILED: de: FEHLGESCHLAGEN +NAME: {} +OK: {} Old snapshots removed.: de_SE: Alte Schnappschüsse entfernt. de: Alte Snapshots erfolgreich entfernt! +REFER: + de: VERWEIST Removing old snapshots ...: - de_SE: Entferne alte Schnappschüsse … - de: Entferne alte Snapshots + de_SE: Entferne alte Schnappschüsse ... + de: Entferne alte Snapshots ... Snapshots created.: de_SE: Schnappschüsse erstellt. de: Snapshots erfolgreich angelegt! @@ -62,8 +65,11 @@ Snapshots transferred.: de_SE: Schnappschüsse gesichert. de: Snapshots erfolgreich übertragen! Transferring snapshots ...: - de_SE: Sichere Schnappschüsse … - de: Übertrage Snapshots. + de_SE: Sichere Schnappschüsse ... + de: Snapshots vom Quell-System auf das Ziel-System übertragen ... + en: Transfer snapshots from source to target side ... +USED: + de: BELEGT abgleich wizard: de: Abgleich Assistent ancestor_name: @@ -74,23 +80,15 @@ cleanup_snapshot: de_SE: Säuberung de: Zu löschender Snapshot en: Obsolete snapshot +compressratio: + de: Kompressionsrate + en: Compression ratio dataset_subname: de_SE: Datensatz-Namensfragment de: Name des Datensatzes en: Name of dataset nothing to do: de: nichts zu tun -transaction: - de_SE: Transaktion - de: Aktion -transfer_snapshot: - de_SE: Sicherung - de: Neuer Datensatz & Snapshot - en: New dataset & snapshot -transfer_snapshot_incremental: - de_SE: Inkrementelle Sicherung - de: Differenz zweier Snapshots - en: Difference between snapshots snapshot: de_SE: Schnappschuss de: Snapshot @@ -102,6 +100,21 @@ snapshot_subparent: de_SE: Datensatz-Namensfragment de: Name des Datensatzes en: Name of dataset +source: + de: Quelle +target: + de: Ziel +transaction: + de_SE: Transaktion + de: Aktion +transfer_snapshot: + de_SE: Sicherung + de: Neuer Datensatz & Snapshot + en: New dataset & snapshot +transfer_snapshot_incremental: + de_SE: Inkrementelle Sicherung + de: Differenz zweier Snapshots + en: Difference between snapshots type: de: Typ written: From 2ce49af93d911ccf0aa5cd48491191687c8f7c3b Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Mon, 20 Jul 2020 22:58:46 +0200 Subject: [PATCH 4/8] grammar fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9b85e9d..19dc342 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ ssh: cipher: aes256-gcm@openssh.com ``` -The prefix can be empty on either side. If a `host` is set to `localhost`, the `user` field can be left empty. Both source and target can be remote hosts or localhost at the same time. `keep_snapshots` is an integer and must be greater or equal to `1`. It specifies the number of snapshots that are kept per dataset on the source side when a cleanup operation is triggered. `suffix` contains the name suffix for new snapshots. Setting `always_changed` to `yes` causes `abgleich` to beliefe that all datasets have always changed since the last snapshot, completely ignoring what ZFS actually reports. No diff will be checked produced for values of `written` lower than `written_threshold`. Checking diffs can be completely deactivated by setting `check_diff` to `no`. `digits` specifies how many digits are used for a decimal number describing the n-th snapshot per dataset per day as part of the name of new snapshots. `ignore` lists stuff underneath the `prefix` which will be ignored by this tool, i.e. no snapshots, backups or cleanups. `ssh` allows to fine-tune the speed of backups. In fast local networks, it is best to set `compression` to `no` because the compression is usually slowing down the transfer. However, for low-bandwidth transmissions, it makes sense to set it to `yes`. For significantly better speed in fast local networks, make sure that both the source and the target system support a common cipher, which is accelerated by [AES-NI](https://en.wikipedia.org/wiki/AES_instruction_set) on both ends. +The prefix can be empty on either side. If a `host` is set to `localhost`, the `user` field can be left empty. Both source and target can be remote hosts or localhost at the same time. `keep_snapshots` is an integer and must be greater or equal to `1`. It specifies the number of snapshots that are kept per dataset on the source side when a cleanup operation is triggered. `suffix` contains the name suffix for new snapshots. Setting `always_changed` to `yes` causes `abgleich` to beliefe that all datasets have always changed since the last snapshot, completely ignoring what ZFS actually reports. No diff will be produced & checked for values of `written` lower than `written_threshold`. Checking diffs can be completely deactivated by setting `check_diff` to `no`. `digits` specifies how many digits are used for a decimal number describing the n-th snapshot per dataset per day as part of the name of new snapshots. `ignore` lists stuff underneath the `prefix` which will be ignored by this tool, i.e. no snapshots, backups or cleanups. `ssh` allows to fine-tune the speed of backups. In fast local networks, it is best to set `compression` to `no` because the compression is usually slowing down the transfer. However, for low-bandwidth transmissions, it makes sense to set it to `yes`. For significantly better speed in fast local networks, make sure that both the source and the target system support a common cipher, which is accelerated by [AES-NI](https://en.wikipedia.org/wiki/AES_instruction_set) on both ends. ## USAGE From e032fc176171aa03be5889d67ef8086a9fd54058 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 22 Jul 2020 20:16:53 +0200 Subject: [PATCH 5/8] fix #2 yaml's CLoader is not always present, allow fall-back --- src/abgleich/core/config.py | 8 ++++++-- src/abgleich/core/i18n.py | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/abgleich/core/config.py b/src/abgleich/core/config.py index 251c3a9..045db00 100644 --- a/src/abgleich/core/config.py +++ b/src/abgleich/core/config.py @@ -33,7 +33,11 @@ import typeguard import yaml -from yaml import CLoader + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import FullLoader as Loader from .abc import ConfigABC from .lib import valid_name @@ -71,7 +75,7 @@ def from_fd(cls, fd: typing.TextIO): "ssh": lambda v: cls._validate(data=v, schema=ssh_schema), } - config = yaml.load(fd.read(), Loader=CLoader) + config = yaml.load(fd.read(), Loader=Loader) cls._validate(data=config, schema=root_schema) return cls(config) diff --git a/src/abgleich/core/i18n.py b/src/abgleich/core/i18n.py index 5007675..2bc34ef 100644 --- a/src/abgleich/core/i18n.py +++ b/src/abgleich/core/i18n.py @@ -33,7 +33,16 @@ import typeguard import yaml -from yaml import CDumper, CLoader + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import FullLoader as Loader + +try: + from yaml import CDumper as Dumper +except ImportError: + from yaml import Dumper # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ # CLASS @@ -71,13 +80,13 @@ def _load(self): self.clear() with open(self._path, "r") as f: - self.update(yaml.load(f.read(), Loader=CLoader)) + self.update(yaml.load(f.read(), Loader=Loader)) def _dump(self): with open(self._path, "w") as f: f.write( - yaml.dump(self.copy(), Dumper=CDumper, allow_unicode=True, indent=4) + yaml.dump(self.copy(), Dumper=Dumper, allow_unicode=True, indent=4) ) From e18aa986ef6c05517301ee752c29afb8d3c28aff Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 22 Jul 2020 20:44:00 +0200 Subject: [PATCH 6/8] fix #6 and fix #10: mountpoint property is not always present + do not ignore zvols when looking for snapshot tasks --- src/abgleich/core/dataset.py | 19 +++++++++++++++++++ src/abgleich/core/zpool.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/abgleich/core/dataset.py b/src/abgleich/core/dataset.py index 8182b31..913c439 100644 --- a/src/abgleich/core/dataset.py +++ b/src/abgleich/core/dataset.py @@ -82,6 +82,25 @@ def __getitem__(self, key: typing.Union[str, int, slice]) -> PropertyABC: return self._properties[key] return self._snapshots[key] + def get( + self, + key: typing.Union[str, int, slice], + default: typing.Union[None, PropertyABC] = None, + ) -> typing.Union[None, PropertyABC]: + + if isinstance(key, str): + return self._properties.get( + key, + Property(key, None, None) if default is None else default, + ) + + assert isinstance(key, int) or isinstance(key, slice) + + try: + return self._snapshots[key] + except IndexError: + return default + @property def changed(self) -> bool: diff --git a/src/abgleich/core/zpool.py b/src/abgleich/core/zpool.py index ca3015c..71e9803 100644 --- a/src/abgleich/core/zpool.py +++ b/src/abgleich/core/zpool.py @@ -254,7 +254,7 @@ def _get_snapshot_transactions_from_dataset( if dataset.subname in self._config["ignore"]: return - if dataset["mountpoint"].value is None: + if dataset.get("mountpoint").value is None and dataset['type'].value == 'filesystem': return if not dataset.changed: return From 88fdb706efce3bcf139e13871fd393cd689009c6 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 22 Jul 2020 20:55:03 +0200 Subject: [PATCH 7/8] logged changes --- CHANGES.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a260b8f..932fe8b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,11 @@ ## 0.0.4 (2020-XX-XX) -- (TBD) +- FEATURE: Improved labels in wizard GUI +- FEATURE: Significantly improved German translation +- FIX: Importing `CLoader` and `CDumper` from `pyyaml` caused crashes if they were not present in `pyyaml` packages, see #2. +- FIX: The `mountpoint` property of ZFS datasets is no longer assumed to be present at all (set or unset). This allows to handle ZVOLs without crashing, see issue #6. +- FIX: Versions 0.0.2 and 0.0.3 were completely ignoring ZVOLs when looking for snapshot tasks, see issue #10. ## 0.0.3 (2020-07-19) From 036d91061e6f53d77f8ff4e22eceb039c9b57b7c Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 22 Jul 2020 20:55:29 +0200 Subject: [PATCH 8/8] prepare for release --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 932fe8b..5113087 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # Changes -## 0.0.4 (2020-XX-XX) +## 0.0.4 (2020-07-22) - FEATURE: Improved labels in wizard GUI - FEATURE: Significantly improved German translation