From c7ae4ccca7e2ade3a54ba92b1d5531c661c35573 Mon Sep 17 00:00:00 2001 From: hija Date: Wed, 26 Aug 2020 15:32:32 +0200 Subject: [PATCH 01/12] Installing b2sdk for b2 support --- setup/management.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/management.sh b/setup/management.sh index 4b398aa21..7de169736 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -32,6 +32,9 @@ done apt_install duplicity python-pip virtualenv certbot hide_output pip2 install --upgrade boto +# b2sdk is installed outside the pipenv, so it can be used by duplicity +hide_output pip3 install b2sdk + # Create a virtualenv for the installation of Python 3 packages # used by the management daemon. inst_dir=/usr/local/lib/mailinabox From 7b4d621cc880de05e1ac6788824a2f810ac4ee8e Mon Sep 17 00:00:00 2001 From: hija Date: Wed, 26 Aug 2020 15:34:07 +0200 Subject: [PATCH 02/12] Added Duplicity PPA so the most recent version is used --- setup/system.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/system.sh b/setup/system.sh index 4d33deb66..07f4aa1b0 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -93,6 +93,9 @@ hide_output add-apt-repository -y universe # Install the certbot PPA. hide_output add-apt-repository -y ppa:certbot/certbot +# Install the duplicity PPA. +hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git + # ### Update Packages # Update system packages to make sure we have the latest upstream versions From 9199294455d0115eddcbc5b064628cf302fb9ae0 Mon Sep 17 00:00:00 2001 From: hija Date: Wed, 26 Aug 2020 15:36:59 +0200 Subject: [PATCH 03/12] Implemented list_target_files for b2 --- management/backup.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/management/backup.py b/management/backup.py index e16515522..0a8a021e2 100755 --- a/management/backup.py +++ b/management/backup.py @@ -456,6 +456,23 @@ def list_target_files(config): raise ValueError(e.reason) return [(key.name[len(path):], key.size) for key in bucket.list(prefix=path)] + elif target.scheme == 'b2': + from b2sdk.v1 import InMemoryAccountInfo, B2Api + from b2sdk.v1.exception import NonExistentBucket + info = InMemoryAccountInfo() + b2_api = B2Api(info) + + # Extract information from target + b2_application_keyid = target.netloc[:target.netloc.index(':')] + b2_application_key = target.netloc[target.netloc.index(':')+1:target.netloc.index('@')] + b2_bucket = target.netloc[target.netloc.index('@')+1:] + + try: + b2_api.authorize_account("production", b2_application_keyid, b2_application_key) + bucket = b2_api.get_bucket_by_name(b2_bucket) + except NonExistentBucket as e: + raise ValueError("B2 Bucket does not exist. Please double check your information!") + return [(key.file_name, key.size) for key, _ in bucket.ls()] else: raise ValueError(config["target"]) From 1081be8ba5de62887466329d4e75dd38b0971c72 Mon Sep 17 00:00:00 2001 From: hija Date: Wed, 26 Aug 2020 15:37:44 +0200 Subject: [PATCH 04/12] Implemented b2 in frontend --- management/templates/system-backup.html | 611 +++++++++++++----------- 1 file changed, 334 insertions(+), 277 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 3860edb7d..e0ffdb63b 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -1,306 +1,363 @@ - -

Backup Status

- -

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3.

- -

Configuration

- -
-
- -
- + #backup-status th { text-align: center; } + #backup-status tr.full-backup td { font-weight: bold; } + + +

Backup Status

+ +

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3.

+ +

Configuration

+ + +
+ +
+ +
+
+ +
+
+

Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.

+

Separately copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files.

+
+
+ +
+
+ +

Backups synced to a remote machine using rsync over SSH, with local + copies in . These files are encrypted, so + they are safe to store anywhere.

Separately copy the encryption + password from to a safe and + secure location. You will need this file to decrypt backup files.

+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
-
- -
-
-

Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.

-

Separately copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files.

+
+ +
+ +
+ Copy the Public SSH Key above, and paste it within the ~/.ssh/authorized_keys + of target user on the backup server specified above. That way you'll enable secure and + passwordless authentication from your mail-in-a-box server and your backup server. +
+
-
- -
-
- -

Backups synced to a remote machine using rsync over SSH, with local - copies in . These files are encrypted, so - they are safe to store anywhere.

Separately copy the encryption - password from to a safe and - secure location. You will need this file to decrypt backup files.

- + +
+
+

Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.

+

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Amazon S3 bucket.

+
-
-
- -
- +
+ +
+ +
-
-
- -
- +
+ +
+ +
-
-
- -
- +
+ +
+ +
-
-
- -
- -
- Copy the Public SSH Key above, and paste it within the ~/.ssh/authorized_keys - of target user on the backup server specified above. That way you'll enable secure and - passwordless authentication from your mail-in-a-box server and your backup server. +
+ +
+
-
- -
-
-

Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.

-

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Amazon S3 bucket.

+
+ +
+ +
-
-
- -
- + +
+
+

Backups are stored in a Backblaze B2 bucket. You must have a Backblaze account already.

+

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Backblaze B2 bucket.

+
-
-
- -
- +
+ +
+ +
-
-
- -
- +
+ +
+ +
-
-
- -
- +
+ +
+ +
-
-
- -
- + +
+ +
+ +
-
- -
- -
- -
This is the minimum number of days backup data is kept for. The box makes an incremental backup, so backup data is often kept much longer. An incremental backup file that is less than this number of days old requires that all previous increments back to the most recent full backup, plus that full backup, remain available.
+ +
+ +
+ +
This is the minimum number of days backup data is kept for. The box makes an incremental backup, so backup data is often kept much longer. An incremental backup file that is less than this number of days old requires that all previous increments back to the most recent full backup, plus that full backup, remain available.
+
-
-
-
- +
+
+ +
-
- - -

Available backups

- -

The backup location currently contains the backups listed below. The total size of the backups is currently .

- - - - - - - - - - -
WhenTypeSizeDeleted in...
- + + \ No newline at end of file From df6fcf3bd2c92b7bfbe325dd6d681fa2b0f4c220 Mon Sep 17 00:00:00 2001 From: hija Date: Wed, 26 Aug 2020 16:14:05 +0200 Subject: [PATCH 05/12] Added b2sdk to virtualenv so backup.py can use it --- setup/management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index 7de169736..aac7c1263 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -53,7 +53,7 @@ hide_output $venv/bin/pip install --upgrade pip hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ flask dnspython python-dateutil \ - "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver + "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver b2sdk # CONFIGURATION From 8085246fedcebae2c2df43bba5a5b127b0ebc519 Mon Sep 17 00:00:00 2001 From: hija Date: Wed, 26 Aug 2020 19:46:09 +0200 Subject: [PATCH 06/12] Changed indentation to match the old one --- management/templates/system-backup.html | 635 ++++++++++++------------ 1 file changed, 316 insertions(+), 319 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index e0ffdb63b..463d811f0 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -1,363 +1,360 @@ - -

Backup Status

- -

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3.

- -

Configuration

- -
-
- -
- -
+#backup-status th { text-align: center; } +#backup-status tr.full-backup td { font-weight: bold; } + + +

Backup Status

+ +

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3.

+ +

Configuration

+ + +
+ +
+
- -
-
-

Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.

-

Separately copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files.

-
+
+ +
+
+

Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.

+

Separately copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files.

- -
-
- -

Backups synced to a remote machine using rsync over SSH, with local - copies in . These files are encrypted, so - they are safe to store anywhere.

Separately copy the encryption - password from to a safe and - secure location. You will need this file to decrypt backup files.

- -
+
+ +
+
+ +

Backups synced to a remote machine using rsync over SSH, with local + copies in . These files are encrypted, so + they are safe to store anywhere.

Separately copy the encryption + password from to a safe and + secure location. You will need this file to decrypt backup files.

+
-
- -
- -
+
+
+ +
+
-
- -
- -
+
+
+ +
+
-
- -
- -
+
+
+ +
+
-
- -
- -
- Copy the Public SSH Key above, and paste it within the ~/.ssh/authorized_keys - of target user on the backup server specified above. That way you'll enable secure and - passwordless authentication from your mail-in-a-box server and your backup server. -
+
+
+ +
+ +
+ Copy the Public SSH Key above, and paste it within the ~/.ssh/authorized_keys + of target user on the backup server specified above. That way you'll enable secure and + passwordless authentication from your mail-in-a-box server and your backup server.
- -
-
-

Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.

-

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Amazon S3 bucket.

-
+
+ +
+
+

Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.

+

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Amazon S3 bucket.

-
- -
- -
+
+
+ +
+
-
- -
- -
+
+
+ +
+
-
- -
- -
+
+
+ +
+
-
- -
- -
+
+
+ +
+
-
- -
- -
+
+
+ +
+
- -
+
+ +
-

Backups are stored in a Backblaze B2 bucket. You must have a Backblaze account already.

-

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Backblaze B2 bucket.

+

Backups are stored in a Backblaze B2 bucket. You must have a Backblaze account already.

+

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Backblaze B2 bucket.

-
-
+
+
- +
-
-
+
+
- +
-
-
+
+
- +
-
- -
+
+
- +
+
+ +
+ +
+ +
This is the minimum number of days backup data is kept for. The box makes an incremental backup, so backup data is often kept much longer. An incremental backup file that is less than this number of days old requires that all previous increments back to the most recent full backup, plus that full backup, remain available.
- -
- -
- -
This is the minimum number of days backup data is kept for. The box makes an incremental backup, so backup data is often kept much longer. An incremental backup file that is less than this number of days old requires that all previous increments back to the most recent full backup, plus that full backup, remain available.
-
+
+
+
+
-
-
- -
-
- - -

Available backups

- -

The backup location currently contains the backups listed below. The total size of the backups is currently .

- - - - - - - - - - -
WhenTypeSizeDeleted in...
- \ No newline at end of file +} + \ No newline at end of file From aa5ba9999e004ad681987ce001ffb64f9e404418 Mon Sep 17 00:00:00 2001 From: hija Date: Sun, 30 Aug 2020 12:05:04 +0200 Subject: [PATCH 07/12] removed unnecessary, readonly field in b2 config --- management/templates/system-backup.html | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 463d811f0..5bb7c506e 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -122,25 +122,19 @@

Configuration

- +
- +
- -
-
-
- -
- +
@@ -285,7 +279,6 @@

Available backups

$("#backup-target-b2-user").val(b2_application_keyid); $("#backup-target-b2-pass").val(b2_applicationkey); $("#backup-target-b2-bucket").val(b2_bucket); - compute_b2_url(); } toggle_form() }) @@ -306,7 +299,8 @@

Available backups

+ "/" + $("#backup-target-rsync-path").val(); target_user = ''; } else if (target_type == "b2") { - target = $('#backup-target-b2-url').val(); + target = 'b2://' + $('#backup-target-b2-user').val() + ':' + $('#backup-target-b2-pass').val() + + '@' + $('#backup-target-b2-bucket').val() target_user = ''; target_pass = ''; } @@ -348,13 +342,4 @@

Available backups

set_host($('#backup-target-s3-host-select').val()); } } - -function compute_b2_url(){ - if ($('#backup-target-b2-user').val() && $('#backup-target-b2-pass').val() && $('#backup-target-b2-bucket').val()){ - $('#backup-target-b2-url').val('b2://' + $('#backup-target-b2-user').val() + ':' + $('#backup-target-b2-pass').val() - + '@' + $('#backup-target-b2-bucket').val()); - }else{ - $('#backup-target-b2-url').val(''); - } -} \ No newline at end of file From ace2bbf5e8d752c63302d5c6784d8a9bb784c100 Mon Sep 17 00:00:00 2001 From: hija Date: Sun, 30 Aug 2020 14:08:46 +0200 Subject: [PATCH 08/12] Removed placeholder text for b2 bucket --- management/templates/system-backup.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 5bb7c506e..297c21fec 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -134,7 +134,7 @@

Configuration

- +
From b62dcc5b21896592887d9b3504bfecbc590af2ff Mon Sep 17 00:00:00 2001 From: hija Date: Fri, 4 Sep 2020 18:28:12 +0200 Subject: [PATCH 09/12] Assigning targetPath to a new var --- management/templates/system-backup.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 297c21fec..94536e902 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -273,9 +273,10 @@

Available backups

$("#backup-target-s3-path").val(hostpath.join('/')); } else if (r.target.substring(0, 5) == "b2://") { $("#backup-target-type").val("b2"); - var b2_application_keyid = r.target.substring(5).split(':')[0]; - var b2_applicationkey = r.target.substring(5).split(':')[1].split('@')[0]; - var b2_bucket = r.target.substring(5).split('@')[1]; + var targetPath = r.target.substring(5); + var b2_application_keyid = targetPath.split(':')[0]; + var b2_applicationkey = targetPath.split(':')[1].split('@')[0]; + var b2_bucket = targetPath.split('@')[1]; $("#backup-target-b2-user").val(b2_application_keyid); $("#backup-target-b2-pass").val(b2_applicationkey); $("#backup-target-b2-bucket").val(b2_bucket); From eecf32890e4b3d954095114e2c0d679e5ef98000 Mon Sep 17 00:00:00 2001 From: hija Date: Fri, 4 Sep 2020 18:29:03 +0200 Subject: [PATCH 10/12] Added link to backblaze --- management/templates/system-backup.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 94536e902..f89242cfc 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -115,7 +115,7 @@

Configuration

-

Backups are stored in a Backblaze B2 bucket. You must have a Backblaze account already.

+

Backups are stored in a Backblaze B2 bucket. You must have a Backblaze account already.

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Backblaze B2 bucket.

From 14bbaafe0b0dd8a5bd591a3a61cb0f07359b4ed7 Mon Sep 17 00:00:00 2001 From: hija Date: Fri, 4 Sep 2020 18:29:53 +0200 Subject: [PATCH 11/12] Fixed indentation --- management/templates/system-backup.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index f89242cfc..e8dc4cbf2 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -115,26 +115,26 @@

Configuration

-

Backups are stored in a Backblaze B2 bucket. You must have a Backblaze account already.

-

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Backblaze B2 bucket.

+

Backups are stored in a Backblaze B2 bucket. You must have a Backblaze account already.

+

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Backblaze B2 bucket.

- +
- +
- +
From 8185b76f07df0e3f0d7632137558b7e5ad82f36f Mon Sep 17 00:00:00 2001 From: hija Date: Fri, 4 Sep 2020 18:34:44 +0200 Subject: [PATCH 12/12] removed python2 boto package --- setup/management.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/setup/management.sh b/setup/management.sh index aac7c1263..c83ced1d7 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -18,11 +18,7 @@ while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do pip3 uninstall -y acme; done -# duplicity is used to make backups of user data. It uses boto -# (via Python 2) to do backups to AWS S3. boto from the Ubuntu -# package manager is too out-of-date -- it doesn't support the newer -# S3 api used in some regions, which breaks backups to those regions. -# See #627, #653. +# duplicity is used to make backups of user data. # # virtualenv is used to isolate the Python 3 packages we # install via pip from the system-installed packages. @@ -30,10 +26,11 @@ done # certbot installs EFF's certbot which we use to # provision free TLS certificates. apt_install duplicity python-pip virtualenv certbot -hide_output pip2 install --upgrade boto -# b2sdk is installed outside the pipenv, so it can be used by duplicity -hide_output pip3 install b2sdk +# b2sdk is used for backblaze backups. +# boto is used for amazon aws backups. +# Both are installed outside the pipenv, so they can be used by duplicity +hide_output pip3 install --upgrade b2sdk boto # Create a virtualenv for the installation of Python 3 packages # used by the management daemon.