PG-1870 Enable WAL encryption in TAP tests

This enables WAL encryption by default when the TAP tests are run with
TDE_MODE=1. Use TDE_MODE_WAL=0 to disable wal encryption while still
having pg_tde enabled.

Author: AndersAstrand

src/bin/pg_basebackup/t/010_pg_basebackup.pl

@@ -10,6 +10,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 program_help_ok('pg_basebackup');
 program_version_ok('pg_basebackup');
 program_options_handling_ok('pg_basebackup');

src/bin/pg_combinebackup/t/003_timeline.pl

@@ -10,6 +10,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 # Can be changed to test the other modes.
 my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
 

src/bin/pg_combinebackup/t/006_db_file_copy.pl

@@ -7,6 +7,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 # Can be changed to test the other modes.
 my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
 

src/bin/pg_combinebackup/t/008_promote.pl

@@ -10,6 +10,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 # Can be changed to test the other modes.
 my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
 

src/bin/pg_rewind/t/001_basic.pl

@@ -11,6 +11,12 @@
 
 use RewindTest;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "copies WAL directly to archive without using archive_command";
+}
+
 sub run_test
 {
 	my $test_mode = shift;

src/bin/pg_verifybackup/t/009_extract.pl

@@ -10,6 +10,13 @@
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
+
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;

src/bin/pg_waldump/t/001_basic.pl

@@ -7,6 +7,11 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all => "pg_waldump needs extra options for encrypted WAL";
+}
+
 program_help_ok('pg_waldump');
 program_version_ok('pg_waldump');
 program_options_handling_ok('pg_waldump');

src/bin/pg_waldump/t/002_save_fullpage.pl

@@ -9,6 +9,11 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all => "pg_waldump needs extra options for encrypted WAL";
+}
+
 my ($blocksize, $walfile_name);
 
 # Function to extract the LSN from the given block structure

src/test/perl/PostgreSQL/Test/TdeCluster.pm

@@ -14,6 +14,7 @@ our ($tde_template_dir);
 BEGIN
 {
 	$ENV{TDE_MODE_NOSKIP} = 0 unless defined($ENV{TDE_MODE_NOSKIP});
+	$ENV{TDE_MODE_WAL} = 1 unless defined($ENV{TDE_MODE_WAL});
 }
 
 sub init
@@ -27,6 +28,12 @@ sub init
 
 	$self->_tde_init_principal_key;
 
+	if ($ENV{TDE_MODE_WAL})
+	{
+		$self->SUPER::append_conf('postgresql.conf',
+			'pg_tde.wal_encrypt = on');
+	}
+
 	return;
 }
 
@@ -45,6 +52,63 @@ sub append_conf
 	$self->SUPER::append_conf($filename, $str);
 }
 
+sub backup
+{
+	my ($self, $backup_name, %params) = @_;
+	my $backup_dir = $self->backup_dir . '/' . $backup_name;
+
+	mkdir $backup_dir or die "mkdir($backup_dir) failed: $!";
+
+	if ($ENV{TDE_MODE_WAL})
+	{
+		PostgreSQL::Test::Utils::system_log('cp', '-R', '-P', '-p',
+			$self->pg_tde_dir, $backup_dir . '/pg_tde',);
+
+		# TODO: More thorough checking for options incompatible with --encrypt-wal
+		$params{backup_options} = [] unless defined $params{backup_options};
+		unless (
+			List::Util::any { $_ eq '-Ft' or $_ eq '-Xnone' }
+			@{ $params{backup_options} })
+		{
+			push @{ $params{backup_options} }, '--encrypt-wal';
+		}
+	}
+
+	$self->SUPER::backup($backup_name, %params);
+}
+
+sub enable_archiving
+{
+	my ($self) = @_;
+	my $path = $self->archive_dir;
+
+	$self->SUPER::enable_archiving;
+	if ($ENV{TDE_MODE_WAL})
+	{
+		$self->adjust_conf('postgresql.conf', 'archive_command',
+			qq('pg_tde_archive_decrypt %f %p "cp \\"%%p\\" \\"$path/%%f\\""')
+		);
+	}
+
+	return;
+}
+
+sub enable_restoring
+{
+	my ($self, $root_node, $standby) = @_;
+	my $path = $root_node->archive_dir;
+
+	$self->SUPER::enable_restoring($root_node, $standby);
+	if ($ENV{TDE_MODE_WAL})
+	{
+		$self->adjust_conf('postgresql.conf', 'restore_command',
+			qq('pg_tde_restore_encrypt %f %p "cp \\"$path/%%f\\" \\"%%p\\""')
+		);
+	}
+
+	return;
+}
+
 sub pg_tde_dir
 {
 	my ($self) = @_;

src/test/recovery/t/039_end_of_wal.pl

@@ -13,6 +13,11 @@
 
 use integer;    # causes / operator to use integer math
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all => 'uses write_wal to hack wal directly';
+}
+
 # Is this a big-endian system ("network" byte order)?  We can't use 'Q' in
 # pack() calls because it's not available in some perl builds, so we need to
 # break 64 bit LSN values into two 'I' values.  Fortunately we don't need to