postgresql/src/bin/pg_verifybackup/t/008_untar.pl

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

116 lines
3.3 KiB
Perl
Raw Normal View History

Server-side gzip compression. pg_basebackup's --compression option now lets you write either "client-gzip" or "server-gzip" instead of just "gzip" to specify where the compression should be performed. If you write simply "gzip" it's taken to mean "client-gzip" unless you also use --target, in which case it is interpreted to mean "server-gzip", because that's the only thing that makes any sense in that case. To make this work, the BASE_BACKUP command now takes new COMPRESSION and COMPRESSION_LEVEL options. At present, pg_basebackup cannot decompress .gz files, so server-side compression will cause a failure if (1) -Ft is not used or (2) -R is used or (3) -D- is used without --no-manifest. Along the way, I removed the information message added by commit 5c649fe153367cdab278738ee4aebbfd158e0546 which occurred if you specified no compression level and told you that the default level had been used instead. That seemed like more output than most people would want. Also along the way, this adds a check to the server for unrecognized base backup options. This repairs a bug introduced by commit 0ba281cb4bf9f5f65529dfa4c8282abb734dd454. This commit also adds some new test cases for pg_verifybackup. They take a server-side backup with and without compression, and then extract the backup if we have the OS facilities available to do so, and then run pg_verifybackup on the extracted directory. That is a good test of the functionality added by this commit and also improves test coverage for the backup target patch (commit 3500ccc39b0dadd1068a03938e4b8ff562587ccc) and for pg_verifybackup itself. Patch by me, with a bug fix by Jeevan Ladhe. The patch set of which this is a part has also had review and/or testing from Tushar Ahuja, Suraj Kharage, Dipesh Pandit, and Mark Dilger. Discussion: http://postgr.es/m/CA+Tgmoa-ST7fMLsVJduOB7Eub=2WjfpHS+QxHVEpUoinf4bOSg@mail.gmail.com
2022-01-24 21:13:18 +01:00
# Copyright (c) 2021-2022, PostgreSQL Global Development Group
# This test case aims to verify that server-side backups and server-side
# backup compression work properly, and it also aims to verify that
# pg_verifybackup can verify a base backup that didn't start out in plain
# format.
use strict;
use warnings;
use Config;
use File::Path qw(rmtree);
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
Server-side gzip compression. pg_basebackup's --compression option now lets you write either "client-gzip" or "server-gzip" instead of just "gzip" to specify where the compression should be performed. If you write simply "gzip" it's taken to mean "client-gzip" unless you also use --target, in which case it is interpreted to mean "server-gzip", because that's the only thing that makes any sense in that case. To make this work, the BASE_BACKUP command now takes new COMPRESSION and COMPRESSION_LEVEL options. At present, pg_basebackup cannot decompress .gz files, so server-side compression will cause a failure if (1) -Ft is not used or (2) -R is used or (3) -D- is used without --no-manifest. Along the way, I removed the information message added by commit 5c649fe153367cdab278738ee4aebbfd158e0546 which occurred if you specified no compression level and told you that the default level had been used instead. That seemed like more output than most people would want. Also along the way, this adds a check to the server for unrecognized base backup options. This repairs a bug introduced by commit 0ba281cb4bf9f5f65529dfa4c8282abb734dd454. This commit also adds some new test cases for pg_verifybackup. They take a server-side backup with and without compression, and then extract the backup if we have the OS facilities available to do so, and then run pg_verifybackup on the extracted directory. That is a good test of the functionality added by this commit and also improves test coverage for the backup target patch (commit 3500ccc39b0dadd1068a03938e4b8ff562587ccc) and for pg_verifybackup itself. Patch by me, with a bug fix by Jeevan Ladhe. The patch set of which this is a part has also had review and/or testing from Tushar Ahuja, Suraj Kharage, Dipesh Pandit, and Mark Dilger. Discussion: http://postgr.es/m/CA+Tgmoa-ST7fMLsVJduOB7Eub=2WjfpHS+QxHVEpUoinf4bOSg@mail.gmail.com
2022-01-24 21:13:18 +01:00
my $primary = PostgreSQL::Test::Cluster->new('primary');
$primary->init(allows_streaming => 1);
$primary->start;
my $backup_path = $primary->backup_dir . '/server-backup';
my $real_backup_path = PostgreSQL::Test::Utils::perl2host($backup_path);
Server-side gzip compression. pg_basebackup's --compression option now lets you write either "client-gzip" or "server-gzip" instead of just "gzip" to specify where the compression should be performed. If you write simply "gzip" it's taken to mean "client-gzip" unless you also use --target, in which case it is interpreted to mean "server-gzip", because that's the only thing that makes any sense in that case. To make this work, the BASE_BACKUP command now takes new COMPRESSION and COMPRESSION_LEVEL options. At present, pg_basebackup cannot decompress .gz files, so server-side compression will cause a failure if (1) -Ft is not used or (2) -R is used or (3) -D- is used without --no-manifest. Along the way, I removed the information message added by commit 5c649fe153367cdab278738ee4aebbfd158e0546 which occurred if you specified no compression level and told you that the default level had been used instead. That seemed like more output than most people would want. Also along the way, this adds a check to the server for unrecognized base backup options. This repairs a bug introduced by commit 0ba281cb4bf9f5f65529dfa4c8282abb734dd454. This commit also adds some new test cases for pg_verifybackup. They take a server-side backup with and without compression, and then extract the backup if we have the OS facilities available to do so, and then run pg_verifybackup on the extracted directory. That is a good test of the functionality added by this commit and also improves test coverage for the backup target patch (commit 3500ccc39b0dadd1068a03938e4b8ff562587ccc) and for pg_verifybackup itself. Patch by me, with a bug fix by Jeevan Ladhe. The patch set of which this is a part has also had review and/or testing from Tushar Ahuja, Suraj Kharage, Dipesh Pandit, and Mark Dilger. Discussion: http://postgr.es/m/CA+Tgmoa-ST7fMLsVJduOB7Eub=2WjfpHS+QxHVEpUoinf4bOSg@mail.gmail.com
2022-01-24 21:13:18 +01:00
my $extract_path = $primary->backup_dir . '/extracted-backup';
my @test_configuration = (
{
'compression_method' => 'none',
'backup_flags' => [],
'backup_archive' => 'base.tar',
'enabled' => 1
},
{
'compression_method' => 'gzip',
'backup_flags' => ['--compress', 'server-gzip'],
'backup_archive' => 'base.tar.gz',
'decompress_program' => $ENV{'GZIP_PROGRAM'},
'decompress_flags' => [ '-d' ],
'enabled' => check_pg_config("#define HAVE_LIBZ 1")
},
{
'compression_method' => 'lz4',
'backup_flags' => ['--compress', 'server-lz4'],
'backup_archive' => 'base.tar.lz4',
'decompress_program' => $ENV{'LZ4'},
'decompress_flags' => [ '-d', '-m'],
'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
Server-side gzip compression. pg_basebackup's --compression option now lets you write either "client-gzip" or "server-gzip" instead of just "gzip" to specify where the compression should be performed. If you write simply "gzip" it's taken to mean "client-gzip" unless you also use --target, in which case it is interpreted to mean "server-gzip", because that's the only thing that makes any sense in that case. To make this work, the BASE_BACKUP command now takes new COMPRESSION and COMPRESSION_LEVEL options. At present, pg_basebackup cannot decompress .gz files, so server-side compression will cause a failure if (1) -Ft is not used or (2) -R is used or (3) -D- is used without --no-manifest. Along the way, I removed the information message added by commit 5c649fe153367cdab278738ee4aebbfd158e0546 which occurred if you specified no compression level and told you that the default level had been used instead. That seemed like more output than most people would want. Also along the way, this adds a check to the server for unrecognized base backup options. This repairs a bug introduced by commit 0ba281cb4bf9f5f65529dfa4c8282abb734dd454. This commit also adds some new test cases for pg_verifybackup. They take a server-side backup with and without compression, and then extract the backup if we have the OS facilities available to do so, and then run pg_verifybackup on the extracted directory. That is a good test of the functionality added by this commit and also improves test coverage for the backup target patch (commit 3500ccc39b0dadd1068a03938e4b8ff562587ccc) and for pg_verifybackup itself. Patch by me, with a bug fix by Jeevan Ladhe. The patch set of which this is a part has also had review and/or testing from Tushar Ahuja, Suraj Kharage, Dipesh Pandit, and Mark Dilger. Discussion: http://postgr.es/m/CA+Tgmoa-ST7fMLsVJduOB7Eub=2WjfpHS+QxHVEpUoinf4bOSg@mail.gmail.com
2022-01-24 21:13:18 +01:00
}
);
for my $tc (@test_configuration)
{
my $method = $tc->{'compression_method'};
SKIP: {
skip "$method compression not supported by this build", 3
if ! $tc->{'enabled'};
skip "no decompressor available for $method", 3
if exists $tc->{'decompress_program'}
&& (!defined $tc->{'decompress_program'}
|| $tc->{'decompress_program'} eq '');
Server-side gzip compression. pg_basebackup's --compression option now lets you write either "client-gzip" or "server-gzip" instead of just "gzip" to specify where the compression should be performed. If you write simply "gzip" it's taken to mean "client-gzip" unless you also use --target, in which case it is interpreted to mean "server-gzip", because that's the only thing that makes any sense in that case. To make this work, the BASE_BACKUP command now takes new COMPRESSION and COMPRESSION_LEVEL options. At present, pg_basebackup cannot decompress .gz files, so server-side compression will cause a failure if (1) -Ft is not used or (2) -R is used or (3) -D- is used without --no-manifest. Along the way, I removed the information message added by commit 5c649fe153367cdab278738ee4aebbfd158e0546 which occurred if you specified no compression level and told you that the default level had been used instead. That seemed like more output than most people would want. Also along the way, this adds a check to the server for unrecognized base backup options. This repairs a bug introduced by commit 0ba281cb4bf9f5f65529dfa4c8282abb734dd454. This commit also adds some new test cases for pg_verifybackup. They take a server-side backup with and without compression, and then extract the backup if we have the OS facilities available to do so, and then run pg_verifybackup on the extracted directory. That is a good test of the functionality added by this commit and also improves test coverage for the backup target patch (commit 3500ccc39b0dadd1068a03938e4b8ff562587ccc) and for pg_verifybackup itself. Patch by me, with a bug fix by Jeevan Ladhe. The patch set of which this is a part has also had review and/or testing from Tushar Ahuja, Suraj Kharage, Dipesh Pandit, and Mark Dilger. Discussion: http://postgr.es/m/CA+Tgmoa-ST7fMLsVJduOB7Eub=2WjfpHS+QxHVEpUoinf4bOSg@mail.gmail.com
2022-01-24 21:13:18 +01:00
# Take a server-side backup.
my @backup = (
'pg_basebackup', '--no-sync', '-cfast', '--target',
"server:$real_backup_path", '-Xfetch'
Server-side gzip compression. pg_basebackup's --compression option now lets you write either "client-gzip" or "server-gzip" instead of just "gzip" to specify where the compression should be performed. If you write simply "gzip" it's taken to mean "client-gzip" unless you also use --target, in which case it is interpreted to mean "server-gzip", because that's the only thing that makes any sense in that case. To make this work, the BASE_BACKUP command now takes new COMPRESSION and COMPRESSION_LEVEL options. At present, pg_basebackup cannot decompress .gz files, so server-side compression will cause a failure if (1) -Ft is not used or (2) -R is used or (3) -D- is used without --no-manifest. Along the way, I removed the information message added by commit 5c649fe153367cdab278738ee4aebbfd158e0546 which occurred if you specified no compression level and told you that the default level had been used instead. That seemed like more output than most people would want. Also along the way, this adds a check to the server for unrecognized base backup options. This repairs a bug introduced by commit 0ba281cb4bf9f5f65529dfa4c8282abb734dd454. This commit also adds some new test cases for pg_verifybackup. They take a server-side backup with and without compression, and then extract the backup if we have the OS facilities available to do so, and then run pg_verifybackup on the extracted directory. That is a good test of the functionality added by this commit and also improves test coverage for the backup target patch (commit 3500ccc39b0dadd1068a03938e4b8ff562587ccc) and for pg_verifybackup itself. Patch by me, with a bug fix by Jeevan Ladhe. The patch set of which this is a part has also had review and/or testing from Tushar Ahuja, Suraj Kharage, Dipesh Pandit, and Mark Dilger. Discussion: http://postgr.es/m/CA+Tgmoa-ST7fMLsVJduOB7Eub=2WjfpHS+QxHVEpUoinf4bOSg@mail.gmail.com
2022-01-24 21:13:18 +01:00
);
push @backup, @{$tc->{'backup_flags'}};
$primary->command_ok(\@backup,
"server side backup, compression $method");
# Verify that the we got the files we expected.
my $backup_files = join(',',
sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
my $expected_backup_files = join(',',
sort ('backup_manifest', $tc->{'backup_archive'}));
is($backup_files,$expected_backup_files,
"found expected backup files, compression $method");
# Decompress.
if (exists $tc->{'decompress_program'})
{
my @decompress = ($tc->{'decompress_program'});
push @decompress, @{$tc->{'decompress_flags'}}
if $tc->{'decompress_flags'};
push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
system_or_bail(@decompress);
}
SKIP: {
my $tar = $ENV{TAR};
# don't check for a working tar here, to accomodate various odd
# cases such as AIX. If tar doesn't work the init_from_backup below
# will fail.
skip "no tar program available", 1
if (!defined $tar || $tar eq '');
# Untar.
mkdir($extract_path);
system_or_bail($tar, 'xf', $backup_path . '/base.tar',
'-C', $extract_path);
# Verify.
$primary->command_ok([ 'pg_verifybackup', '-n',
'-m', "$backup_path/backup_manifest", '-e', $extract_path ],
"verify backup, compression $method");
}
# Cleanup.
unlink($backup_path . '/backup_manifest');
unlink($backup_path . '/base.tar');
rmtree($extract_path);
}
}
done_testing();