rdelaage
/
postgresql
mirror of https://git.postgresql.org/git/postgresql.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Try again to add test coverage for pg_combinebackup w/tablespaces. 

My previous attempt to add this had to be reverted in commit
82023d47de. I've revised the problematic
code a bit; hopefully it is OK now.

Discussion: http://postgr.es/m/CA+Tgmobiv1QJR5PEJoDKeZDrJHZFRmi4XmWOqufN49DJj-3e2g@mail.gmail.com
This commit is contained in:
Robert Haas 2024-04-23 16:33:19 -04:00
parent ba3e6e2bca
commit 89ad3e1316
3 changed files with 126 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/bin/pg_basebackup/t/010_pg_basebackup.pl
View File

@ -407,25 +407,12 @@ SKIP:
my $node2 = PostgreSQL::Test::Cluster->new('replica');
# Recover main data directory
$node2->init_from_backup($node, 'tarbackup2', tar_program => $tar);
# Recover tablespace into a new directory (not where it was!)
my $repTsDir = "$tempdir/tblspc1replica";
my $realRepTsDir = "$real_sys_tempdir/tblspc1replica";
mkdir $repTsDir;
PostgreSQL::Test::Utils::system_or_bail($tar, 'xf', $tblspc_tars[0],
'-C', $repTsDir);
# Update tablespace map to point to new directory.
# XXX Ideally pg_basebackup would handle this.
# Recover the backup
$tblspc_tars[0] =~ m|/([0-9]*)\.tar$|;
my $tblspcoid = $1;
my $escapedRepTsDir = $realRepTsDir;
$escapedRepTsDir =~ s/\\/\\\\/g;
open my $mapfile, '>', $node2->data_dir . '/tablespace_map' or die $!;
print $mapfile "$tblspcoid $escapedRepTsDir\n";
close $mapfile;
my $realRepTsDir = "$real_sys_tempdir/tblspc1replica";
$node2->init_from_backup($node, 'tarbackup2', tar_program => $tar,
'tablespace_map' => { $tblspcoid => $realRepTsDir });
$node2->start;
my $result = $node2->safe_psql('postgres', 'SELECT * FROM test1');

37
src/bin/pg_combinebackup/t/002_compare_backups.pl
View File

@ -7,11 +7,15 @@ use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
# Set up a new database instance.
my $primary = PostgreSQL::Test::Cluster->new('primary');
$primary->init(has_archiving => 1, allows_streaming => 1);
$primary->append_conf('postgresql.conf', 'summarize_wal = on');
$primary->start;
my $tsprimary = $tempdir . '/ts';
mkdir($tsprimary) || die "mkdir $tsprimary: $!";
# Create some test tables, each containing one row of data, plus a whole
# extra database.
@ -29,24 +33,42 @@ INSERT INTO will_get_dropped VALUES (1, 'initial test row');
CREATE TABLE will_get_rewritten (a int, b text);
INSERT INTO will_get_rewritten VALUES (1, 'initial test row');
CREATE DATABASE db_will_get_dropped;
CREATE TABLESPACE ts1 LOCATION '$tsprimary';
CREATE TABLE will_not_change_in_ts (a int, b text) TABLESPACE ts1;
INSERT INTO will_not_change_in_ts VALUES (1, 'initial test row');
CREATE TABLE will_change_in_ts (a int, b text) TABLESPACE ts1;
INSERT INTO will_change_in_ts VALUES (1, 'initial test row');
CREATE TABLE will_get_dropped_in_ts (a int, b text);
INSERT INTO will_get_dropped_in_ts VALUES (1, 'initial test row');
EOM
# Read list of tablespace OIDs. There should be just one.
my @tsoids = grep { /^\d+/ } slurp_dir($primary->data_dir . '/pg_tblspc');
is(0+@tsoids, 1, "exactly one user-defined tablespace");
my $tsoid = $tsoids[0];
# Take a full backup.
my $backup1path = $primary->backup_dir . '/backup1';
my $tsbackup1path = $tempdir . '/ts1backup';
mkdir($tsbackup1path) || die "mkdir $tsbackup1path: $!";
$primary->command_ok(
[ 'pg_basebackup', '-D', $backup1path, '--no-sync', '-cfast' ],
"full backup");
[ 'pg_basebackup', '-D', $backup1path, '--no-sync', '-cfast',
"-T${tsprimary}=${tsbackup1path}" ], "full backup");
# Now make some database changes.
$primary->safe_psql('postgres', <<EOM);
UPDATE will_change SET b = 'modified value' WHERE a = 1;
UPDATE will_change_in_ts SET b = 'modified value' WHERE a = 1;
INSERT INTO will_grow
SELECT g, 'additional row' FROM generate_series(2, 5000) g;
TRUNCATE will_shrink;
VACUUM will_get_vacuumed;
DROP TABLE will_get_dropped;
DROP TABLE will_get_dropped_in_ts;
CREATE TABLE newly_created (a int, b text);
INSERT INTO newly_created VALUES (1, 'row for new table');
CREATE TABLE newly_created_in_ts (a int, b text) TABLESPACE ts1;
INSERT INTO newly_created_in_ts VALUES (1, 'row for new table');
VACUUM FULL will_get_rewritten;
DROP DATABASE db_will_get_dropped;
CREATE DATABASE db_newly_created;
@ -54,8 +76,11 @@ EOM
# Take an incremental backup.
my $backup2path = $primary->backup_dir . '/backup2';
my $tsbackup2path = $tempdir . '/tsbackup2';
mkdir($tsbackup2path) || die "mkdir $tsbackup2path: $!";
$primary->command_ok(
[ 'pg_basebackup', '-D', $backup2path, '--no-sync', '-cfast',
"-T${tsprimary}=${tsbackup2path}",
'--incremental', $backup1path . '/backup_manifest' ],
"incremental backup");
@ -78,9 +103,11 @@ $primary->poll_query_until('postgres', $archive_wait_query)
# Perform PITR from the full backup. Disable archive_mode so that the archive
# doesn't find out about the new timeline; that way, the later PITR below will
# choose the same timeline.
my $tspitr1path = $tempdir . '/tspitr1';
my $pitr1 = PostgreSQL::Test::Cluster->new('pitr1');
$pitr1->init_from_backup($primary, 'backup1',
standby => 1, has_restoring => 1);
standby => 1, has_restoring => 1,
tablespace_map => { $tsoid => $tspitr1path });
$pitr1->append_conf('postgresql.conf', qq{
recovery_target_lsn = '$lsn'
recovery_target_action = 'promote'
@ -90,10 +117,12 @@ $pitr1->start();
# Perform PITR to the same LSN from the incremental backup. Use the same
# basic configuration as before.
my $tspitr2path = $tempdir . '/tspitr2';
my $pitr2 = PostgreSQL::Test::Cluster->new('pitr2');
$pitr2->init_from_backup($primary, 'backup2',
standby => 1, has_restoring => 1,
combine_with_prior => [ 'backup1' ]);
combine_with_prior => [ 'backup1' ],
tablespace_map => { $tsbackup2path => $tspitr2path });
$pitr2->append_conf('postgresql.conf', qq{
recovery_target_lsn = '$lsn'
recovery_target_action = 'promote'

96
src/test/perl/PostgreSQL/Test/Cluster.pm
View File

@ -777,7 +777,7 @@ sub backup_fs_cold
=pod
=item $node->init_from_backup(root_node, backup_name)
=item $node->init_from_backup(root_node, backup_name, %params)
Initialize a node from a backup, which may come from this node or a different
node. root_node must be a PostgreSQL::Test::Cluster reference, backup_name the string name
@ -787,8 +787,13 @@ Does not start the node after initializing it.
By default, the backup is assumed to be plain format. To restore from
a tar-format backup, pass the name of the tar program to use in the
keyword parameter tar_program. Note that tablespace tar files aren't
handled here.
keyword parameter tar_program.
If there are tablespace present in the backup, include tablespace_map as
a keyword parameter whose values is a hash. When combine_with_prior is used,
the hash keys are the tablespace pathnames used in the backup; otherwise,
they are tablespace OIDs. In either case, the values are the tablespace
pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
@ -843,12 +848,20 @@ sub init_from_backup
}
local %ENV = $self->_get_env();
PostgreSQL::Test::Utils::system_or_bail('pg_combinebackup', '-d',
@prior_backup_path, $backup_path, '-o', $data_path);
my @combineargs = ('pg_combinebackup', '-d');
if (exists $params{tablespace_map})
{
while (my ($olddir, $newdir) = each %{$params{tablespace_map}})
{
push @combineargs, "-T$olddir=$newdir";
}
}
push @combineargs, @prior_backup_path, $backup_path, '-o', $data_path;
PostgreSQL::Test::Utils::system_or_bail(@combineargs);
}
elsif (defined $params{tar_program})
{
mkdir($data_path);
mkdir($data_path) || die "mkdir $data_path: $!";
PostgreSQL::Test::Utils::system_or_bail($params{tar_program}, 'xf',
$backup_path . '/base.tar',
'-C', $data_path);
@ -856,11 +869,80 @@ sub init_from_backup
$params{tar_program}, 'xf',
$backup_path . '/pg_wal.tar', '-C',
$data_path . '/pg_wal');
# We need to generate a tablespace_map file.
open(my $tsmap, ">", "$data_path/tablespace_map")
|| die "$data_path/tablespace_map: $!";
# Extract tarfiles and add tablespace_map entries
my @tstars = grep { /^\d+.tar/ }
PostgreSQL::Test::Utils::slurp_dir($backup_path);
for my $tstar (@tstars)
{
my $tsoid = $tstar;
$tsoid =~ s/\.tar$//;
die "no tablespace mapping for $tstar"
if !exists $params{tablespace_map} ||
!exists $params{tablespace_map}{$tsoid};
my $newdir = $params{tablespace_map}{$tsoid};
mkdir($newdir) || die "mkdir $newdir: $!";
PostgreSQL::Test::Utils::system_or_bail($params{tar_program}, 'xf',
$backup_path . '/' . $tstar, '-C', $newdir);
my $escaped_newdir = $newdir;
$escaped_newdir =~ s/\\/\\\\/g;
print $tsmap "$tsoid $escaped_newdir\n";
}
# Close tablespace_map.
close($tsmap);
}
else
{
my @tsoids;
rmdir($data_path);
PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
# Copy the main backup. If we see a tablespace directory for which we
# have a tablespace mapping, skip it, but remember that we saw it.
PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path,
'filterfn' => sub {
my ($path) = @_;
if ($path =~ /^pg_tblspc\/(\d+)$/ &&
exists $params{tablespace_map}{$1})
{
push @tsoids, $1;
return 0;
}
return 1;
});
if (@tsoids > 0)
{
# We need to generate a tablespace_map file.
open(my $tsmap, ">", "$data_path/tablespace_map")
|| die "$data_path/tablespace_map: $!";
# Now use the list of tablespace links to copy each tablespace.
for my $tsoid (@tsoids)
{
die "no tablespace mapping for $tsoid"
if !exists $params{tablespace_map} ||
!exists $params{tablespace_map}{$tsoid};
my $olddir = $backup_path . '/pg_tblspc/' . $tsoid;
my $newdir = $params{tablespace_map}{$tsoid};
PostgreSQL::Test::RecursiveCopy::copypath($olddir, $newdir);
my $escaped_newdir = $newdir;
$escaped_newdir =~ s/\\/\\\\/g;
print $tsmap "$tsoid $escaped_newdir\n";
}
# Close tablespace_map.
close($tsmap);
}
}
chmod(0700, $data_path) or die $!;