From dcf2a0db8529b81a57d421e0fd59297d10d5f0b6 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 18 Mar 2019 10:35:01 +0900 Subject: [PATCH] Fix pg_rewind when rewinding new database with tables included This fixes an issue introduced by 266b6ac, which has added filters to exclude file patterns on the target and source data directories to reduce the number of files transferred. Filters get applied to both the target and source data files, and include pg_internal.init which is present for each database once relations are created on it. However, if the target differed from the source with at least one new database with relations, the rewind would fail due to the exclusion filters applied on the target files, causing pg_internal.init to still be present on the target database folder, while its contents should have been completely removed so as there is nothing remaining inside at the time of the folder deletion. Applying exclusion filters on the source files is fine, because this way the amount of data copied from the source to the target is reduced. And actually, not applying the filters on the target is what pg_rewind should do, because this causes such files to be automatically removed during the rewind on the target. Exclusion filters apply to paths which are removed or recreated automatically at startup, so removing all those files on the target during the rewind is a win. The existing set of TAP tests already stresses the rewind of databases, but it did not include any tables on those newly-created databases. Creating extra tables in this case is enough to reproduce the failure, so the existing tests are extended to close the gap. Reported-by: Mithun Cy Author: Michael Paquier Discussion: https://postgr.es/m/CADq3xVYt6_pO7ZzmjOqPgY9HWsL=kLd-_tNyMtdfjKqEALDyTA@mail.gmail.com Backpatch-through: 11 --- src/bin/pg_rewind/filemap.c | 13 +++++++------ src/bin/pg_rewind/t/002_databases.pl | 12 ++++++++++-- src/bin/pg_rewind/t/RewindTest.pm | 6 ++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index 222b56f58a..d70f118938 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -147,7 +147,10 @@ process_source_file(const char *path, file_type_t type, size_t newsize, Assert(map->array == NULL); - /* ignore any path matching the exclusion filters */ + /* + * Skip any files matching the exclusion filters. This has the effect to + * remove all those files on the target. + */ if (check_file_excluded(path, true)) return; @@ -334,12 +337,10 @@ process_target_file(const char *path, file_type_t type, size_t oldsize, file_entry_t *entry; /* - * Ignore any path matching the exclusion filters. This is not actually - * mandatory for target files, but this does not hurt and let's be - * consistent with the source processing. + * Do not apply any exclusion filters here. This has advantage to remove + * from the target data folder all paths which have been filtered out from + * the source data folder when processing the source files. */ - if (check_file_excluded(path, false)) - return; snprintf(localpath, sizeof(localpath), "%s/%s", datadir_target, path); if (lstat(localpath, &statbuf) < 0) diff --git a/src/bin/pg_rewind/t/002_databases.pl b/src/bin/pg_rewind/t/002_databases.pl index 6dc05720a1..0562c21549 100644 --- a/src/bin/pg_rewind/t/002_databases.pl +++ b/src/bin/pg_rewind/t/002_databases.pl @@ -15,19 +15,27 @@ sub run_test RewindTest::setup_cluster($test_mode, ['-g']); RewindTest::start_master(); - # Create a database in master. + # Create a database in master with a table. master_psql('CREATE DATABASE inmaster'); + master_psql('CREATE TABLE inmaster_tab (a int)', 'inmaster'); RewindTest::create_standby($test_mode); - # Create another database, the creation is replicated to the standby + # Create another database with another table, the creation is + # replicated to the standby. master_psql('CREATE DATABASE beforepromotion'); + master_psql('CREATE TABLE beforepromotion_tab (a int)', + 'beforepromotion'); RewindTest::promote_standby(); # Create databases in the old master and the new promoted standby. master_psql('CREATE DATABASE master_afterpromotion'); + master_psql('CREATE TABLE master_promotion_tab (a int)', + 'master_afterpromotion'); standby_psql('CREATE DATABASE standby_afterpromotion'); + standby_psql('CREATE TABLE standby_promotion_tab (a int)', + 'standby_afterpromotion'); # The clusters are now diverged. diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 60b54119e7..8469c9c2d3 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -68,18 +68,20 @@ our $node_standby; sub master_psql { my $cmd = shift; + my $dbname = shift || 'postgres'; system_or_bail 'psql', '-q', '--no-psqlrc', '-d', - $node_master->connstr('postgres'), '-c', "$cmd"; + $node_master->connstr($dbname), '-c', "$cmd"; return; } sub standby_psql { my $cmd = shift; + my $dbname = shift || 'postgres'; system_or_bail 'psql', '-q', '--no-psqlrc', '-d', - $node_standby->connstr('postgres'), '-c', "$cmd"; + $node_standby->connstr($dbname), '-c', "$cmd"; return; }