diff --git a/doc/src/sgml/ref/pg_checksums.sgml b/doc/src/sgml/ref/pg_checksums.sgml index 6a47dda683..5343a8aa7e 100644 --- a/doc/src/sgml/ref/pg_checksums.sgml +++ b/doc/src/sgml/ref/pg_checksums.sgml @@ -16,7 +16,7 @@ PostgreSQL documentation pg_checksums - verify data checksums in a PostgreSQL database cluster + enable, disable or check data checksums in a PostgreSQL database cluster @@ -36,10 +36,19 @@ PostgreSQL documentation Description - pg_checksums verifies data checksums in a - PostgreSQL cluster. The server must be shut - down cleanly before running pg_checksums. - The exit status is zero if there are no checksum errors, otherwise nonzero. + pg_checksums checks, enables or disables data + checksums in a PostgreSQL cluster. The server + must be shut down cleanly before running + pg_checksums. The exit status is zero if there + are no checksum errors when checking them, and nonzero if at least one + checksum failure is detected. If enabling or disabling checksums, the + exit status is nonzero if the operation failed. + + + + While checking or enabling checksums needs to scan or write every file in + the cluster, disabling checksums will only update the file + pg_control. @@ -60,6 +69,37 @@ PostgreSQL documentation + + + + + + Checks checksums. This is the default mode if nothing else is + specified. + + + + + + + + + + Disables checksums. + + + + + + + + + + Enables checksums. + + + + @@ -119,4 +159,33 @@ PostgreSQL documentation + + + Notes + + When disabling or enabling checksums in a replication setup of multiple + clusters, it is recommended to stop all the clusters before doing + the switch to all the clusters consistently. When using a replication + setup with tools which perform direct copies of relation file blocks + (for example ), enabling or disabling + checksums can lead to page corruptions in the shape of incorrect + checksums if the operation is not done consistently across all nodes. + Destroying all the standbys in the setup first, enabling or disabling + checksums on the primary and finally recreating the standbys from + scratch is also safe. + + + If pg_checksums is aborted or killed in + its operation while enabling or disabling checksums, the cluster + will have the same state with respect of checksums as before the + operation and pg_checksums needs to be + restarted. + + + When enabling checksums in a cluster, the operation can potentially + take a long time if the data directory is large. During this operation, + the cluster or other programs that write to the data directory must not + be started or else data loss may occur. + + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index b7ebc11017..25238cd69b 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -1,7 +1,8 @@ /*------------------------------------------------------------------------- * * pg_checksums.c - * Verifies page level checksums in an offline cluster. + * Checks, enables or disables page level checksums for an offline + * cluster * * Copyright (c) 2010-2019, PostgreSQL Global Development Group * @@ -17,14 +18,15 @@ #include #include -#include "catalog/pg_control.h" +#include "access/xlog_internal.h" #include "common/controldata_utils.h" +#include "common/file_perm.h" +#include "common/file_utils.h" #include "getopt_long.h" #include "pg_getopt.h" #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/checksum_impl.h" -#include "storage/fd.h" static int64 files = 0; @@ -35,16 +37,38 @@ static ControlFileData *ControlFile; static char *only_relfilenode = NULL; static bool verbose = false; +typedef enum +{ + PG_MODE_CHECK, + PG_MODE_DISABLE, + PG_MODE_ENABLE +} PgChecksumMode; + +/* + * Filename components. + * + * XXX: fd.h is not declared here as frontend side code is not able to + * interact with the backend-side definitions for the various fsync + * wrappers. + */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + +static PgChecksumMode mode = PG_MODE_CHECK; + static const char *progname; static void usage(void) { - printf(_("%s verifies data checksums in a PostgreSQL database cluster.\n\n"), progname); + printf(_("%s enables, disables or verifies data checksums in a PostgreSQL database cluster.\n\n"), progname); printf(_("Usage:\n")); printf(_(" %s [OPTION]... [DATADIR]\n"), progname); printf(_("\nOptions:\n")); printf(_(" [-D, --pgdata=]DATADIR data directory\n")); + printf(_(" -c, --check check data checksums (default)\n")); + printf(_(" -d, --disable disable data checksums\n")); + printf(_(" -e, --enable enable data checksums\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -r RELFILENODE check only relation with specified relfilenode\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -90,8 +114,14 @@ scan_file(const char *fn, BlockNumber segmentno) PageHeader header = (PageHeader) buf.data; int f; BlockNumber blockno; + int flags; + + Assert(mode == PG_MODE_ENABLE || + mode == PG_MODE_CHECK); + + flags = (mode == PG_MODE_ENABLE) ? O_RDWR : O_RDONLY; + f = open(fn, PG_BINARY | flags, 0); - f = open(fn, O_RDONLY | PG_BINARY, 0); if (f < 0) { fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), @@ -121,18 +151,47 @@ scan_file(const char *fn, BlockNumber segmentno) continue; csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE); - if (csum != header->pd_checksum) + if (mode == PG_MODE_CHECK) { - if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION) - fprintf(stderr, _("%s: checksum verification failed in file \"%s\", block %u: calculated checksum %X but block contains %X\n"), - progname, fn, blockno, csum, header->pd_checksum); - badblocks++; + if (csum != header->pd_checksum) + { + if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION) + fprintf(stderr, _("%s: checksum verification failed in file \"%s\", block %u: calculated checksum %X but block contains %X\n"), + progname, fn, blockno, csum, header->pd_checksum); + badblocks++; + } + } + else if (mode == PG_MODE_ENABLE) + { + /* Set checksum in page header */ + header->pd_checksum = csum; + + /* Seek back to beginning of block */ + if (lseek(f, -BLCKSZ, SEEK_CUR) < 0) + { + fprintf(stderr, _("%s: seek failed for block %d in file \"%s\": %s\n"), progname, blockno, fn, strerror(errno)); + exit(1); + } + + /* Write block with checksum */ + if (write(f, buf.data, BLCKSZ) != BLCKSZ) + { + fprintf(stderr, _("%s: could not update checksum of block %d in file \"%s\": %s\n"), + progname, blockno, fn, strerror(errno)); + exit(1); + } } } if (verbose) - fprintf(stderr, - _("%s: checksums verified in file \"%s\"\n"), progname, fn); + { + if (mode == PG_MODE_CHECK) + fprintf(stderr, + _("%s: checksums verified in file \"%s\"\n"), progname, fn); + if (mode == PG_MODE_ENABLE) + fprintf(stderr, + _("%s: checksums enabled in file \"%s\"\n"), progname, fn); + } close(f); } @@ -234,7 +293,10 @@ int main(int argc, char *argv[]) { static struct option long_options[] = { + {"check", no_argument, NULL, 'c'}, {"pgdata", required_argument, NULL, 'D'}, + {"disable", no_argument, NULL, 'd'}, + {"enable", no_argument, NULL, 'e'}, {"verbose", no_argument, NULL, 'v'}, {NULL, 0, NULL, 0} }; @@ -262,10 +324,19 @@ main(int argc, char *argv[]) } } - while ((c = getopt_long(argc, argv, "D:r:v", long_options, &option_index)) != -1) + while ((c = getopt_long(argc, argv, "cD:der:v", long_options, &option_index)) != -1) { switch (c) { + case 'c': + mode = PG_MODE_CHECK; + break; + case 'd': + mode = PG_MODE_DISABLE; + break; + case 'e': + mode = PG_MODE_ENABLE; + break; case 'v': verbose = true; break; @@ -312,6 +383,15 @@ main(int argc, char *argv[]) exit(1); } + /* Relfilenode checking only works in --check mode */ + if (mode != PG_MODE_CHECK && only_relfilenode) + { + fprintf(stderr, _("%s: relfilenode option only possible with --check\n"), progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } + /* Check if cluster is running */ ControlFile = get_controlfile(DataDir, progname, &crc_ok); if (!crc_ok) @@ -339,29 +419,72 @@ main(int argc, char *argv[]) if (ControlFile->state != DB_SHUTDOWNED && ControlFile->state != DB_SHUTDOWNED_IN_RECOVERY) { - fprintf(stderr, _("%s: cluster must be shut down to verify checksums\n"), progname); + fprintf(stderr, _("%s: cluster must be shut down\n"), progname); exit(1); } - if (ControlFile->data_checksum_version == 0) + if (ControlFile->data_checksum_version == 0 && + mode == PG_MODE_CHECK) { fprintf(stderr, _("%s: data checksums are not enabled in cluster\n"), progname); exit(1); } + if (ControlFile->data_checksum_version == 0 && + mode == PG_MODE_DISABLE) + { + fprintf(stderr, _("%s: data checksums are already disabled in cluster.\n"), progname); + exit(1); + } + if (ControlFile->data_checksum_version > 0 && + mode == PG_MODE_ENABLE) + { + fprintf(stderr, _("%s: data checksums are already enabled in cluster.\n"), progname); + exit(1); + } - /* Scan all files */ - scan_directory(DataDir, "global"); - scan_directory(DataDir, "base"); - scan_directory(DataDir, "pg_tblspc"); + /* Operate on all files if checking or enabling checksums */ + if (mode == PG_MODE_CHECK || mode == PG_MODE_ENABLE) + { + scan_directory(DataDir, "global"); + scan_directory(DataDir, "base"); + scan_directory(DataDir, "pg_tblspc"); - printf(_("Checksum scan completed\n")); - printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version); - printf(_("Files scanned: %s\n"), psprintf(INT64_FORMAT, files)); - printf(_("Blocks scanned: %s\n"), psprintf(INT64_FORMAT, blocks)); - printf(_("Bad checksums: %s\n"), psprintf(INT64_FORMAT, badblocks)); + printf(_("Checksum operation completed\n")); + printf(_("Files scanned: %s\n"), psprintf(INT64_FORMAT, files)); + printf(_("Blocks scanned: %s\n"), psprintf(INT64_FORMAT, blocks)); + if (mode == PG_MODE_CHECK) + { + printf(_("Bad checksums: %s\n"), psprintf(INT64_FORMAT, badblocks)); + printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version); - if (badblocks > 0) - return 1; + if (badblocks > 0) + exit(1); + } + } + + /* + * Finally make the data durable on disk if enabling or disabling + * checksums. Flush first the data directory for safety, and then update + * the control file to keep the switch consistent. + */ + if (mode == PG_MODE_ENABLE || mode == PG_MODE_DISABLE) + { + ControlFile->data_checksum_version = + (mode == PG_MODE_ENABLE) ? PG_DATA_CHECKSUM_VERSION : 0; + + printf(_("Syncing data directory\n")); + fsync_pgdata(DataDir, progname, PG_VERSION_NUM); + + printf(_("Updating control file\n")); + update_controlfile(DataDir, progname, ControlFile, true); + + if (verbose) + printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version); + if (mode == PG_MODE_ENABLE) + printf(_("Checksums enabled in cluster\n")); + else + printf(_("Checksums disabled in cluster\n")); + } return 0; } diff --git a/src/bin/pg_checksums/t/002_actions.pl b/src/bin/pg_checksums/t/002_actions.pl index 97284e8930..3ab18a6b89 100644 --- a/src/bin/pg_checksums/t/002_actions.pl +++ b/src/bin/pg_checksums/t/002_actions.pl @@ -5,7 +5,7 @@ use strict; use warnings; use PostgresNode; use TestLib; -use Test::More tests => 45; +use Test::More tests => 62; # Utility routine to create and check a table with corrupted checksums @@ -38,8 +38,8 @@ sub check_relation_corruption # Checksums are correct for single relfilenode as the table is not # corrupted yet. - command_ok(['pg_checksums', '-D', $pgdata, - '-r', $relfilenode_corrupted], + command_ok(['pg_checksums', '--check', '-D', $pgdata, '-r', + $relfilenode_corrupted], "succeeds for single relfilenode on tablespace $tablespace with offline cluster"); # Time to create some corruption @@ -49,15 +49,15 @@ sub check_relation_corruption close $file; # Checksum checks on single relfilenode fail - $node->command_checks_all([ 'pg_checksums', '-D', $pgdata, '-r', - $relfilenode_corrupted], + $node->command_checks_all([ 'pg_checksums', '--check', '-D', $pgdata, + '-r', $relfilenode_corrupted], 1, [qr/Bad checksums:.*1/], [qr/checksum verification failed/], "fails with corrupted data for single relfilenode on tablespace $tablespace"); # Global checksum checks fail as well - $node->command_checks_all([ 'pg_checksums', '-D', $pgdata], + $node->command_checks_all([ 'pg_checksums', '--check', '-D', $pgdata], 1, [qr/Bad checksums:.*1/], [qr/checksum verification failed/], @@ -67,22 +67,22 @@ sub check_relation_corruption $node->start; $node->safe_psql('postgres', "DROP TABLE $table;"); $node->stop; - $node->command_ok(['pg_checksums', '-D', $pgdata], + $node->command_ok(['pg_checksums', '--check', '-D', $pgdata], "succeeds again after table drop on tablespace $tablespace"); $node->start; return; } -# Initialize node with checksums enabled. +# Initialize node with checksums disabled. my $node = get_new_node('node_checksum'); -$node->init(extra => ['--data-checksums']); +$node->init(); my $pgdata = $node->data_dir; -# Control file should know that checksums are enabled. +# Control file should know that checksums are disabled. command_like(['pg_controldata', $pgdata], - qr/Data page checksum version:.*1/, - 'checksums enabled in control file'); + qr/Data page checksum version:.*0/, + 'checksums disabled in control file'); # These are correct but empty files, so they should pass through. append_to_file "$pgdata/global/99999", ""; @@ -100,13 +100,59 @@ append_to_file "$pgdata/global/pgsql_tmp_123", "foo"; mkdir "$pgdata/global/pgsql_tmp"; append_to_file "$pgdata/global/pgsql_tmp/1.1", "foo"; +# Enable checksums. +command_ok(['pg_checksums', '--enable', '-D', $pgdata], + "checksums successfully enabled in cluster"); + +# Successive attempt to enable checksums fails. +command_fails(['pg_checksums', '--enable', '-D', $pgdata], + "enabling checksums fails if already enabled"); + +# Control file should know that checksums are enabled. +command_like(['pg_controldata', $pgdata], + qr/Data page checksum version:.*1/, + 'checksums enabled in control file'); + +# Disable checksums again. +command_ok(['pg_checksums', '--disable', '-D', $pgdata], + "checksums successfully disabled in cluster"); + +# Successive attempt to disable checksums fails. +command_fails(['pg_checksums', '--disable', '-D', $pgdata], + "disabling checksums fails if already disabled"); + +# Control file should know that checksums are disabled. +command_like(['pg_controldata', $pgdata], + qr/Data page checksum version:.*0/, + 'checksums disabled in control file'); + +# Enable checksums again for follow-up tests. +command_ok(['pg_checksums', '--enable', '-D', $pgdata], + "checksums successfully enabled in cluster"); + +# Control file should know that checksums are enabled. +command_like(['pg_controldata', $pgdata], + qr/Data page checksum version:.*1/, + 'checksums enabled in control file'); + # Checksums pass on a newly-created cluster -command_ok(['pg_checksums', '-D', $pgdata], +command_ok(['pg_checksums', '--check', '-D', $pgdata], "succeeds with offline cluster"); +# Checksums are verified if no other arguments are specified +command_ok(['pg_checksums', '-D', $pgdata], + "verifies checksums as default action"); + +# Specific relation files cannot be requested when action is --disable +# or --enable. +command_fails(['pg_checksums', '--disable', '-r', '1234', '-D', $pgdata], + "fails when relfilenodes are requested and action is --disable"); +command_fails(['pg_checksums', '--enable', '-r', '1234', '-D', $pgdata], + "fails when relfilenodes are requested and action is --enable"); + # Checks cannot happen with an online cluster $node->start; -command_fails(['pg_checksums', '-D', $pgdata], +command_fails(['pg_checksums', '--check', '-D', $pgdata], "fails with online cluster"); # Check corruption of table on default tablespace. @@ -133,7 +179,7 @@ sub fail_corrupt my $file_name = "$pgdata/global/$file"; append_to_file $file_name, "foo"; - $node->command_checks_all([ 'pg_checksums', '-D', $pgdata], + $node->command_checks_all([ 'pg_checksums', '--check', '-D', $pgdata], 1, [qr/^$/], [qr/could not read block 0 in file.*$file\":/], diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index b301bce4b1..195b146974 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1710,6 +1710,7 @@ PgBenchExprType PgBenchFunction PgBenchValue PgBenchValueType +PgChecksumMode PgFdwAnalyzeState PgFdwDirectModifyState PgFdwModifyState