diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index fda4690eab..4a42999b18 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -219,6 +219,7 @@ Complete list of usable sgml source files in this directory. + diff --git a/doc/src/sgml/ref/pg_walsummary.sgml b/doc/src/sgml/ref/pg_walsummary.sgml new file mode 100644 index 0000000000..93e265ead7 --- /dev/null +++ b/doc/src/sgml/ref/pg_walsummary.sgml @@ -0,0 +1,122 @@ + + + + + pg_walsummary + + + + pg_walsummary + 1 + Application + + + + pg_walsummary + print contents of WAL summary files + + + + + pg_walsummary + option + file + + + + + Description + + pg_walsummary is used to print the contents of + WAL summary files. These binary files are found with the + pg_wal/summaries subdirectory of the data directory, + and can be converted to text using this tool. This is not ordinarily + necessary, since WAL summary files primarily exist to support + incremental backup, + but it may be useful for debugging purposes. + + + + A WAL summary file is indexed by tablespace OID, relation OID, and relation + fork. For each relation fork, it stores the list of blocks that were + modified by WAL within the range summarized in the file. It can also + store a "limit block," which is 0 if the relation fork was created or + truncated within the relevant WAL range, and otherwise the shortest length + to which the relation fork was truncated. If the relation fork was not + created, deleted, or truncated within the relevant WAL range, the limit + block is undefined or infinite and will not be printed by this tool. + + + + + Options + + + + + + + + + By default, pg_walsummary prints one line of output + for each range of one or more consecutive modified blocks. This can + make the output a lot briefer, since a relation where all blocks from + 0 through 999 were modified will produce only one line of output rather + than 1000 separate lines. This option requests a separate line of + output for every modified block. + + + + + + + + + + Do not print any output, except for errors. This can be useful + when you want to know whether a WAL summary file can be successfully + parsed but don't care about the contents. + + + + + + + + + + Shows help about pg_walsummary command line + arguments, and exits. + + + + + + + + + + + Environment + + + The environment variable PG_COLOR specifies whether to use + color in diagnostic messages. Possible values are + always, auto and + never. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index a07d2b5e01..aa94f6adf6 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -289,6 +289,7 @@ &pgtesttiming; &pgupgrade; &pgwaldump; + &pgwalsummary; &postgres; diff --git a/src/bin/Makefile b/src/bin/Makefile index 117ffdab6a..fc789da17b 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -31,6 +31,7 @@ SUBDIRS = \ pg_upgrade \ pg_verifybackup \ pg_waldump \ + pg_walsummary \ pgbench \ psql \ scripts diff --git a/src/bin/meson.build b/src/bin/meson.build index f292752c5c..aa60ebaa30 100644 --- a/src/bin/meson.build +++ b/src/bin/meson.build @@ -17,6 +17,7 @@ subdir('pg_test_timing') subdir('pg_upgrade') subdir('pg_verifybackup') subdir('pg_waldump') +subdir('pg_walsummary') subdir('pgbench') subdir('pgevent') subdir('psql') diff --git a/src/bin/pg_walsummary/.gitignore b/src/bin/pg_walsummary/.gitignore new file mode 100644 index 0000000000..d71ec192fa --- /dev/null +++ b/src/bin/pg_walsummary/.gitignore @@ -0,0 +1 @@ +pg_walsummary diff --git a/src/bin/pg_walsummary/Makefile b/src/bin/pg_walsummary/Makefile new file mode 100644 index 0000000000..2c24bc6db5 --- /dev/null +++ b/src/bin/pg_walsummary/Makefile @@ -0,0 +1,48 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/bin/pg_walsummary +# +# Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/bin/pg_walsummary/Makefile +# +#------------------------------------------------------------------------- + +PGFILEDESC = "pg_walsummary - print contents of WAL summary files" +PGAPPICON=win32 + +subdir = src/bin/pg_walsummary +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) +LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils + +OBJS = \ + $(WIN32RES) \ + pg_walsummary.o + +all: pg_walsummary + +pg_walsummary: $(OBJS) | submake-libpgport submake-libpgfeutils + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) + + +install: all installdirs + $(INSTALL_PROGRAM) pg_walsummary$(X) '$(DESTDIR)$(bindir)/pg_walsummary$(X)' + +installdirs: + $(MKDIR_P) '$(DESTDIR)$(bindir)' + +uninstall: + rm -f '$(DESTDIR)$(bindir)/pg_walsummary$(X)' + +clean distclean maintainer-clean: + rm -f pg_walsummary$(X) $(OBJS) + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) diff --git a/src/bin/pg_walsummary/meson.build b/src/bin/pg_walsummary/meson.build new file mode 100644 index 0000000000..f886a4cb36 --- /dev/null +++ b/src/bin/pg_walsummary/meson.build @@ -0,0 +1,30 @@ +# Copyright (c) 2022-2023, PostgreSQL Global Development Group + +pg_walsummary_sources = files( + 'pg_walsummary.c', +) + +if host_system == 'windows' + pg_walsummary_sources += rc_bin_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'pg_walsummary', + '--FILEDESC', 'pg_walsummary - print contents of WAL summary files',]) +endif + +pg_walsummary = executable('pg_walsummary', + pg_walsummary_sources, + dependencies: [frontend_code], + kwargs: default_bin_args, +) +bin_targets += pg_walsummary + +tests += { + 'name': 'pg_walsummary', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'tap': { + 'tests': [ + 't/001_basic.pl', + 't/002_blocks.pl', + ], + } +} diff --git a/src/bin/pg_walsummary/nls.mk b/src/bin/pg_walsummary/nls.mk new file mode 100644 index 0000000000..f411dcfe9e --- /dev/null +++ b/src/bin/pg_walsummary/nls.mk @@ -0,0 +1,6 @@ +# src/bin/pg_combinebackup/nls.mk +CATALOG_NAME = pg_walsummary +GETTEXT_FILES = $(FRONTEND_COMMON_GETTEXT_FILES) \ + pg_walsummary.c +GETTEXT_TRIGGERS = $(FRONTEND_COMMON_GETTEXT_TRIGGERS) +GETTEXT_FLAGS = $(FRONTEND_COMMON_GETTEXT_FLAGS) diff --git a/src/bin/pg_walsummary/pg_walsummary.c b/src/bin/pg_walsummary/pg_walsummary.c new file mode 100644 index 0000000000..0c0225eeb8 --- /dev/null +++ b/src/bin/pg_walsummary/pg_walsummary.c @@ -0,0 +1,280 @@ +/*------------------------------------------------------------------------- + * + * pg_walsummary.c + * Prints the contents of WAL summary files. + * + * Copyright (c) 2017-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_walsummary/pg_walsummary.c + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include +#include + +#include "common/blkreftable.h" +#include "common/logging.h" +#include "fe_utils/option_utils.h" +#include "lib/stringinfo.h" +#include "getopt_long.h" + +typedef struct ws_options +{ + bool individual; + bool quiet; +} ws_options; + +typedef struct ws_file_info +{ + int fd; + char *filename; +} ws_file_info; + +static BlockNumber *block_buffer = NULL; +static unsigned block_buffer_size = 512; /* Initial size. */ + +static void dump_one_relation(ws_options *opt, RelFileLocator *rlocator, + ForkNumber forknum, BlockNumber limit_block, + BlockRefTableReader *reader); +static void help(const char *progname); +static int compare_block_numbers(const void *a, const void *b); +static int walsummary_read_callback(void *callback_arg, void *data, + int length); +static void walsummary_error_callback(void *callback_arg, char *fmt,...) pg_attribute_printf(2, 3); + +/* + * Main program. + */ +int +main(int argc, char *argv[]) +{ + static struct option long_options[] = { + {"individual", no_argument, NULL, 'i'}, + {"quiet", no_argument, NULL, 'q'}, + {NULL, 0, NULL, 0} + }; + + const char *progname; + int optindex; + int c; + ws_options opt; + + memset(&opt, 0, sizeof(ws_options)); + + pg_logging_init(argv[0]); + progname = get_progname(argv[0]); + handle_help_version_opts(argc, argv, progname, help); + + /* process command-line options */ + while ((c = getopt_long(argc, argv, "f:iqw:", + long_options, &optindex)) != -1) + { + switch (c) + { + case 'i': + opt.individual = true; + break; + case 'q': + opt.quiet = true; + break; + default: + /* getopt_long already emitted a complaint */ + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + } + + if (optind >= argc) + { + pg_log_error("%s: no input files specified", progname); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + + while (optind < argc) + { + ws_file_info ws; + BlockRefTableReader *reader; + RelFileLocator rlocator; + ForkNumber forknum; + BlockNumber limit_block; + + ws.filename = argv[optind++]; + if ((ws.fd = open(ws.filename, O_RDONLY | PG_BINARY, 0)) < 0) + pg_fatal("could not open file \"%s\": %m", ws.filename); + + reader = CreateBlockRefTableReader(walsummary_read_callback, &ws, + ws.filename, + walsummary_error_callback, NULL); + while (BlockRefTableReaderNextRelation(reader, &rlocator, &forknum, + &limit_block)) + dump_one_relation(&opt, &rlocator, forknum, limit_block, reader); + + DestroyBlockRefTableReader(reader); + close(ws.fd); + } + + exit(0); +} + +/* + * Dump details for one relation. + */ +static void +dump_one_relation(ws_options *opt, RelFileLocator *rlocator, + ForkNumber forknum, BlockNumber limit_block, + BlockRefTableReader *reader) +{ + unsigned i = 0; + unsigned nblocks; + BlockNumber startblock = InvalidBlockNumber; + BlockNumber endblock = InvalidBlockNumber; + + /* Dump limit block, if any. */ + if (limit_block != InvalidBlockNumber) + printf("TS %u, DB %u, REL %u, FORK %s: limit %u\n", + rlocator->spcOid, rlocator->dbOid, rlocator->relNumber, + forkNames[forknum], limit_block); + + /* If we haven't allocated a block buffer yet, do that now. */ + if (block_buffer == NULL) + block_buffer = palloc_array(BlockNumber, block_buffer_size); + + /* Try to fill the block buffer. */ + nblocks = BlockRefTableReaderGetBlocks(reader, + block_buffer, + block_buffer_size); + + /* If we filled the block buffer completely, we must enlarge it. */ + while (nblocks >= block_buffer_size) + { + unsigned new_size; + + /* Double the size, being careful about overflow. */ + new_size = block_buffer_size * 2; + if (new_size < block_buffer_size) + new_size = PG_UINT32_MAX; + block_buffer = repalloc_array(block_buffer, BlockNumber, new_size); + + /* Try to fill the newly-allocated space. */ + nblocks += + BlockRefTableReaderGetBlocks(reader, + block_buffer + block_buffer_size, + new_size - block_buffer_size); + + /* Save the new size for later calls. */ + block_buffer_size = new_size; + } + + /* If we don't need to produce any output, skip the rest of this. */ + if (opt->quiet) + return; + + /* + * Sort the returned block numbers. If the block reference table was using + * the bitmap representation for a given chunk, the block numbers in that + * chunk will already be sorted, but when the array-of-offsets + * representation is used, we can receive block numbers here out of order. + */ + qsort(block_buffer, nblocks, sizeof(BlockNumber), compare_block_numbers); + + /* Dump block references. */ + while (i < nblocks) + { + /* + * Find the next range of blocks to print, but if --individual was + * specified, then consider each block a separate range. + */ + startblock = endblock = block_buffer[i++]; + if (!opt->individual) + { + while (i < nblocks && block_buffer[i] == endblock + 1) + { + endblock++; + i++; + } + } + + /* Format this range of block numbers as a string. */ + if (startblock == endblock) + printf("TS %u, DB %u, REL %u, FORK %s: block %u\n", + rlocator->spcOid, rlocator->dbOid, rlocator->relNumber, + forkNames[forknum], startblock); + else + printf("TS %u, DB %u, REL %u, FORK %s: blocks %u..%u\n", + rlocator->spcOid, rlocator->dbOid, rlocator->relNumber, + forkNames[forknum], startblock, endblock); + } +} + +/* + * Quicksort comparator for block numbers. + */ +static int +compare_block_numbers(const void *a, const void *b) +{ + BlockNumber aa = *(BlockNumber *) a; + BlockNumber bb = *(BlockNumber *) b; + + if (aa > bb) + return 1; + else if (aa == bb) + return 0; + else + return -1; +} + +/* + * Error callback. + */ +void +walsummary_error_callback(void *callback_arg, char *fmt,...) +{ + va_list ap; + + va_start(ap, fmt); + pg_log_generic_v(PG_LOG_ERROR, PG_LOG_PRIMARY, fmt, ap); + va_end(ap); + + exit(1); +} + +/* + * Read callback. + */ +int +walsummary_read_callback(void *callback_arg, void *data, int length) +{ + ws_file_info *ws = callback_arg; + int rc; + + if ((rc = read(ws->fd, data, length)) < 0) + pg_fatal("could not read file \"%s\": %m", ws->filename); + + return rc; +} + +/* + * help + * + * Prints help page for the program + * + * progname: the name of the executed program, such as "pg_walsummary" + */ +static void +help(const char *progname) +{ + printf(_("%s prints the contents of a WAL summary file.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]... FILE...\n"), progname); + printf(_("\nOptions:\n")); + printf(_(" -i, --individual list block numbers individually, not as ranges\n")); + printf(_(" -q, --quiet don't print anything, just parse the files\n")); + printf(_(" -?, --help show this help, then exit\n")); + + printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT); + printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); +} diff --git a/src/bin/pg_walsummary/t/001_basic.pl b/src/bin/pg_walsummary/t/001_basic.pl new file mode 100644 index 0000000000..10a232a150 --- /dev/null +++ b/src/bin/pg_walsummary/t/001_basic.pl @@ -0,0 +1,19 @@ +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Utils; +use Test::More; + +my $tempdir = PostgreSQL::Test::Utils::tempdir; + +program_help_ok('pg_walsummary'); +program_version_ok('pg_walsummary'); +program_options_handling_ok('pg_walsummary'); + +command_fails_like( + ['pg_walsummary'], + qr/no input files specified/, + 'input files must be specified'); + +done_testing(); diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl new file mode 100644 index 0000000000..170976f9e2 --- /dev/null +++ b/src/bin/pg_walsummary/t/002_blocks.pl @@ -0,0 +1,88 @@ +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use File::Compare; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Set up a new database instance. +my $node1 = PostgreSQL::Test::Cluster->new('node1'); +$node1->init(has_archiving => 1, allows_streaming => 1); +$node1->append_conf('postgresql.conf', 'summarize_wal = on'); +$node1->start; + +# See what's been summarized up until now. +my $progress = $node1->safe_psql('postgres', <safe_psql('postgres', <poll_query_until('postgres', < '$summarized_lsn' +) +EOM + +# Again check the progress of WAL summarization. +$progress = $node1->safe_psql('postgres', <safe_psql('postgres', <poll_query_until('postgres', < '$summarized_lsn' +) +EOM + +# Figure out the exact details for the new sumamry file. +my $details = $node1->safe_psql('postgres', < '$summarized_lsn' +EOM +my ($tli, $start_lsn, $end_lsn) = split(/\|/, $details); +note("examining summary for TLI $tli from $start_lsn to $end_lsn"); + +# Reconstruct the full pathname for the WAL summary file. +my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary", + $node1->data_dir, $tli, + split(m@/@, $start_lsn), + split(m@/@, $end_lsn); +ok(-f $filename, "WAL summary file exists"); + +# Run pg_walsummary on it. We expect block 0 to be modified, but block 1 +# to be unmodified, so the output should say block 0, not block 0..1 or +# similar. +my ($stdout, $stderr) = run_command([ 'pg_walsummary', $filename ]); +like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified"); +is($stderr, '', 'stderr is empty'); + +done_testing(); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 5fd46b7bd1..f582eb59e7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -4039,3 +4039,5 @@ cb_tablespace_mapping manifest_data manifest_writer rfile +ws_options +ws_file_info