From ffc9ddaea33f6dfd3dfa95828a0970fbb617bf8a Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 15 Jul 2021 15:53:20 +0900 Subject: [PATCH] Add TAP tests for ZLIB compression for pg_receivewal There is a non-trivial amount of code that handles ZLIB compression in pg_receivewal, from basics like the format name, the calculation of the start streaming position and of course the compression itself, but there was no automated coverage for it. This commit introduces a set of conditional tests (if the build supports ZLIB) to cover the creation of ZLIB-compressed WAL segments, the handling of the partial, compressed, WAL segments and the compression operation in itself. Note that there is an extra phase checking the validity of the generated files by using directly a gzip command, passed down by the Makefile of pg_receivewal. This part is skipped if the command cannot be found, something likely going to happen on Windows with MSVC except if one sets the variable GZIP_PROGRAM in the environment of the test. This set of tests will become handy for upcoming patches that add more options for the compression methods used by pg_receivewal, like LZ4, to make sure that no existing facilities are broken. Author: Georgios Kokolatos Reviewed-by: Gilles Darold, Michael Paquier Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com --- src/bin/pg_basebackup/Makefile | 6 +- src/bin/pg_basebackup/t/020_pg_receivewal.pl | 79 +++++++++++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile index 66e0070f1a..459d514183 100644 --- a/src/bin/pg_basebackup/Makefile +++ b/src/bin/pg_basebackup/Makefile @@ -18,8 +18,12 @@ subdir = src/bin/pg_basebackup top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -# make this available to TAP test scripts +# make these available to TAP test scripts export TAR +# Note that GZIP cannot be used directly as this environment variable is +# used by the command "gzip" to pass down options, so stick with a different +# name. +export GZIP_PROGRAM=$(GZIP) override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) diff --git a/src/bin/pg_basebackup/t/020_pg_receivewal.pl b/src/bin/pg_basebackup/t/020_pg_receivewal.pl index a547c97ef1..305c501e2f 100644 --- a/src/bin/pg_basebackup/t/020_pg_receivewal.pl +++ b/src/bin/pg_basebackup/t/020_pg_receivewal.pl @@ -5,7 +5,7 @@ use strict; use warnings; use TestLib; use PostgresNode; -use Test::More tests => 19; +use Test::More tests => 27; program_help_ok('pg_receivewal'); program_version_ok('pg_receivewal'); @@ -58,7 +58,9 @@ chomp($nextlsn); $primary->psql('postgres', 'INSERT INTO test_table VALUES (generate_series(1,100));'); -# Stream up to the given position. +# Stream up to the given position. This is necessary to have a fixed +# started point for the next commands done in this test, with or without +# compression involved. $primary->command_ok( [ 'pg_receivewal', '-D', $stream_dir, '--verbose', @@ -66,6 +68,79 @@ $primary->command_ok( ], 'streaming some WAL with --synchronous'); +# Verify that one partial file was generated and keep track of it +my @partial_wals = glob "$stream_dir/*\.partial"; +is(scalar(@partial_wals), 1, "one partial WAL segment was created"); + +# Check ZLIB compression if available. +SKIP: +{ + skip "postgres was not built with ZLIB support", 5 + if (!check_pg_config("#define HAVE_LIBZ 1")); + + # Generate more WAL worth one completed, compressed, segment. + $primary->psql('postgres', 'SELECT pg_switch_wal();'); + $nextlsn = + $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();'); + chomp($nextlsn); + $primary->psql('postgres', + 'INSERT INTO test_table VALUES (generate_series(100,200));'); + + $primary->command_ok( + [ + 'pg_receivewal', '-D', $stream_dir, '--verbose', + '--endpos', $nextlsn, '--compress', '1' + ], + "streaming some WAL using ZLIB compression"); + + # Verify that the stored files are generated with their expected + # names. + my @zlib_wals = glob "$stream_dir/*.gz"; + is(scalar(@zlib_wals), 1, + "one WAL segment compressed with ZLIB was created"); + my @zlib_partial_wals = glob "$stream_dir/*.gz.partial"; + is(scalar(@zlib_partial_wals), + 1, "one partial WAL segment compressed with ZLIB was created"); + + # Verify that the start streaming position is computed correctly by + # comparing it with the partial file generated previously. The name + # of the previous partial, now-completed WAL segment is updated, keeping + # its base number. + $partial_wals[0] =~ s/\.partial$/.gz/; + is($zlib_wals[0] =~ m/$partial_wals[0]/, + 1, "one partial WAL segment is now completed"); + # Update the list of partial wals with the current one. + @partial_wals = @zlib_partial_wals; + + # There is one complete and one partial file compressed with ZLIB. + # Check the integrity of both, if gzip is a command available. + my $gzip = $ENV{GZIP_PROGRAM}; + skip "program gzip is not found in your system", 1 + if ( !defined $gzip + || $gzip eq '' + || system_log($gzip, '--version') != 0); + + push(@zlib_wals, @zlib_partial_wals); + my $gzip_is_valid = system_log($gzip, '--test', @zlib_wals); + is($gzip_is_valid, 0, + "gzip verified the integrity of compressed WAL segments"); +} + +# Verify that the start streaming position is computed and that the value is +# correct regardless of whether ZLIB is available. +$primary->psql('postgres', 'SELECT pg_switch_wal();'); +$nextlsn = + $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();'); +chomp($nextlsn); +$primary->psql('postgres', + 'INSERT INTO test_table VALUES (generate_series(200,300));'); +$primary->command_ok( + [ 'pg_receivewal', '-D', $stream_dir, '--verbose', '--endpos', $nextlsn ], + "streaming some WAL"); + +$partial_wals[0] =~ s/(\.gz)?.partial//; +ok(-e $partial_wals[0], "check that previously partial WAL is now complete"); + # Permissions on WAL files should be default SKIP: {