2021-05-07 16:56:14 +02:00
|
|
|
|
2023-01-02 21:00:37 +01:00
|
|
|
# Copyright (c) 2021-2023, PostgreSQL Global Development Group
|
2021-05-07 16:56:14 +02:00
|
|
|
|
2017-02-09 22:23:46 +01:00
|
|
|
use strict;
|
|
|
|
use warnings;
|
2021-10-24 16:28:19 +02:00
|
|
|
use PostgreSQL::Test::Utils;
|
|
|
|
use PostgreSQL::Test::Cluster;
|
2022-02-11 20:54:44 +01:00
|
|
|
use Test::More;
|
2017-02-09 22:23:46 +01:00
|
|
|
|
|
|
|
program_help_ok('pg_receivewal');
|
|
|
|
program_version_ok('pg_receivewal');
|
|
|
|
program_options_handling_ok('pg_receivewal');
|
2017-09-11 22:30:50 +02:00
|
|
|
|
2018-04-07 23:45:39 +02:00
|
|
|
# Set umask so test directories and files are created with default permissions
|
|
|
|
umask(0077);
|
|
|
|
|
2021-10-24 16:28:19 +02:00
|
|
|
my $primary = PostgreSQL::Test::Cluster->new('primary');
|
2021-10-29 03:41:44 +02:00
|
|
|
$primary->init(allows_streaming => 1, extra => ['--wal-segsize=1']);
|
2017-09-11 22:30:50 +02:00
|
|
|
$primary->start;
|
|
|
|
|
|
|
|
my $stream_dir = $primary->basedir . '/archive_wal';
|
|
|
|
mkdir($stream_dir);
|
|
|
|
|
|
|
|
# Sanity checks for command line options.
|
|
|
|
$primary->command_fails(['pg_receivewal'],
|
|
|
|
'pg_receivewal needs target directory specified');
|
|
|
|
$primary->command_fails(
|
|
|
|
[ 'pg_receivewal', '-D', $stream_dir, '--create-slot', '--drop-slot' ],
|
|
|
|
'failure if both --create-slot and --drop-slot specified');
|
|
|
|
$primary->command_fails(
|
|
|
|
[ 'pg_receivewal', '-D', $stream_dir, '--create-slot' ],
|
|
|
|
'failure if --create-slot specified without --slot');
|
2017-10-29 08:16:55 +01:00
|
|
|
$primary->command_fails(
|
|
|
|
[ 'pg_receivewal', '-D', $stream_dir, '--synchronous', '--no-sync' ],
|
|
|
|
'failure if --synchronous specified with --no-sync');
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
$primary->command_fails_like(
|
Rework compression options of pg_receivewal
Since babbbb5 and the introduction of LZ4 in pg_receivewal, the
compression of the WAL archived is controlled by two options:
- --compression-method with "gzip", "none" or "lz4" as possible value.
- --compress=N to specify a compression level. This includes a
backward-incompatible change where a value of 0 leads to a failure
instead of no compression enforced.
This commit takes advantage of a4b5754 and 3603f7c to rework the
compression options of pg_receivewal, as of:
- The removal of --compression-method.
- The extenction of --compress to use the same grammar as pg_basebackup,
with a METHOD:DETAIL format, where a METHOD is "gzip", "none" or "lz4"
and a DETAIL is a comma-separated list of options, the only keyword
supported is now "level" to control the compression level. If only an
integer is specified as value of this option, "none" is implied on 0
and "gzip" is implied otherwise. This brings back --compress to be
backward-compatible with ~14, while still supporting LZ4.
This has also the advantage of centralizing the set of checks used by
pg_receivewal to validate its compression options.
Author: Michael Paquier
Reviewed-by: Robert Haas, Georgios Kokolatos
Discussion: https://postgr.es/m/YlPQGNAAa04raObK@paquier.xyz
2022-04-13 04:09:51 +02:00
|
|
|
[ 'pg_receivewal', '-D', $stream_dir, '--compress', 'none:1', ],
|
|
|
|
qr/\Qpg_receivewal: error: invalid compression specification: compression algorithm "none" does not accept a compression level/,
|
|
|
|
'failure if --compress none:N (where N > 0)');
|
2017-09-11 22:30:50 +02:00
|
|
|
|
|
|
|
# Slot creation and drop
|
|
|
|
my $slot_name = 'test';
|
|
|
|
$primary->command_ok(
|
|
|
|
[ 'pg_receivewal', '--slot', $slot_name, '--create-slot' ],
|
|
|
|
'creating a replication slot');
|
2017-09-26 22:41:20 +02:00
|
|
|
my $slot = $primary->slot($slot_name);
|
|
|
|
is($slot->{'slot_type'}, 'physical', 'physical replication slot was created');
|
Rework compression options of pg_receivewal
Since babbbb5 and the introduction of LZ4 in pg_receivewal, the
compression of the WAL archived is controlled by two options:
- --compression-method with "gzip", "none" or "lz4" as possible value.
- --compress=N to specify a compression level. This includes a
backward-incompatible change where a value of 0 leads to a failure
instead of no compression enforced.
This commit takes advantage of a4b5754 and 3603f7c to rework the
compression options of pg_receivewal, as of:
- The removal of --compression-method.
- The extenction of --compress to use the same grammar as pg_basebackup,
with a METHOD:DETAIL format, where a METHOD is "gzip", "none" or "lz4"
and a DETAIL is a comma-separated list of options, the only keyword
supported is now "level" to control the compression level. If only an
integer is specified as value of this option, "none" is implied on 0
and "gzip" is implied otherwise. This brings back --compress to be
backward-compatible with ~14, while still supporting LZ4.
This has also the advantage of centralizing the set of checks used by
pg_receivewal to validate its compression options.
Author: Michael Paquier
Reviewed-by: Robert Haas, Georgios Kokolatos
Discussion: https://postgr.es/m/YlPQGNAAa04raObK@paquier.xyz
2022-04-13 04:09:51 +02:00
|
|
|
is($slot->{'restart_lsn'}, '', 'restart LSN of new slot is null');
|
2017-09-11 22:30:50 +02:00
|
|
|
$primary->command_ok([ 'pg_receivewal', '--slot', $slot_name, '--drop-slot' ],
|
|
|
|
'dropping a replication slot');
|
2017-09-26 22:41:20 +02:00
|
|
|
is($primary->slot($slot_name)->{'slot_type'},
|
|
|
|
'', 'replication slot was removed');
|
2017-09-11 22:30:50 +02:00
|
|
|
|
|
|
|
# Generate some WAL. Use --synchronous at the same time to add more
|
|
|
|
# code coverage. Switch to the next segment first so that subsequent
|
|
|
|
# restarts of pg_receivewal will see this segment as full..
|
2021-10-29 03:41:44 +02:00
|
|
|
$primary->psql('postgres', 'CREATE TABLE test_table(x integer PRIMARY KEY);');
|
2017-09-11 22:30:50 +02:00
|
|
|
$primary->psql('postgres', 'SELECT pg_switch_wal();');
|
|
|
|
my $nextlsn =
|
|
|
|
$primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
|
|
|
|
chomp($nextlsn);
|
2021-10-29 03:41:44 +02:00
|
|
|
$primary->psql('postgres', 'INSERT INTO test_table VALUES (1);');
|
2017-09-11 22:30:50 +02:00
|
|
|
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
# Stream up to the given position. This is necessary to have a fixed
|
|
|
|
# started point for the next commands done in this test, with or without
|
|
|
|
# compression involved.
|
2017-09-11 22:30:50 +02:00
|
|
|
$primary->command_ok(
|
2018-05-09 16:14:46 +02:00
|
|
|
[
|
2023-05-19 23:24:48 +02:00
|
|
|
'pg_receivewal', '-D', $stream_dir, '--verbose',
|
|
|
|
'--endpos', $nextlsn, '--synchronous', '--no-loop'
|
2018-05-09 16:14:46 +02:00
|
|
|
],
|
2017-09-11 22:30:50 +02:00
|
|
|
'streaming some WAL with --synchronous');
|
2018-04-07 23:45:39 +02:00
|
|
|
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
# Verify that one partial file was generated and keep track of it
|
|
|
|
my @partial_wals = glob "$stream_dir/*\.partial";
|
|
|
|
is(scalar(@partial_wals), 1, "one partial WAL segment was created");
|
|
|
|
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
note "Testing pg_receivewal with compression methods";
|
|
|
|
|
2021-07-31 03:13:15 +02:00
|
|
|
# Check ZLIB compression if available.
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
SKIP:
|
|
|
|
{
|
2021-07-31 03:13:15 +02:00
|
|
|
skip "postgres was not built with ZLIB support", 5
|
|
|
|
if (!check_pg_config("#define HAVE_LIBZ 1"));
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
|
|
|
|
# Generate more WAL worth one completed, compressed, segment.
|
|
|
|
$primary->psql('postgres', 'SELECT pg_switch_wal();');
|
|
|
|
$nextlsn =
|
|
|
|
$primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
|
|
|
|
chomp($nextlsn);
|
2021-10-29 03:41:44 +02:00
|
|
|
$primary->psql('postgres', 'INSERT INTO test_table VALUES (2);');
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
|
|
|
|
$primary->command_ok(
|
|
|
|
[
|
2023-05-19 23:24:48 +02:00
|
|
|
'pg_receivewal', '-D', $stream_dir, '--verbose',
|
|
|
|
'--endpos', $nextlsn, '--compress', 'gzip:1',
|
2021-07-30 14:28:03 +02:00
|
|
|
'--no-loop'
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
],
|
|
|
|
"streaming some WAL using ZLIB compression");
|
|
|
|
|
|
|
|
# Verify that the stored files are generated with their expected
|
|
|
|
# names.
|
|
|
|
my @zlib_wals = glob "$stream_dir/*.gz";
|
|
|
|
is(scalar(@zlib_wals), 1,
|
|
|
|
"one WAL segment compressed with ZLIB was created");
|
|
|
|
my @zlib_partial_wals = glob "$stream_dir/*.gz.partial";
|
|
|
|
is(scalar(@zlib_partial_wals),
|
|
|
|
1, "one partial WAL segment compressed with ZLIB was created");
|
|
|
|
|
|
|
|
# Verify that the start streaming position is computed correctly by
|
|
|
|
# comparing it with the partial file generated previously. The name
|
|
|
|
# of the previous partial, now-completed WAL segment is updated, keeping
|
|
|
|
# its base number.
|
|
|
|
$partial_wals[0] =~ s/\.partial$/.gz/;
|
2021-07-27 03:58:21 +02:00
|
|
|
is($zlib_wals[0] eq $partial_wals[0],
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
1, "one partial WAL segment is now completed");
|
|
|
|
# Update the list of partial wals with the current one.
|
|
|
|
@partial_wals = @zlib_partial_wals;
|
|
|
|
|
2021-07-15 14:25:03 +02:00
|
|
|
# Check the integrity of the completed segment, if gzip is a command
|
|
|
|
# available.
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
my $gzip = $ENV{GZIP_PROGRAM};
|
|
|
|
skip "program gzip is not found in your system", 1
|
2022-02-15 05:41:40 +01:00
|
|
|
if (!defined $gzip
|
|
|
|
|| $gzip eq '');
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
|
|
|
|
my $gzip_is_valid = system_log($gzip, '--test', @zlib_wals);
|
|
|
|
is($gzip_is_valid, 0,
|
|
|
|
"gzip verified the integrity of compressed WAL segments");
|
|
|
|
}
|
|
|
|
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
# Check LZ4 compression if available
|
|
|
|
SKIP:
|
|
|
|
{
|
|
|
|
skip "postgres was not built with LZ4 support", 5
|
2022-03-15 18:06:25 +01:00
|
|
|
if (!check_pg_config("#define USE_LZ4 1"));
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
|
|
|
|
# Generate more WAL including one completed, compressed segment.
|
|
|
|
$primary->psql('postgres', 'SELECT pg_switch_wal();');
|
|
|
|
$nextlsn =
|
|
|
|
$primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
|
|
|
|
chomp($nextlsn);
|
|
|
|
$primary->psql('postgres', 'INSERT INTO test_table VALUES (3);');
|
|
|
|
|
|
|
|
# Stream up to the given position.
|
|
|
|
$primary->command_ok(
|
|
|
|
[
|
2023-05-19 23:24:48 +02:00
|
|
|
'pg_receivewal', '-D', $stream_dir, '--verbose',
|
|
|
|
'--endpos', $nextlsn, '--no-loop', '--compress',
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
'lz4'
|
|
|
|
],
|
Rework compression options of pg_receivewal
Since babbbb5 and the introduction of LZ4 in pg_receivewal, the
compression of the WAL archived is controlled by two options:
- --compression-method with "gzip", "none" or "lz4" as possible value.
- --compress=N to specify a compression level. This includes a
backward-incompatible change where a value of 0 leads to a failure
instead of no compression enforced.
This commit takes advantage of a4b5754 and 3603f7c to rework the
compression options of pg_receivewal, as of:
- The removal of --compression-method.
- The extenction of --compress to use the same grammar as pg_basebackup,
with a METHOD:DETAIL format, where a METHOD is "gzip", "none" or "lz4"
and a DETAIL is a comma-separated list of options, the only keyword
supported is now "level" to control the compression level. If only an
integer is specified as value of this option, "none" is implied on 0
and "gzip" is implied otherwise. This brings back --compress to be
backward-compatible with ~14, while still supporting LZ4.
This has also the advantage of centralizing the set of checks used by
pg_receivewal to validate its compression options.
Author: Michael Paquier
Reviewed-by: Robert Haas, Georgios Kokolatos
Discussion: https://postgr.es/m/YlPQGNAAa04raObK@paquier.xyz
2022-04-13 04:09:51 +02:00
|
|
|
'streaming some WAL using --compress=lz4');
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
|
|
|
|
# Verify that the stored files are generated with their expected
|
|
|
|
# names.
|
|
|
|
my @lz4_wals = glob "$stream_dir/*.lz4";
|
|
|
|
is(scalar(@lz4_wals), 1,
|
|
|
|
"one WAL segment compressed with LZ4 was created");
|
|
|
|
my @lz4_partial_wals = glob "$stream_dir/*.lz4.partial";
|
|
|
|
is(scalar(@lz4_partial_wals),
|
|
|
|
1, "one partial WAL segment compressed with LZ4 was created");
|
|
|
|
|
|
|
|
# Verify that the start streaming position is computed correctly by
|
|
|
|
# comparing it with the partial file generated previously. The name
|
|
|
|
# of the previous partial, now-completed WAL segment is updated, keeping
|
|
|
|
# its base number.
|
|
|
|
$partial_wals[0] =~ s/(\.gz)?\.partial$/.lz4/;
|
|
|
|
is($lz4_wals[0] eq $partial_wals[0],
|
|
|
|
1, "one partial WAL segment is now completed");
|
|
|
|
# Update the list of partial wals with the current one.
|
|
|
|
@partial_wals = @lz4_partial_wals;
|
|
|
|
|
|
|
|
# Check the integrity of the completed segment, if LZ4 is an available
|
|
|
|
# command.
|
|
|
|
my $lz4 = $ENV{LZ4};
|
|
|
|
skip "program lz4 is not found in your system", 1
|
2022-02-15 05:41:40 +01:00
|
|
|
if (!defined $lz4
|
|
|
|
|| $lz4 eq '');
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
|
|
|
|
my $lz4_is_valid = system_log($lz4, '-t', @lz4_wals);
|
|
|
|
is($lz4_is_valid, 0,
|
|
|
|
"lz4 verified the integrity of compressed WAL segments");
|
|
|
|
}
|
|
|
|
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
# Verify that the start streaming position is computed and that the value is
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
# correct regardless of whether any compression is available.
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
$primary->psql('postgres', 'SELECT pg_switch_wal();');
|
|
|
|
$nextlsn =
|
|
|
|
$primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
|
|
|
|
chomp($nextlsn);
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
$primary->psql('postgres', 'INSERT INTO test_table VALUES (4);');
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
$primary->command_ok(
|
2021-07-30 14:28:03 +02:00
|
|
|
[
|
2023-05-19 23:24:48 +02:00
|
|
|
'pg_receivewal', '-D', $stream_dir, '--verbose',
|
|
|
|
'--endpos', $nextlsn, '--no-loop'
|
2021-07-30 14:28:03 +02:00
|
|
|
],
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
"streaming some WAL");
|
|
|
|
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
$partial_wals[0] =~ s/(\.gz|\.lz4)?.partial//;
|
Add TAP tests for ZLIB compression for pg_receivewal
There is a non-trivial amount of code that handles ZLIB compression in
pg_receivewal, from basics like the format name, the calculation of the
start streaming position and of course the compression itself, but there
was no automated coverage for it.
This commit introduces a set of conditional tests (if the build supports
ZLIB) to cover the creation of ZLIB-compressed WAL segments, the
handling of the partial, compressed, WAL segments and the compression
operation in itself. Note that there is an extra phase checking the
validity of the generated files by using directly a gzip command, passed
down by the Makefile of pg_receivewal. This part is skipped if the
command cannot be found, something likely going to happen on Windows
with MSVC except if one sets the variable GZIP_PROGRAM in the
environment of the test.
This set of tests will become handy for upcoming patches that add more
options for the compression methods used by pg_receivewal, like LZ4, to
make sure that no existing facilities are broken.
Author: Georgios Kokolatos
Reviewed-by: Gilles Darold, Michael Paquier
Discussion: https://postgr.es/m/07BK3Mk5aEOsTwGaY77qBVyf9GjoEzn8TMgHLyPGfEFPIpTEmoQuP2P4c7teesjSg-LPeUafsp1flnPeQYINMSMB_UpggJDoduB5EDYBqaQ=@protonmail.com
2021-07-15 08:53:20 +02:00
|
|
|
ok(-e $partial_wals[0], "check that previously partial WAL is now complete");
|
|
|
|
|
2018-04-07 23:45:39 +02:00
|
|
|
# Permissions on WAL files should be default
|
|
|
|
SKIP:
|
|
|
|
{
|
|
|
|
skip "unix-style permissions not supported on Windows", 1
|
|
|
|
if ($windows_os);
|
|
|
|
|
|
|
|
ok(check_mode_recursive($stream_dir, 0700, 0600),
|
|
|
|
"check stream dir permissions");
|
|
|
|
}
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
|
|
|
|
note "Testing pg_receivewal with slot as starting streaming point";
|
|
|
|
|
|
|
|
# When using a replication slot, archiving should be resumed from the slot's
|
|
|
|
# restart LSN. Use a new archive location and new slot for this test.
|
|
|
|
my $slot_dir = $primary->basedir . '/slot_wal';
|
|
|
|
mkdir($slot_dir);
|
|
|
|
$slot_name = 'archive_slot';
|
|
|
|
|
|
|
|
# Setup the slot, reserving WAL at creation (corresponding to the
|
2021-10-29 03:41:44 +02:00
|
|
|
# last redo LSN here, actually, so use a checkpoint to reduce the
|
|
|
|
# number of segments archived).
|
|
|
|
$primary->psql('postgres', 'checkpoint;');
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
$primary->psql('postgres',
|
|
|
|
"SELECT pg_create_physical_replication_slot('$slot_name', true);");
|
|
|
|
|
|
|
|
# Get the segment name associated with the slot's restart LSN, that should
|
|
|
|
# be archived.
|
|
|
|
my $walfile_streamed = $primary->safe_psql(
|
|
|
|
'postgres',
|
|
|
|
"SELECT pg_walfile_name(restart_lsn)
|
|
|
|
FROM pg_replication_slots
|
|
|
|
WHERE slot_name = '$slot_name';");
|
|
|
|
|
|
|
|
# Switch to a new segment, to make sure that the segment retained by the
|
|
|
|
# slot is still streamed. This may not be necessary, but play it safe.
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
$primary->psql('postgres', 'INSERT INTO test_table VALUES (5);');
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
$primary->psql('postgres', 'SELECT pg_switch_wal();');
|
|
|
|
$nextlsn =
|
|
|
|
$primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
|
|
|
|
chomp($nextlsn);
|
|
|
|
|
2021-10-29 03:41:44 +02:00
|
|
|
# Add a bit more data to accelerate the end of the next pg_receivewal
|
|
|
|
# commands.
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
$primary->psql('postgres', 'INSERT INTO test_table VALUES (6);');
|
2021-10-29 03:41:44 +02:00
|
|
|
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
# Check case where the slot does not exist.
|
|
|
|
$primary->command_fails_like(
|
|
|
|
[
|
2023-05-19 23:24:48 +02:00
|
|
|
'pg_receivewal', '-D', $slot_dir, '--slot',
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
'nonexistentslot', '-n', '--no-sync', '--verbose',
|
2023-05-19 23:24:48 +02:00
|
|
|
'--endpos', $nextlsn
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
],
|
2022-09-25 00:38:35 +02:00
|
|
|
qr/pg_receivewal: error: replication slot "nonexistentslot" does not exist/,
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
'pg_receivewal fails with non-existing slot');
|
|
|
|
$primary->command_ok(
|
|
|
|
[
|
2023-05-19 23:24:48 +02:00
|
|
|
'pg_receivewal', '-D', $slot_dir, '--slot',
|
|
|
|
$slot_name, '-n', '--no-sync', '--verbose',
|
|
|
|
'--endpos', $nextlsn
|
Allow pg_receivewal to stream from a slot's restart LSN
Prior to this patch, when running pg_receivewal, the streaming start
point would be the current location of the archives if anything is
found in the local directory where WAL segments are written, and
pg_receivewal would fall back to the current WAL flush location if there
are no archives, as of the result of an IDENTIFY_SYSTEM command.
If for some reason the WAL files from pg_receivewal were moved, it is
better to try a restart where we left at, which is the replication
slot's restart_lsn instead of skipping right to the current flush
location, to avoid holes in the WAL backed up. This commit changes
pg_receivewal to use the following sequence of methods to determine the
starting streaming LSN:
- Scan the local archives.
- Use the slot's restart_lsn, if supported by the backend and if a slot
is defined.
- Fallback to the current flush LSN as reported by IDENTIFY_SYSTEM.
To keep compatibility with older server versions, we only attempt to use
READ_REPLICATION_SLOT if the backend version is at least 15, and
fallback to the older behavior of streaming from the current flush
LSN if the command is not supported.
Some TAP tests are added to cover this feature.
Author: Ronan Dunklau
Reviewed-by: Kyotaro Horiguchi, Michael Paquier, Bharath Rupireddy
Discussion: https://postgr.es/m/18708360.4lzOvYHigE@aivenronan
2021-10-26 02:30:37 +02:00
|
|
|
],
|
|
|
|
"WAL streamed from the slot's restart_lsn");
|
|
|
|
ok(-e "$slot_dir/$walfile_streamed",
|
|
|
|
"WAL from the slot's restart_lsn has been archived");
|
2021-11-01 05:16:04 +01:00
|
|
|
|
|
|
|
# Test timeline switch using a replication slot, requiring a promoted
|
|
|
|
# standby.
|
|
|
|
my $backup_name = "basebackup";
|
|
|
|
$primary->backup($backup_name);
|
|
|
|
my $standby = PostgreSQL::Test::Cluster->new("standby");
|
|
|
|
$standby->init_from_backup($primary, $backup_name, has_streaming => 1);
|
|
|
|
$standby->start;
|
|
|
|
|
|
|
|
# Create a replication slot on this new standby
|
|
|
|
my $archive_slot = "archive_slot";
|
|
|
|
$standby->psql(
|
|
|
|
'',
|
|
|
|
"CREATE_REPLICATION_SLOT $archive_slot PHYSICAL (RESERVE_WAL)",
|
|
|
|
replication => 1);
|
|
|
|
# Wait for standby catchup
|
Clean up TAP tests' usage of wait_for_catchup().
By default, wait_for_catchup() waits for the replication connection
to reach the primary's write LSN. That's fine, but in an apparent
attempt to save one query round-trip, it was coded so that we
executed pg_current_wal_lsn() again during each probe query.
Thus, we presented the standby with a moving target to be reached.
(While the test script itself couldn't be causing the write LSN
to advance while it's blocked in wait_for_catchup(), it's plenty
plausible that background activity such as autovacuum is emitting
more WAL.) That could make the test take longer than necessary,
and potentially it could mask bugs by allowing the standby to process
more WAL than a strict interpretation of the test scenario allows.
So, change wait_for_catchup() to do it "by the book", explicitly
collecting the write LSN to wait for at the outset.
Also, various call sites were instructing wait_for_catchup() to
wait for the standby to reach the primary's insert LSN rather than
its write LSN. This also seems like a bad idea. While in most
test scenarios those are the same, if they are different then the
inserted-but-not-yet-written WAL is not presently available to the
standby. The test isn't doing anything to make it become so, so
again we have the potential for unwanted test delay, perhaps even
a test timeout. (Again, background activity would be needed to
make this more than a hypothetical problem.) Hence, change the
callers where necessary so that the wait target is always the
primary's write LSN.
While at it, simplify callers by making use of wait_for_catchup's
default arguments wherever possible (the preceding change makes
this possible in more places than it was before). And rewrite
wait_for_catchup's documentation a bit.
Patch by me; thanks to Julien Rouhaud for review.
Discussion: https://postgr.es/m/2368336.1641843098@sss.pgh.pa.us
2022-01-16 19:29:02 +01:00
|
|
|
$primary->wait_for_catchup($standby);
|
2021-11-01 05:16:04 +01:00
|
|
|
# Get a walfilename from before the promotion to make sure it is archived
|
|
|
|
# after promotion
|
2023-05-19 23:24:48 +02:00
|
|
|
my $standby_slot = $standby->slot($archive_slot);
|
2021-11-01 05:16:04 +01:00
|
|
|
my $replication_slot_lsn = $standby_slot->{'restart_lsn'};
|
|
|
|
|
|
|
|
# pg_walfile_name() is not supported while in recovery, so use the primary
|
|
|
|
# to build the segment name. Both nodes are on the same timeline, so this
|
|
|
|
# produces a segment name with the timeline we are switching from.
|
|
|
|
my $walfile_before_promotion =
|
|
|
|
$primary->safe_psql('postgres',
|
|
|
|
"SELECT pg_walfile_name('$replication_slot_lsn');");
|
|
|
|
# Everything is setup, promote the standby to trigger a timeline switch.
|
|
|
|
$standby->promote;
|
|
|
|
|
|
|
|
# Force a segment switch to make sure at least one full WAL is archived
|
|
|
|
# on the new timeline.
|
|
|
|
my $walfile_after_promotion = $standby->safe_psql('postgres',
|
|
|
|
"SELECT pg_walfile_name(pg_current_wal_insert_lsn());");
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
$standby->psql('postgres', 'INSERT INTO test_table VALUES (7);');
|
2021-11-01 05:16:04 +01:00
|
|
|
$standby->psql('postgres', 'SELECT pg_switch_wal();');
|
|
|
|
$nextlsn =
|
|
|
|
$standby->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
|
|
|
|
chomp($nextlsn);
|
|
|
|
# This speeds up the operation.
|
Add support for LZ4 compression in pg_receivewal
pg_receivewal gains a new option, --compression-method=lz4, available
when the code is compiled with --with-lz4. Similarly to gzip, this
gives the possibility to compress archived WAL segments with LZ4. This
option is not compatible with --compress.
The implementation uses LZ4 frames, and is compatible with simple lz4
commands. Like gzip, using --synchronous ensures that any data will be
flushed to disk within the current .partial segment, so as it is
possible to retrieve as much WAL data as possible even from a
non-completed segment (this requires completing the partial file with
zeros up to the WAL segment size supported by the backend after
decompression, but this is the same as gzip).
The calculation of the streaming start LSN is able to transparently find
and check LZ4-compressed segments. Contrary to gzip where the
uncompressed size is directly stored in the object read, the LZ4 chunk
protocol does not store the uncompressed data by default. There is
contentSize that can be used with LZ4 frames by that would not help if
using an archive that includes segments compressed with the defaults of
a "lz4" command, where this is not stored. So, this commit has taken
the most extensible approach by decompressing the already-archived
segment to check its uncompressed size, through a blank output buffer in
chunks of 64kB (no actual performance difference noticed with 8kB, 16kB
or 32kB, and the operation in itself is actually fast).
Tests have been added to verify the creation and correctness of the
generated LZ4 files. The latter is achieved by the use of command
"lz4", if found in the environment.
The tar-based WAL method in walmethods.c, used now only by
pg_basebackup, does not know yet about LZ4. Its code could be extended
for this purpose.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-05 03:33:25 +01:00
|
|
|
$standby->psql('postgres', 'INSERT INTO test_table VALUES (8);');
|
2021-11-01 05:16:04 +01:00
|
|
|
|
|
|
|
# Now try to resume from the slot after the promotion.
|
|
|
|
my $timeline_dir = $primary->basedir . '/timeline_wal';
|
|
|
|
mkdir($timeline_dir);
|
|
|
|
|
|
|
|
$standby->command_ok(
|
|
|
|
[
|
2023-05-19 23:24:48 +02:00
|
|
|
'pg_receivewal', '-D', $timeline_dir, '--verbose',
|
|
|
|
'--endpos', $nextlsn, '--slot', $archive_slot,
|
|
|
|
'--no-sync', '-n'
|
2021-11-01 05:16:04 +01:00
|
|
|
],
|
|
|
|
"Stream some wal after promoting, resuming from the slot's position");
|
|
|
|
ok(-e "$timeline_dir/$walfile_before_promotion",
|
|
|
|
"WAL segment $walfile_before_promotion archived after timeline jump");
|
|
|
|
ok(-e "$timeline_dir/$walfile_after_promotion",
|
|
|
|
"WAL segment $walfile_after_promotion archived after timeline jump");
|
|
|
|
ok(-e "$timeline_dir/00000002.history",
|
|
|
|
"timeline history file archived after timeline jump");
|
2022-02-11 20:54:44 +01:00
|
|
|
|
|
|
|
done_testing();
|