2016-10-23 15:16:31 +02:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* walmethods.c - implementations of different ways to write received wal
|
|
|
|
*
|
|
|
|
* NOTE! The caller must ensure that only one method is instantiated in
|
|
|
|
* any given program, and that it's only instantiated once!
|
|
|
|
*
|
2021-01-02 19:06:25 +01:00
|
|
|
* Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
|
2016-10-23 15:16:31 +02:00
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
|
|
|
* src/bin/pg_basebackup/walmethods.c
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "postgres_fe.h"
|
|
|
|
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <time.h>
|
|
|
|
#include <unistd.h>
|
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
#include <zlib.h>
|
|
|
|
#endif
|
|
|
|
|
2018-04-07 23:45:39 +02:00
|
|
|
#include "common/file_perm.h"
|
2016-10-23 15:16:31 +02:00
|
|
|
#include "common/file_utils.h"
|
2019-10-23 06:08:53 +02:00
|
|
|
#include "pgtar.h"
|
2016-10-23 15:16:31 +02:00
|
|
|
#include "receivelog.h"
|
|
|
|
#include "streamutil.h"
|
|
|
|
|
|
|
|
/* Size of zlib buffer for .tar.gz */
|
|
|
|
#define ZLIB_OUT_SIZE 4096
|
|
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
2017-06-30 20:40:50 +02:00
|
|
|
* WalDirectoryMethod - write wal to a directory looking like pg_wal
|
2016-10-23 15:16:31 +02:00
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Global static data for this method
|
|
|
|
*/
|
|
|
|
typedef struct DirectoryMethodData
|
|
|
|
{
|
|
|
|
char *basedir;
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
WalCompressionMethod compression_method;
|
|
|
|
int compression_level;
|
2016-10-23 15:16:31 +02:00
|
|
|
bool sync;
|
|
|
|
} DirectoryMethodData;
|
|
|
|
static DirectoryMethodData *dir_data = NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Local file handle
|
|
|
|
*/
|
|
|
|
typedef struct DirectoryMethodFile
|
|
|
|
{
|
|
|
|
int fd;
|
|
|
|
off_t currpos;
|
|
|
|
char *pathname;
|
|
|
|
char *fullpath;
|
|
|
|
char *temp_suffix;
|
2017-01-17 12:10:26 +01:00
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
gzFile gzfp;
|
|
|
|
#endif
|
2016-10-23 15:16:31 +02:00
|
|
|
} DirectoryMethodFile;
|
|
|
|
|
2017-04-12 13:43:59 +02:00
|
|
|
static const char *
|
2016-10-23 15:16:31 +02:00
|
|
|
dir_getlasterror(void)
|
|
|
|
{
|
|
|
|
/* Directory method always sets errno, so just use strerror */
|
|
|
|
return strerror(errno);
|
|
|
|
}
|
|
|
|
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
static char *
|
|
|
|
dir_get_file_name(const char *pathname, const char *temp_suffix)
|
|
|
|
{
|
|
|
|
char *filename = pg_malloc0(MAXPGPATH * sizeof(char));
|
|
|
|
|
|
|
|
snprintf(filename, MAXPGPATH, "%s%s%s",
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
pathname,
|
|
|
|
dir_data->compression_method == COMPRESSION_GZIP ? ".gz" : "",
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
temp_suffix ? temp_suffix : "");
|
|
|
|
|
|
|
|
return filename;
|
|
|
|
}
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
static Walfile
|
|
|
|
dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size)
|
|
|
|
{
|
|
|
|
static char tmppath[MAXPGPATH];
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
char *filename;
|
2016-10-23 15:16:31 +02:00
|
|
|
int fd;
|
|
|
|
DirectoryMethodFile *f;
|
2017-01-17 12:10:26 +01:00
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
gzFile gzfp = NULL;
|
|
|
|
#endif
|
2016-10-23 15:16:31 +02:00
|
|
|
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
filename = dir_get_file_name(pathname, temp_suffix);
|
|
|
|
snprintf(tmppath, sizeof(tmppath), "%s/%s",
|
|
|
|
dir_data->basedir, filename);
|
2021-07-26 04:13:37 +02:00
|
|
|
pg_free(filename);
|
2016-10-23 15:16:31 +02:00
|
|
|
|
2017-01-17 12:10:26 +01:00
|
|
|
/*
|
|
|
|
* Open a file for non-compressed as well as compressed files. Tracking
|
|
|
|
* the file descriptor is important for dir_sync() method as gzflush()
|
|
|
|
* does not do any system calls to fsync() to make changes permanent on
|
|
|
|
* disk.
|
|
|
|
*/
|
2018-04-07 23:45:39 +02:00
|
|
|
fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, pg_file_create_mode);
|
2016-10-23 15:16:31 +02:00
|
|
|
if (fd < 0)
|
|
|
|
return NULL;
|
|
|
|
|
2017-01-17 12:10:26 +01:00
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (dir_data->compression_method == COMPRESSION_GZIP)
|
2017-01-17 12:10:26 +01:00
|
|
|
{
|
|
|
|
gzfp = gzdopen(fd, "wb");
|
|
|
|
if (gzfp == NULL)
|
|
|
|
{
|
|
|
|
close(fd);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (gzsetparams(gzfp, dir_data->compression_level,
|
2017-01-17 12:10:26 +01:00
|
|
|
Z_DEFAULT_STRATEGY) != Z_OK)
|
|
|
|
{
|
|
|
|
gzclose(gzfp);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Do pre-padding on non-compressed files */
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (pad_to_size && dir_data->compression_method == COMPRESSION_NONE)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
2018-09-01 21:27:12 +02:00
|
|
|
PGAlignedXLogBlock zerobuf;
|
2016-10-23 15:16:31 +02:00
|
|
|
int bytes;
|
|
|
|
|
2018-09-01 21:27:12 +02:00
|
|
|
memset(zerobuf.data, 0, XLOG_BLCKSZ);
|
2016-10-23 15:16:31 +02:00
|
|
|
for (bytes = 0; bytes < pad_to_size; bytes += XLOG_BLCKSZ)
|
|
|
|
{
|
2018-08-04 22:31:18 +02:00
|
|
|
errno = 0;
|
2018-09-01 21:27:12 +02:00
|
|
|
if (write(fd, zerobuf.data, XLOG_BLCKSZ) != XLOG_BLCKSZ)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
int save_errno = errno;
|
|
|
|
|
|
|
|
close(fd);
|
2018-06-25 04:19:05 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If write didn't set errno, assume problem is no disk space.
|
|
|
|
*/
|
|
|
|
errno = save_errno ? save_errno : ENOSPC;
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (lseek(fd, 0, SEEK_SET) != 0)
|
|
|
|
{
|
|
|
|
int save_errno = errno;
|
|
|
|
|
|
|
|
close(fd);
|
|
|
|
errno = save_errno;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fsync WAL file and containing directory, to ensure the file is
|
|
|
|
* persistently created and zeroed (if padded). That's particularly
|
|
|
|
* important when using synchronous mode, where the file is modified and
|
|
|
|
* fsynced in-place, without a directory fsync.
|
|
|
|
*/
|
|
|
|
if (dir_data->sync)
|
|
|
|
{
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
if (fsync_fname(tmppath, false) != 0 ||
|
|
|
|
fsync_parent_path(tmppath) != 0)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
2017-01-17 12:10:26 +01:00
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (dir_data->compression_method == COMPRESSION_GZIP)
|
2017-01-17 12:10:26 +01:00
|
|
|
gzclose(gzfp);
|
|
|
|
else
|
|
|
|
#endif
|
|
|
|
close(fd);
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
f = pg_malloc0(sizeof(DirectoryMethodFile));
|
2017-01-17 12:10:26 +01:00
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (dir_data->compression_method == COMPRESSION_GZIP)
|
2017-01-17 12:10:26 +01:00
|
|
|
f->gzfp = gzfp;
|
|
|
|
#endif
|
2016-10-23 15:16:31 +02:00
|
|
|
f->fd = fd;
|
|
|
|
f->currpos = 0;
|
|
|
|
f->pathname = pg_strdup(pathname);
|
|
|
|
f->fullpath = pg_strdup(tmppath);
|
|
|
|
if (temp_suffix)
|
|
|
|
f->temp_suffix = pg_strdup(temp_suffix);
|
|
|
|
|
|
|
|
return f;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t
|
|
|
|
dir_write(Walfile f, const void *buf, size_t count)
|
|
|
|
{
|
|
|
|
ssize_t r;
|
|
|
|
DirectoryMethodFile *df = (DirectoryMethodFile *) f;
|
|
|
|
|
|
|
|
Assert(f != NULL);
|
|
|
|
|
2017-01-17 12:10:26 +01:00
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (dir_data->compression_method == COMPRESSION_GZIP)
|
2017-01-17 12:10:26 +01:00
|
|
|
r = (ssize_t) gzwrite(df->gzfp, buf, count);
|
|
|
|
else
|
|
|
|
#endif
|
|
|
|
r = write(df->fd, buf, count);
|
2016-10-23 15:16:31 +02:00
|
|
|
if (r > 0)
|
|
|
|
df->currpos += r;
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
static off_t
|
|
|
|
dir_get_current_pos(Walfile f)
|
|
|
|
{
|
|
|
|
Assert(f != NULL);
|
|
|
|
|
|
|
|
/* Use a cached value to prevent lots of reseeks */
|
|
|
|
return ((DirectoryMethodFile *) f)->currpos;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
dir_close(Walfile f, WalCloseMethod method)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
DirectoryMethodFile *df = (DirectoryMethodFile *) f;
|
|
|
|
static char tmppath[MAXPGPATH];
|
|
|
|
static char tmppath2[MAXPGPATH];
|
|
|
|
|
|
|
|
Assert(f != NULL);
|
|
|
|
|
2017-01-17 12:10:26 +01:00
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (dir_data->compression_method == COMPRESSION_GZIP)
|
2017-01-17 12:10:26 +01:00
|
|
|
r = gzclose(df->gzfp);
|
|
|
|
else
|
|
|
|
#endif
|
|
|
|
r = close(df->fd);
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
if (r == 0)
|
|
|
|
{
|
|
|
|
/* Build path to the current version of the file */
|
|
|
|
if (method == CLOSE_NORMAL && df->temp_suffix)
|
|
|
|
{
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
char *filename;
|
|
|
|
char *filename2;
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
/*
|
|
|
|
* If we have a temp prefix, normal operation is to rename the
|
|
|
|
* file.
|
|
|
|
*/
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
filename = dir_get_file_name(df->pathname, df->temp_suffix);
|
|
|
|
snprintf(tmppath, sizeof(tmppath), "%s/%s",
|
|
|
|
dir_data->basedir, filename);
|
2021-07-26 04:13:37 +02:00
|
|
|
pg_free(filename);
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
|
|
|
|
/* permanent name, so no need for the prefix */
|
|
|
|
filename2 = dir_get_file_name(df->pathname, NULL);
|
|
|
|
snprintf(tmppath2, sizeof(tmppath2), "%s/%s",
|
|
|
|
dir_data->basedir, filename2);
|
2021-07-26 04:13:37 +02:00
|
|
|
pg_free(filename2);
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
r = durable_rename(tmppath, tmppath2);
|
2016-10-23 15:16:31 +02:00
|
|
|
}
|
|
|
|
else if (method == CLOSE_UNLINK)
|
|
|
|
{
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
char *filename;
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
/* Unlink the file once it's closed */
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
filename = dir_get_file_name(df->pathname, df->temp_suffix);
|
|
|
|
snprintf(tmppath, sizeof(tmppath), "%s/%s",
|
|
|
|
dir_data->basedir, filename);
|
2021-07-26 04:13:37 +02:00
|
|
|
pg_free(filename);
|
2016-10-23 15:16:31 +02:00
|
|
|
r = unlink(tmppath);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Else either CLOSE_NORMAL and no temp suffix, or
|
|
|
|
* CLOSE_NO_RENAME. In this case, fsync the file and containing
|
|
|
|
* directory if sync mode is requested.
|
|
|
|
*/
|
|
|
|
if (dir_data->sync)
|
|
|
|
{
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
r = fsync_fname(df->fullpath, false);
|
2016-10-23 15:16:31 +02:00
|
|
|
if (r == 0)
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
r = fsync_parent_path(df->fullpath);
|
2016-10-23 15:16:31 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pg_free(df->pathname);
|
|
|
|
pg_free(df->fullpath);
|
|
|
|
if (df->temp_suffix)
|
|
|
|
pg_free(df->temp_suffix);
|
|
|
|
pg_free(df);
|
|
|
|
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2016-10-23 18:04:34 +02:00
|
|
|
dir_sync(Walfile f)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
Assert(f != NULL);
|
|
|
|
|
|
|
|
if (!dir_data->sync)
|
|
|
|
return 0;
|
|
|
|
|
2017-01-17 12:10:26 +01:00
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (dir_data->compression_method == COMPRESSION_GZIP)
|
2017-01-17 12:10:26 +01:00
|
|
|
{
|
|
|
|
if (gzflush(((DirectoryMethodFile *) f)->gzfp, Z_SYNC_FLUSH) != Z_OK)
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
return fsync(((DirectoryMethodFile *) f)->fd);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t
|
|
|
|
dir_get_file_size(const char *pathname)
|
|
|
|
{
|
|
|
|
struct stat statbuf;
|
|
|
|
static char tmppath[MAXPGPATH];
|
|
|
|
|
|
|
|
snprintf(tmppath, sizeof(tmppath), "%s/%s",
|
|
|
|
dir_data->basedir, pathname);
|
|
|
|
|
|
|
|
if (stat(tmppath, &statbuf) != 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
return statbuf.st_size;
|
|
|
|
}
|
|
|
|
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
static WalCompressionMethod
|
|
|
|
dir_compression_method(void)
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
{
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
return dir_data->compression_method;
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
}
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
static bool
|
|
|
|
dir_existsfile(const char *pathname)
|
|
|
|
{
|
|
|
|
static char tmppath[MAXPGPATH];
|
|
|
|
int fd;
|
|
|
|
|
|
|
|
snprintf(tmppath, sizeof(tmppath), "%s/%s",
|
|
|
|
dir_data->basedir, pathname);
|
|
|
|
|
|
|
|
fd = open(tmppath, O_RDONLY | PG_BINARY, 0);
|
|
|
|
if (fd < 0)
|
|
|
|
return false;
|
|
|
|
close(fd);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool
|
|
|
|
dir_finish(void)
|
|
|
|
{
|
|
|
|
if (dir_data->sync)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Files are fsynced when they are closed, but we need to fsync the
|
|
|
|
* directory entry here as well.
|
|
|
|
*/
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
if (fsync_fname(dir_data->basedir, true) != 0)
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
WalWriteMethod *
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
CreateWalDirectoryMethod(const char *basedir,
|
|
|
|
WalCompressionMethod compression_method,
|
|
|
|
int compression_level, bool sync)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
WalWriteMethod *method;
|
|
|
|
|
|
|
|
method = pg_malloc0(sizeof(WalWriteMethod));
|
|
|
|
method->open_for_write = dir_open_for_write;
|
|
|
|
method->write = dir_write;
|
|
|
|
method->get_current_pos = dir_get_current_pos;
|
|
|
|
method->get_file_size = dir_get_file_size;
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
method->get_file_name = dir_get_file_name;
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
method->compression_method = dir_compression_method;
|
2016-10-23 15:16:31 +02:00
|
|
|
method->close = dir_close;
|
2016-10-23 18:04:34 +02:00
|
|
|
method->sync = dir_sync;
|
2016-10-23 15:16:31 +02:00
|
|
|
method->existsfile = dir_existsfile;
|
|
|
|
method->finish = dir_finish;
|
|
|
|
method->getlasterror = dir_getlasterror;
|
|
|
|
|
|
|
|
dir_data = pg_malloc0(sizeof(DirectoryMethodData));
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
dir_data->compression_method = compression_method;
|
|
|
|
dir_data->compression_level = compression_level;
|
2016-10-23 15:16:31 +02:00
|
|
|
dir_data->basedir = pg_strdup(basedir);
|
|
|
|
dir_data->sync = sync;
|
|
|
|
|
|
|
|
return method;
|
|
|
|
}
|
|
|
|
|
2016-10-25 18:57:56 +02:00
|
|
|
void
|
|
|
|
FreeWalDirectoryMethod(void)
|
|
|
|
{
|
|
|
|
pg_free(dir_data->basedir);
|
|
|
|
pg_free(dir_data);
|
|
|
|
}
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
2017-06-30 20:40:50 +02:00
|
|
|
* WalTarMethod - write wal to a tar file containing pg_wal contents
|
2016-10-23 15:16:31 +02:00
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
typedef struct TarMethodFile
|
|
|
|
{
|
|
|
|
off_t ofs_start; /* Where does the *header* for this file start */
|
|
|
|
off_t currpos;
|
2020-04-24 16:38:10 +02:00
|
|
|
char header[TAR_BLOCK_SIZE];
|
2016-10-23 15:16:31 +02:00
|
|
|
char *pathname;
|
|
|
|
size_t pad_to_size;
|
|
|
|
} TarMethodFile;
|
|
|
|
|
|
|
|
typedef struct TarMethodData
|
|
|
|
{
|
|
|
|
char *tarfilename;
|
|
|
|
int fd;
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
WalCompressionMethod compression_method;
|
|
|
|
int compression_level;
|
2016-10-23 15:16:31 +02:00
|
|
|
bool sync;
|
|
|
|
TarMethodFile *currentfile;
|
|
|
|
char lasterror[1024];
|
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
z_streamp zp;
|
|
|
|
void *zlibOut;
|
|
|
|
#endif
|
|
|
|
} TarMethodData;
|
|
|
|
static TarMethodData *tar_data = NULL;
|
|
|
|
|
|
|
|
#define tar_clear_error() tar_data->lasterror[0] = '\0'
|
2017-08-03 04:44:46 +02:00
|
|
|
#define tar_set_error(msg) strlcpy(tar_data->lasterror, _(msg), sizeof(tar_data->lasterror))
|
2016-10-23 15:16:31 +02:00
|
|
|
|
2017-04-12 13:43:59 +02:00
|
|
|
static const char *
|
2016-10-23 15:16:31 +02:00
|
|
|
tar_getlasterror(void)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If a custom error is set, return that one. Otherwise, assume errno is
|
|
|
|
* set and return that one.
|
|
|
|
*/
|
|
|
|
if (tar_data->lasterror[0])
|
|
|
|
return tar_data->lasterror;
|
|
|
|
return strerror(errno);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
static bool
|
|
|
|
tar_write_compressed_data(void *buf, size_t count, bool flush)
|
|
|
|
{
|
|
|
|
tar_data->zp->next_in = buf;
|
|
|
|
tar_data->zp->avail_in = count;
|
|
|
|
|
|
|
|
while (tar_data->zp->avail_in || flush)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
|
|
|
|
r = deflate(tar_data->zp, flush ? Z_FINISH : Z_NO_FLUSH);
|
|
|
|
if (r == Z_STREAM_ERROR)
|
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not compress data");
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (tar_data->zp->avail_out < ZLIB_OUT_SIZE)
|
|
|
|
{
|
|
|
|
size_t len = ZLIB_OUT_SIZE - tar_data->zp->avail_out;
|
|
|
|
|
2018-08-04 22:31:18 +02:00
|
|
|
errno = 0;
|
2016-10-23 15:16:31 +02:00
|
|
|
if (write(tar_data->fd, tar_data->zlibOut, len) != len)
|
2018-06-25 04:19:05 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If write didn't set errno, assume problem is no disk space.
|
|
|
|
*/
|
|
|
|
if (errno == 0)
|
|
|
|
errno = ENOSPC;
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
2018-06-25 04:19:05 +02:00
|
|
|
}
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
tar_data->zp->next_out = tar_data->zlibOut;
|
|
|
|
tar_data->zp->avail_out = ZLIB_OUT_SIZE;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (r == Z_STREAM_END)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (flush)
|
|
|
|
{
|
|
|
|
/* Reset the stream for writing */
|
|
|
|
if (deflateReset(tar_data->zp) != Z_OK)
|
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not reset compression stream");
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static ssize_t
|
|
|
|
tar_write(Walfile f, const void *buf, size_t count)
|
|
|
|
{
|
|
|
|
ssize_t r;
|
|
|
|
|
|
|
|
Assert(f != NULL);
|
|
|
|
tar_clear_error();
|
|
|
|
|
|
|
|
/* Tarfile will always be positioned at the end */
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (!tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
r = write(tar_data->fd, buf, count);
|
|
|
|
if (r > 0)
|
|
|
|
((TarMethodFile *) f)->currpos += r;
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
else
|
|
|
|
{
|
2019-01-29 01:16:24 +01:00
|
|
|
if (!tar_write_compressed_data(unconstify(void *, buf), count, false))
|
2016-10-23 15:16:31 +02:00
|
|
|
return -1;
|
|
|
|
((TarMethodFile *) f)->currpos += count;
|
|
|
|
return count;
|
|
|
|
}
|
2016-10-23 16:07:31 +02:00
|
|
|
#else
|
2016-10-23 16:00:42 +02:00
|
|
|
else
|
|
|
|
/* Can't happen - compression enabled with no libz */
|
|
|
|
return -1;
|
2016-10-23 16:07:31 +02:00
|
|
|
#endif
|
2016-10-23 15:16:31 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool
|
|
|
|
tar_write_padding_data(TarMethodFile *f, size_t bytes)
|
|
|
|
{
|
2018-09-01 21:27:12 +02:00
|
|
|
PGAlignedXLogBlock zerobuf;
|
2016-10-23 15:16:31 +02:00
|
|
|
size_t bytesleft = bytes;
|
|
|
|
|
2018-09-01 21:27:12 +02:00
|
|
|
memset(zerobuf.data, 0, XLOG_BLCKSZ);
|
2016-10-23 15:16:31 +02:00
|
|
|
while (bytesleft)
|
|
|
|
{
|
2018-09-01 21:27:12 +02:00
|
|
|
size_t bytestowrite = Min(bytesleft, XLOG_BLCKSZ);
|
|
|
|
ssize_t r = tar_write(f, zerobuf.data, bytestowrite);
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
if (r < 0)
|
|
|
|
return false;
|
|
|
|
bytesleft -= r;
|
|
|
|
}
|
2016-10-30 14:10:39 +01:00
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
static char *
|
|
|
|
tar_get_file_name(const char *pathname, const char *temp_suffix)
|
|
|
|
{
|
|
|
|
char *filename = pg_malloc0(MAXPGPATH * sizeof(char));
|
|
|
|
|
|
|
|
snprintf(filename, MAXPGPATH, "%s%s",
|
|
|
|
pathname, temp_suffix ? temp_suffix : "");
|
|
|
|
|
|
|
|
return filename;
|
|
|
|
}
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
static Walfile
|
|
|
|
tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size)
|
|
|
|
{
|
|
|
|
int save_errno;
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
char *tmppath;
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
tar_clear_error();
|
|
|
|
|
|
|
|
if (tar_data->fd < 0)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We open the tar file only when we first try to write to it.
|
|
|
|
*/
|
|
|
|
tar_data->fd = open(tar_data->tarfilename,
|
2018-04-07 23:45:39 +02:00
|
|
|
O_WRONLY | O_CREAT | PG_BINARY,
|
|
|
|
pg_file_create_mode);
|
2016-10-23 15:16:31 +02:00
|
|
|
if (tar_data->fd < 0)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
tar_data->zp = (z_streamp) pg_malloc(sizeof(z_stream));
|
|
|
|
tar_data->zp->zalloc = Z_NULL;
|
|
|
|
tar_data->zp->zfree = Z_NULL;
|
|
|
|
tar_data->zp->opaque = Z_NULL;
|
|
|
|
tar_data->zp->next_out = tar_data->zlibOut;
|
|
|
|
tar_data->zp->avail_out = ZLIB_OUT_SIZE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize deflation library. Adding the magic value 16 to the
|
|
|
|
* default 15 for the windowBits parameter makes the output be
|
|
|
|
* gzip instead of zlib.
|
|
|
|
*/
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (deflateInit2(tar_data->zp, tar_data->compression_level,
|
|
|
|
Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
pg_free(tar_data->zp);
|
|
|
|
tar_data->zp = NULL;
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not initialize compression library");
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* There's no tar header itself, the file starts with regular files */
|
|
|
|
}
|
|
|
|
|
|
|
|
Assert(tar_data->currentfile == NULL);
|
|
|
|
if (tar_data->currentfile != NULL)
|
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("implementation error: tar files can't have more than one open file");
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
tar_data->currentfile = pg_malloc0(sizeof(TarMethodFile));
|
|
|
|
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
tmppath = tar_get_file_name(pathname, temp_suffix);
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
/* Create a header with size set to 0 - we will fill out the size on close */
|
|
|
|
if (tarCreateHeader(tar_data->currentfile->header, tmppath, NULL, 0, S_IRUSR | S_IWUSR, 0, 0, time(NULL)) != TAR_OK)
|
|
|
|
{
|
|
|
|
pg_free(tar_data->currentfile);
|
2021-07-26 04:13:37 +02:00
|
|
|
pg_free(tmppath);
|
2016-10-23 15:16:31 +02:00
|
|
|
tar_data->currentfile = NULL;
|
|
|
|
tar_set_error("could not create tar header");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2021-07-26 04:13:37 +02:00
|
|
|
pg_free(tmppath);
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
/* Flush existing data */
|
|
|
|
if (!tar_write_compressed_data(NULL, 0, true))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/* Turn off compression for header */
|
|
|
|
if (deflateParams(tar_data->zp, 0, 0) != Z_OK)
|
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not change compression parameters");
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
tar_data->currentfile->ofs_start = lseek(tar_data->fd, 0, SEEK_CUR);
|
|
|
|
if (tar_data->currentfile->ofs_start == -1)
|
|
|
|
{
|
|
|
|
save_errno = errno;
|
|
|
|
pg_free(tar_data->currentfile);
|
|
|
|
tar_data->currentfile = NULL;
|
|
|
|
errno = save_errno;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
tar_data->currentfile->currpos = 0;
|
|
|
|
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (!tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
2018-08-04 22:31:18 +02:00
|
|
|
errno = 0;
|
2020-04-24 16:38:10 +02:00
|
|
|
if (write(tar_data->fd, tar_data->currentfile->header,
|
|
|
|
TAR_BLOCK_SIZE) != TAR_BLOCK_SIZE)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
save_errno = errno;
|
|
|
|
pg_free(tar_data->currentfile);
|
|
|
|
tar_data->currentfile = NULL;
|
2018-06-25 04:19:05 +02:00
|
|
|
/* if write didn't set errno, assume problem is no disk space */
|
|
|
|
errno = save_errno ? save_errno : ENOSPC;
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* Write header through the zlib APIs but with no compression */
|
2020-04-24 16:38:10 +02:00
|
|
|
if (!tar_write_compressed_data(tar_data->currentfile->header,
|
|
|
|
TAR_BLOCK_SIZE, true))
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/* Re-enable compression for the rest of the file */
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (deflateParams(tar_data->zp, tar_data->compression_level, 0) != Z_OK)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not change compression parameters");
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
tar_data->currentfile->pathname = pg_strdup(pathname);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Uncompressed files are padded on creation, but for compression we can't
|
|
|
|
* do that
|
|
|
|
*/
|
|
|
|
if (pad_to_size)
|
|
|
|
{
|
|
|
|
tar_data->currentfile->pad_to_size = pad_to_size;
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (!tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
/* Uncompressed, so pad now */
|
|
|
|
tar_write_padding_data(tar_data->currentfile, pad_to_size);
|
|
|
|
/* Seek back to start */
|
2020-04-24 16:38:10 +02:00
|
|
|
if (lseek(tar_data->fd,
|
|
|
|
tar_data->currentfile->ofs_start + TAR_BLOCK_SIZE,
|
|
|
|
SEEK_SET) != tar_data->currentfile->ofs_start + TAR_BLOCK_SIZE)
|
2016-10-23 15:16:31 +02:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
tar_data->currentfile->currpos = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return tar_data->currentfile;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t
|
|
|
|
tar_get_file_size(const char *pathname)
|
|
|
|
{
|
|
|
|
tar_clear_error();
|
|
|
|
|
|
|
|
/* Currently not used, so not supported */
|
|
|
|
errno = ENOSYS;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
static WalCompressionMethod
|
|
|
|
tar_compression_method(void)
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
{
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
return tar_data->compression_method;
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
}
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
static off_t
|
|
|
|
tar_get_current_pos(Walfile f)
|
|
|
|
{
|
|
|
|
Assert(f != NULL);
|
|
|
|
tar_clear_error();
|
|
|
|
|
|
|
|
return ((TarMethodFile *) f)->currpos;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2016-10-23 18:04:34 +02:00
|
|
|
tar_sync(Walfile f)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
Assert(f != NULL);
|
|
|
|
tar_clear_error();
|
|
|
|
|
2016-10-25 18:56:21 +02:00
|
|
|
if (!tar_data->sync)
|
|
|
|
return 0;
|
|
|
|
|
2016-10-23 15:16:31 +02:00
|
|
|
/*
|
|
|
|
* Always sync the whole tarfile, because that's all we can do. This makes
|
|
|
|
* no sense on compressed files, so just ignore those.
|
|
|
|
*/
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
return fsync(tar_data->fd);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
tar_close(Walfile f, WalCloseMethod method)
|
|
|
|
{
|
|
|
|
ssize_t filesize;
|
|
|
|
int padding;
|
|
|
|
TarMethodFile *tf = (TarMethodFile *) f;
|
|
|
|
|
|
|
|
Assert(f != NULL);
|
|
|
|
tar_clear_error();
|
|
|
|
|
|
|
|
if (method == CLOSE_UNLINK)
|
|
|
|
{
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
tar_set_error("unlink not supported with compression");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Unlink the file that we just wrote to the tar. We do this by
|
|
|
|
* truncating it to the start of the header. This is safe as we only
|
|
|
|
* allow writing of the very last file.
|
|
|
|
*/
|
|
|
|
if (ftruncate(tar_data->fd, tf->ofs_start) != 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
pg_free(tf->pathname);
|
|
|
|
pg_free(tf);
|
|
|
|
tar_data->currentfile = NULL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Pad the file itself with zeroes if necessary. Note that this is
|
|
|
|
* different from the tar format padding -- this is the padding we asked
|
|
|
|
* for when the file was opened.
|
|
|
|
*/
|
|
|
|
if (tf->pad_to_size)
|
|
|
|
{
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* A compressed tarfile is padded on close since we cannot know
|
|
|
|
* the size of the compressed output until the end.
|
|
|
|
*/
|
|
|
|
size_t sizeleft = tf->pad_to_size - tf->currpos;
|
|
|
|
|
|
|
|
if (sizeleft)
|
|
|
|
{
|
|
|
|
if (!tar_write_padding_data(tf, sizeleft))
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* An uncompressed tarfile was padded on creation, so just adjust
|
|
|
|
* the current position as if we seeked to the end.
|
|
|
|
*/
|
|
|
|
tf->currpos = tf->pad_to_size;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2020-04-24 16:38:10 +02:00
|
|
|
* Get the size of the file, and pad out to a multiple of the tar block
|
|
|
|
* size.
|
2016-10-23 15:16:31 +02:00
|
|
|
*/
|
|
|
|
filesize = tar_get_current_pos(f);
|
2020-04-24 16:38:10 +02:00
|
|
|
padding = tarPaddingBytesRequired(filesize);
|
2016-10-23 15:16:31 +02:00
|
|
|
if (padding)
|
|
|
|
{
|
2020-04-24 16:38:10 +02:00
|
|
|
char zerobuf[TAR_BLOCK_SIZE];
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
MemSet(zerobuf, 0, padding);
|
|
|
|
if (tar_write(f, zerobuf, padding) != padding)
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
/* Flush the current buffer */
|
|
|
|
if (!tar_write_compressed_data(NULL, 0, true))
|
|
|
|
{
|
|
|
|
errno = EINVAL;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Now go back and update the header with the correct filesize and
|
|
|
|
* possibly also renaming the file. We overwrite the entire current header
|
|
|
|
* when done, including the checksum.
|
|
|
|
*/
|
|
|
|
print_tar_number(&(tf->header[124]), 12, filesize);
|
|
|
|
|
|
|
|
if (method == CLOSE_NORMAL)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We overwrite it with what it was before if we have no tempname,
|
|
|
|
* since we're going to write the buffer anyway.
|
|
|
|
*/
|
|
|
|
strlcpy(&(tf->header[0]), tf->pathname, 100);
|
|
|
|
|
|
|
|
print_tar_number(&(tf->header[148]), 8, tarChecksum(((TarMethodFile *) f)->header));
|
|
|
|
if (lseek(tar_data->fd, tf->ofs_start, SEEK_SET) != ((TarMethodFile *) f)->ofs_start)
|
|
|
|
return -1;
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (!tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
2018-08-04 22:31:18 +02:00
|
|
|
errno = 0;
|
2020-04-24 16:38:10 +02:00
|
|
|
if (write(tar_data->fd, tf->header, TAR_BLOCK_SIZE) != TAR_BLOCK_SIZE)
|
2018-06-25 04:19:05 +02:00
|
|
|
{
|
|
|
|
/* if write didn't set errno, assume problem is no disk space */
|
|
|
|
if (errno == 0)
|
|
|
|
errno = ENOSPC;
|
2016-10-23 15:16:31 +02:00
|
|
|
return -1;
|
2018-06-25 04:19:05 +02:00
|
|
|
}
|
2016-10-23 15:16:31 +02:00
|
|
|
}
|
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* Turn off compression */
|
|
|
|
if (deflateParams(tar_data->zp, 0, 0) != Z_OK)
|
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not change compression parameters");
|
2016-10-23 15:16:31 +02:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Overwrite the header, assuming the size will be the same */
|
2020-04-24 16:38:10 +02:00
|
|
|
if (!tar_write_compressed_data(tar_data->currentfile->header,
|
|
|
|
TAR_BLOCK_SIZE, true))
|
2016-10-23 15:16:31 +02:00
|
|
|
return -1;
|
|
|
|
|
|
|
|
/* Turn compression back on */
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (deflateParams(tar_data->zp, tar_data->compression_level, 0) != Z_OK)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not change compression parameters");
|
2016-10-23 15:16:31 +02:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Move file pointer back down to end, so we can write the next file */
|
|
|
|
if (lseek(tar_data->fd, 0, SEEK_END) < 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
/* Always fsync on close, so the padding gets fsynced */
|
2018-06-26 02:41:58 +02:00
|
|
|
if (tar_sync(f) < 0)
|
2019-07-29 07:41:06 +02:00
|
|
|
exit(1);
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
/* Clean up and done */
|
|
|
|
pg_free(tf->pathname);
|
|
|
|
pg_free(tf);
|
|
|
|
tar_data->currentfile = NULL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool
|
|
|
|
tar_existsfile(const char *pathname)
|
|
|
|
{
|
|
|
|
tar_clear_error();
|
|
|
|
/* We only deal with new tarfiles, so nothing externally created exists */
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool
|
|
|
|
tar_finish(void)
|
|
|
|
{
|
|
|
|
char zerobuf[1024];
|
|
|
|
|
|
|
|
tar_clear_error();
|
|
|
|
|
|
|
|
if (tar_data->currentfile)
|
|
|
|
{
|
|
|
|
if (tar_close(tar_data->currentfile, CLOSE_NORMAL) != 0)
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2018-06-26 02:41:58 +02:00
|
|
|
/* A tarfile always ends with two empty blocks */
|
2016-10-23 15:16:31 +02:00
|
|
|
MemSet(zerobuf, 0, sizeof(zerobuf));
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (!tar_data->compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
2018-08-04 22:31:18 +02:00
|
|
|
errno = 0;
|
2016-10-23 15:16:31 +02:00
|
|
|
if (write(tar_data->fd, zerobuf, sizeof(zerobuf)) != sizeof(zerobuf))
|
2018-06-25 04:19:05 +02:00
|
|
|
{
|
|
|
|
/* if write didn't set errno, assume problem is no disk space */
|
|
|
|
if (errno == 0)
|
|
|
|
errno = ENOSPC;
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
2018-06-25 04:19:05 +02:00
|
|
|
}
|
2016-10-23 15:16:31 +02:00
|
|
|
}
|
|
|
|
#ifdef HAVE_LIBZ
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (!tar_write_compressed_data(zerobuf, sizeof(zerobuf), false))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Also flush all data to make sure the gzip stream is finished */
|
|
|
|
tar_data->zp->next_in = NULL;
|
|
|
|
tar_data->zp->avail_in = 0;
|
|
|
|
while (true)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
|
|
|
|
r = deflate(tar_data->zp, Z_FINISH);
|
|
|
|
|
|
|
|
if (r == Z_STREAM_ERROR)
|
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not compress data");
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (tar_data->zp->avail_out < ZLIB_OUT_SIZE)
|
|
|
|
{
|
|
|
|
size_t len = ZLIB_OUT_SIZE - tar_data->zp->avail_out;
|
|
|
|
|
2018-08-04 22:31:18 +02:00
|
|
|
errno = 0;
|
2016-10-23 15:16:31 +02:00
|
|
|
if (write(tar_data->fd, tar_data->zlibOut, len) != len)
|
2018-06-25 04:19:05 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If write didn't set errno, assume problem is no disk
|
|
|
|
* space.
|
|
|
|
*/
|
|
|
|
if (errno == 0)
|
|
|
|
errno = ENOSPC;
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
2018-06-25 04:19:05 +02:00
|
|
|
}
|
2016-10-23 15:16:31 +02:00
|
|
|
}
|
|
|
|
if (r == Z_STREAM_END)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (deflateEnd(tar_data->zp) != Z_OK)
|
|
|
|
{
|
2017-08-05 00:31:01 +02:00
|
|
|
tar_set_error("could not close compression stream");
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* sync the empty blocks as well, since they're after the last file */
|
2016-10-25 18:56:21 +02:00
|
|
|
if (tar_data->sync)
|
2018-06-26 02:41:58 +02:00
|
|
|
{
|
|
|
|
if (fsync(tar_data->fd) != 0)
|
|
|
|
return false;
|
|
|
|
}
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
if (close(tar_data->fd) != 0)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
tar_data->fd = -1;
|
|
|
|
|
|
|
|
if (tar_data->sync)
|
|
|
|
{
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
if (fsync_fname(tar_data->tarfilename, false) != 0)
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
if (fsync_parent_path(tar_data->tarfilename) != 0)
|
2016-10-23 15:16:31 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
/*
|
|
|
|
* The argument compression_method is currently ignored. It is in place for
|
|
|
|
* symmetry with CreateWalDirectoryMethod which uses it for distinguishing
|
|
|
|
* between the different compression methods. CreateWalTarMethod and its family
|
|
|
|
* of functions handle only zlib compression.
|
|
|
|
*/
|
2016-10-23 15:16:31 +02:00
|
|
|
WalWriteMethod *
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
CreateWalTarMethod(const char *tarbase,
|
|
|
|
WalCompressionMethod compression_method,
|
|
|
|
int compression_level, bool sync)
|
2016-10-23 15:16:31 +02:00
|
|
|
{
|
|
|
|
WalWriteMethod *method;
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
const char *suffix = (compression_level != 0) ? ".tar.gz" : ".tar";
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
method = pg_malloc0(sizeof(WalWriteMethod));
|
|
|
|
method->open_for_write = tar_open_for_write;
|
|
|
|
method->write = tar_write;
|
|
|
|
method->get_current_pos = tar_get_current_pos;
|
|
|
|
method->get_file_size = tar_get_file_size;
|
Fix some issues with WAL segment opening for pg_receivewal --compress
The logic handling the opening of new WAL segments was fuzzy when using
--compress if a partial, non-compressed, segment with the same base name
existed in the repository storing those files. In this case, using
--compress would cause the code to first check for the existence and the
size of a non-compressed segment, followed by the opening of a new
compressed, partial, segment. The code was accidentally working
correctly on most platforms as the buildfarm has proved, except
bowerbird where gzflush() could fail in this code path. It is wrong
anyway to take the code path used pre-padding when creating a new
partial, non-compressed, segment, so let's fix it.
Note that this issue exists when users mix successive runs of
pg_receivewal with or without compression, as discovered with the tests
introduced by ffc9dda.
While on it, this refactors the code so as code paths that need to know
about the ".gz" suffix are down from four to one in walmethods.c, easing
a bit the introduction of new compression methods. This addresses a
second issue where log messages generated for an unexpected failure
would not show the compressed segment name involved, which was
confusing, printing instead the name of the non-compressed equivalent.
Reported-by: Georgios Kokolatos
Discussion: https://postgr.es/m/YPDLz2x3o1aX2wRh@paquier.xyz
Backpatch-through: 10
2021-07-20 05:12:39 +02:00
|
|
|
method->get_file_name = tar_get_file_name;
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
method->compression_method = tar_compression_method;
|
2016-10-23 15:16:31 +02:00
|
|
|
method->close = tar_close;
|
2016-10-23 18:04:34 +02:00
|
|
|
method->sync = tar_sync;
|
2016-10-23 15:16:31 +02:00
|
|
|
method->existsfile = tar_existsfile;
|
|
|
|
method->finish = tar_finish;
|
|
|
|
method->getlasterror = tar_getlasterror;
|
|
|
|
|
|
|
|
tar_data = pg_malloc0(sizeof(TarMethodData));
|
|
|
|
tar_data->tarfilename = pg_malloc0(strlen(tarbase) + strlen(suffix) + 1);
|
|
|
|
sprintf(tar_data->tarfilename, "%s%s", tarbase, suffix);
|
|
|
|
tar_data->fd = -1;
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
tar_data->compression_method = compression_method;
|
|
|
|
tar_data->compression_level = compression_level;
|
2016-10-23 15:16:31 +02:00
|
|
|
tar_data->sync = sync;
|
2016-10-23 16:00:42 +02:00
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (compression_level)
|
2016-10-23 15:16:31 +02:00
|
|
|
tar_data->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
|
2016-10-23 16:00:42 +02:00
|
|
|
#endif
|
2016-10-23 15:16:31 +02:00
|
|
|
|
|
|
|
return method;
|
|
|
|
}
|
2016-10-25 18:57:56 +02:00
|
|
|
|
|
|
|
void
|
|
|
|
FreeWalTarMethod(void)
|
|
|
|
{
|
|
|
|
pg_free(tar_data->tarfilename);
|
|
|
|
#ifdef HAVE_LIBZ
|
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the
compression of WAL segments using gzip, with a value of 0 (the default)
meaning that no compression can be used.
This commit introduces a new option, called --compression-method, able
to use as values "none", the default, and "gzip", to make things more
extensible. The case of --compress=0 becomes fuzzy with this option
layer, so we have made the choice to make pg_receivewal return an error
when using "none" and a non-zero compression level, meaning that the
authorized values of --compress are now [1,9] instead of [0,9]. Not
specifying --compress with "gzip" as compression method makes
pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).
The code in charge of finding the streaming start LSN when scanning the
existing archives is refactored and made more extensible. While on it,
rename "compression" to "compression_level" in walmethods.c, to reduce
the confusion with the introduction of the compression method, even if
the tar method used by pg_basebackup does not rely on the compression
method (yet, at least), but just on the compression level (this area
could be improved more, actually).
This is in preparation for an upcoming patch that adds LZ4 support to
pg_receivewal.
Author: Georgios Kokolatos
Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,
Robert Haas
Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
2021-11-04 03:10:31 +01:00
|
|
|
if (tar_data->compression_level)
|
2016-10-25 18:57:56 +02:00
|
|
|
pg_free(tar_data->zlibOut);
|
|
|
|
#endif
|
|
|
|
pg_free(tar_data);
|
|
|
|
}
|