Switch pg_dump to use compression specifications
Compression specifications are currently used by pg_basebackup and pg_receivewal, and are able to let the user control in an extended way the method and level of compression used. As an effect of this commit, pg_dump's -Z/--compress is now able to use more than just an integer, as of the grammar "method[:detail]". The method can be either "none" or "gzip", and can optionally take a detail string. If the detail string is only an integer, it defines the compression level. A comma-separated list of keywords can also be used method allows for more options, the only keyword supported now is "level". The change is backward-compatible, hence specifying only an integer leads to no compression for a level of 0 and gzip compression when the level is greater than 0. Most of the code changes are straight-forward, as pg_dump was relying on an integer tracking the compression level to check for gzip or no compression. These are changed to use a compression specification and the algorithm stored in it. As of this change, note that the dump format is not bumped because there is no need yet to track the compression algorithm in the TOC entries. Hence, we still rely on the compression level to make the difference when reading them. This will be mandatory once a new compression method is added, though. In order to keep the code simpler when parsing the compression specification, the code is changed so as pg_dump now fails hard when using gzip on -Z/--compress without its support compiled, rather than enforcing no compression without the user knowing about it except through a warning. Like before this commit, archive and custom formats are compressed by default when the code is compiled with gzip, and left uncompressed without gzip. Author: Georgios Kokolatos Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/O4mutIrCES8ZhlXJiMvzsivT7ztAMja2lkdL1LJx6O5f22I2W8PBIeLKz7mDLwxHoibcnRAYJXm1pH4tyUNC4a8eDzLn22a6Pb1S74Niexg=@pm.me
This commit is contained in:
parent
edf12e7bbd
commit
5e73a60488
|
@ -644,17 +644,39 @@ PostgreSQL documentation
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry>
|
<varlistentry>
|
||||||
<term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
|
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
|
||||||
<term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
|
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
|
||||||
|
<term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
|
||||||
|
<term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
|
||||||
<listitem>
|
<listitem>
|
||||||
<para>
|
<para>
|
||||||
Specify the compression level to use. Zero means no compression.
|
Specify the compression method and/or the compression level to use.
|
||||||
|
The compression method can be set to <literal>gzip</literal> or
|
||||||
|
<literal>none</literal> for no compression.
|
||||||
|
A compression detail string can optionally be specified. If the
|
||||||
|
detail string is an integer, it specifies the compression level.
|
||||||
|
Otherwise, it should be a comma-separated list of items, each of the
|
||||||
|
form <literal>keyword</literal> or <literal>keyword=value</literal>.
|
||||||
|
Currently, the only supported keyword is <literal>level</literal>.
|
||||||
|
</para>
|
||||||
|
|
||||||
|
<para>
|
||||||
|
If no compression level is specified, the default compression
|
||||||
|
level will be used. If only a level is specified without mentioning
|
||||||
|
an algorithm, <literal>gzip</literal> compression will be used if
|
||||||
|
the level is greater than <literal>0</literal>, and no compression
|
||||||
|
will be used if the level is <literal>0</literal>.
|
||||||
|
</para>
|
||||||
|
|
||||||
|
<para>
|
||||||
For the custom and directory archive formats, this specifies compression of
|
For the custom and directory archive formats, this specifies compression of
|
||||||
individual table-data segments, and the default is to compress
|
individual table-data segments, and the default is to compress using
|
||||||
at a moderate level.
|
<literal>gzip</literal> at a moderate level. For plain text output,
|
||||||
For plain text output, setting a nonzero compression level causes
|
setting a nonzero compression level causes the entire output file to be compressed,
|
||||||
the entire output file to be compressed, as though it had been
|
as though it had been fed through <application>gzip</application>; but the default
|
||||||
fed through <application>gzip</application>; but the default is not to compress.
|
is not to compress.
|
||||||
|
</para>
|
||||||
|
<para>
|
||||||
The tar archive format currently does not support compression at all.
|
The tar archive format currently does not support compression at all.
|
||||||
</para>
|
</para>
|
||||||
</listitem>
|
</listitem>
|
||||||
|
|
|
@ -64,7 +64,7 @@
|
||||||
/* typedef appears in compress_io.h */
|
/* typedef appears in compress_io.h */
|
||||||
struct CompressorState
|
struct CompressorState
|
||||||
{
|
{
|
||||||
CompressionAlgorithm comprAlg;
|
pg_compress_specification compression_spec;
|
||||||
WriteFunc writeF;
|
WriteFunc writeF;
|
||||||
|
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
|
@ -74,9 +74,6 @@ struct CompressorState
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
|
|
||||||
int *level);
|
|
||||||
|
|
||||||
/* Routines that support zlib compressed data I/O */
|
/* Routines that support zlib compressed data I/O */
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
static void InitCompressorZlib(CompressorState *cs, int level);
|
static void InitCompressorZlib(CompressorState *cs, int level);
|
||||||
|
@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
|
||||||
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
|
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
|
||||||
const char *data, size_t dLen);
|
const char *data, size_t dLen);
|
||||||
|
|
||||||
/*
|
|
||||||
* Interprets a numeric 'compression' value. The algorithm implied by the
|
|
||||||
* value (zlib or none at the moment), is returned in *alg, and the
|
|
||||||
* zlib compression level in *level.
|
|
||||||
*/
|
|
||||||
static void
|
|
||||||
ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
|
|
||||||
{
|
|
||||||
if (compression == Z_DEFAULT_COMPRESSION ||
|
|
||||||
(compression > 0 && compression <= 9))
|
|
||||||
*alg = COMPR_ALG_LIBZ;
|
|
||||||
else if (compression == 0)
|
|
||||||
*alg = COMPR_ALG_NONE;
|
|
||||||
else
|
|
||||||
{
|
|
||||||
pg_fatal("invalid compression code: %d", compression);
|
|
||||||
*alg = COMPR_ALG_NONE; /* keep compiler quiet */
|
|
||||||
}
|
|
||||||
|
|
||||||
/* The level is just the passed-in value. */
|
|
||||||
if (level)
|
|
||||||
*level = compression;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Public interface routines */
|
/* Public interface routines */
|
||||||
|
|
||||||
/* Allocate a new compressor */
|
/* Allocate a new compressor */
|
||||||
CompressorState *
|
CompressorState *
|
||||||
AllocateCompressor(int compression, WriteFunc writeF)
|
AllocateCompressor(const pg_compress_specification compression_spec,
|
||||||
|
WriteFunc writeF)
|
||||||
{
|
{
|
||||||
CompressorState *cs;
|
CompressorState *cs;
|
||||||
CompressionAlgorithm alg;
|
|
||||||
int level;
|
|
||||||
|
|
||||||
ParseCompressionOption(compression, &alg, &level);
|
|
||||||
|
|
||||||
#ifndef HAVE_LIBZ
|
#ifndef HAVE_LIBZ
|
||||||
if (alg == COMPR_ALG_LIBZ)
|
if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
|
||||||
pg_fatal("not built with zlib support");
|
pg_fatal("not built with zlib support");
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
|
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
|
||||||
cs->writeF = writeF;
|
cs->writeF = writeF;
|
||||||
cs->comprAlg = alg;
|
cs->compression_spec = compression_spec;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Perform compression algorithm specific initialization.
|
* Perform compression algorithm specific initialization.
|
||||||
*/
|
*/
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
if (alg == COMPR_ALG_LIBZ)
|
if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
|
||||||
InitCompressorZlib(cs, level);
|
InitCompressorZlib(cs, cs->compression_spec.level);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
return cs;
|
return cs;
|
||||||
|
@ -154,15 +124,12 @@ AllocateCompressor(int compression, WriteFunc writeF)
|
||||||
* out with ahwrite().
|
* out with ahwrite().
|
||||||
*/
|
*/
|
||||||
void
|
void
|
||||||
ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
|
ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compression_spec,
|
||||||
|
ReadFunc readF)
|
||||||
{
|
{
|
||||||
CompressionAlgorithm alg;
|
if (compression_spec.algorithm == PG_COMPRESSION_NONE)
|
||||||
|
|
||||||
ParseCompressionOption(compression, &alg, NULL);
|
|
||||||
|
|
||||||
if (alg == COMPR_ALG_NONE)
|
|
||||||
ReadDataFromArchiveNone(AH, readF);
|
ReadDataFromArchiveNone(AH, readF);
|
||||||
if (alg == COMPR_ALG_LIBZ)
|
if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
|
||||||
{
|
{
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
ReadDataFromArchiveZlib(AH, readF);
|
ReadDataFromArchiveZlib(AH, readF);
|
||||||
|
@ -179,18 +146,23 @@ void
|
||||||
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
|
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
|
||||||
const void *data, size_t dLen)
|
const void *data, size_t dLen)
|
||||||
{
|
{
|
||||||
switch (cs->comprAlg)
|
switch (cs->compression_spec.algorithm)
|
||||||
{
|
{
|
||||||
case COMPR_ALG_LIBZ:
|
case PG_COMPRESSION_GZIP:
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
WriteDataToArchiveZlib(AH, cs, data, dLen);
|
WriteDataToArchiveZlib(AH, cs, data, dLen);
|
||||||
#else
|
#else
|
||||||
pg_fatal("not built with zlib support");
|
pg_fatal("not built with zlib support");
|
||||||
#endif
|
#endif
|
||||||
break;
|
break;
|
||||||
case COMPR_ALG_NONE:
|
case PG_COMPRESSION_NONE:
|
||||||
WriteDataToArchiveNone(AH, cs, data, dLen);
|
WriteDataToArchiveNone(AH, cs, data, dLen);
|
||||||
break;
|
break;
|
||||||
|
case PG_COMPRESSION_LZ4:
|
||||||
|
/* fallthrough */
|
||||||
|
case PG_COMPRESSION_ZSTD:
|
||||||
|
pg_fatal("invalid compression method");
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -201,7 +173,7 @@ void
|
||||||
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
|
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
|
||||||
{
|
{
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
if (cs->comprAlg == COMPR_ALG_LIBZ)
|
if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
|
||||||
EndCompressorZlib(AH, cs);
|
EndCompressorZlib(AH, cs);
|
||||||
#endif
|
#endif
|
||||||
free(cs);
|
free(cs);
|
||||||
|
@ -453,20 +425,27 @@ cfopen_read(const char *path, const char *mode)
|
||||||
{
|
{
|
||||||
cfp *fp;
|
cfp *fp;
|
||||||
|
|
||||||
|
pg_compress_specification compression_spec = {0};
|
||||||
|
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
if (hasSuffix(path, ".gz"))
|
if (hasSuffix(path, ".gz"))
|
||||||
fp = cfopen(path, mode, 1);
|
{
|
||||||
|
compression_spec.algorithm = PG_COMPRESSION_GZIP;
|
||||||
|
fp = cfopen(path, mode, compression_spec);
|
||||||
|
}
|
||||||
else
|
else
|
||||||
#endif
|
#endif
|
||||||
{
|
{
|
||||||
fp = cfopen(path, mode, 0);
|
compression_spec.algorithm = PG_COMPRESSION_NONE;
|
||||||
|
fp = cfopen(path, mode, compression_spec);
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
if (fp == NULL)
|
if (fp == NULL)
|
||||||
{
|
{
|
||||||
char *fname;
|
char *fname;
|
||||||
|
|
||||||
fname = psprintf("%s.gz", path);
|
fname = psprintf("%s.gz", path);
|
||||||
fp = cfopen(fname, mode, 1);
|
compression_spec.algorithm = PG_COMPRESSION_GZIP;
|
||||||
|
fp = cfopen(fname, mode, compression_spec);
|
||||||
free_keep_errno(fname);
|
free_keep_errno(fname);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -479,26 +458,27 @@ cfopen_read(const char *path, const char *mode)
|
||||||
* be a filemode as accepted by fopen() and gzopen() that indicates writing
|
* be a filemode as accepted by fopen() and gzopen() that indicates writing
|
||||||
* ("w", "wb", "a", or "ab").
|
* ("w", "wb", "a", or "ab").
|
||||||
*
|
*
|
||||||
* If 'compression' is non-zero, a gzip compressed stream is opened, and
|
* If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
|
||||||
* 'compression' indicates the compression level used. The ".gz" suffix
|
* and 'compression_spec.level' used. The ".gz" suffix is automatically added to
|
||||||
* is automatically added to 'path' in that case.
|
* 'path' in that case.
|
||||||
*
|
*
|
||||||
* On failure, return NULL with an error code in errno.
|
* On failure, return NULL with an error code in errno.
|
||||||
*/
|
*/
|
||||||
cfp *
|
cfp *
|
||||||
cfopen_write(const char *path, const char *mode, int compression)
|
cfopen_write(const char *path, const char *mode,
|
||||||
|
const pg_compress_specification compression_spec)
|
||||||
{
|
{
|
||||||
cfp *fp;
|
cfp *fp;
|
||||||
|
|
||||||
if (compression == 0)
|
if (compression_spec.algorithm == PG_COMPRESSION_NONE)
|
||||||
fp = cfopen(path, mode, 0);
|
fp = cfopen(path, mode, compression_spec);
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
char *fname;
|
char *fname;
|
||||||
|
|
||||||
fname = psprintf("%s.gz", path);
|
fname = psprintf("%s.gz", path);
|
||||||
fp = cfopen(fname, mode, compression);
|
fp = cfopen(fname, mode, compression_spec);
|
||||||
free_keep_errno(fname);
|
free_keep_errno(fname);
|
||||||
#else
|
#else
|
||||||
pg_fatal("not built with zlib support");
|
pg_fatal("not built with zlib support");
|
||||||
|
@ -509,26 +489,27 @@ cfopen_write(const char *path, const char *mode, int compression)
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
|
* Opens file 'path' in 'mode'. If compression is GZIP, the file
|
||||||
* is opened with libz gzopen(), otherwise with plain fopen().
|
* is opened with libz gzopen(), otherwise with plain fopen().
|
||||||
*
|
*
|
||||||
* On failure, return NULL with an error code in errno.
|
* On failure, return NULL with an error code in errno.
|
||||||
*/
|
*/
|
||||||
cfp *
|
cfp *
|
||||||
cfopen(const char *path, const char *mode, int compression)
|
cfopen(const char *path, const char *mode,
|
||||||
|
const pg_compress_specification compression_spec)
|
||||||
{
|
{
|
||||||
cfp *fp = pg_malloc(sizeof(cfp));
|
cfp *fp = pg_malloc(sizeof(cfp));
|
||||||
|
|
||||||
if (compression != 0)
|
if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
|
||||||
{
|
{
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
if (compression != Z_DEFAULT_COMPRESSION)
|
if (compression_spec.level != Z_DEFAULT_COMPRESSION)
|
||||||
{
|
{
|
||||||
/* user has specified a compression level, so tell zlib to use it */
|
/* user has specified a compression level, so tell zlib to use it */
|
||||||
char mode_compression[32];
|
char mode_compression[32];
|
||||||
|
|
||||||
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
|
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
|
||||||
mode, compression);
|
mode, compression_spec.level);
|
||||||
fp->compressedfp = gzopen(path, mode_compression);
|
fp->compressedfp = gzopen(path, mode_compression);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|
|
@ -21,12 +21,6 @@
|
||||||
#define ZLIB_OUT_SIZE 4096
|
#define ZLIB_OUT_SIZE 4096
|
||||||
#define ZLIB_IN_SIZE 4096
|
#define ZLIB_IN_SIZE 4096
|
||||||
|
|
||||||
typedef enum
|
|
||||||
{
|
|
||||||
COMPR_ALG_NONE,
|
|
||||||
COMPR_ALG_LIBZ
|
|
||||||
} CompressionAlgorithm;
|
|
||||||
|
|
||||||
/* Prototype for callback function to WriteDataToArchive() */
|
/* Prototype for callback function to WriteDataToArchive() */
|
||||||
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
|
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
|
||||||
|
|
||||||
|
@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
|
||||||
/* struct definition appears in compress_io.c */
|
/* struct definition appears in compress_io.c */
|
||||||
typedef struct CompressorState CompressorState;
|
typedef struct CompressorState CompressorState;
|
||||||
|
|
||||||
extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
|
extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
|
||||||
extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
|
WriteFunc writeF);
|
||||||
|
extern void ReadDataFromArchive(ArchiveHandle *AH,
|
||||||
|
const pg_compress_specification compression_spec,
|
||||||
ReadFunc readF);
|
ReadFunc readF);
|
||||||
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
|
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
|
||||||
const void *data, size_t dLen);
|
const void *data, size_t dLen);
|
||||||
|
@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
|
||||||
|
|
||||||
typedef struct cfp cfp;
|
typedef struct cfp cfp;
|
||||||
|
|
||||||
extern cfp *cfopen(const char *path, const char *mode, int compression);
|
extern cfp *cfopen(const char *path, const char *mode,
|
||||||
|
const pg_compress_specification compression_spec);
|
||||||
|
extern cfp *cfdopen(int fd, const char *mode,
|
||||||
|
pg_compress_specification compression_spec);
|
||||||
extern cfp *cfopen_read(const char *path, const char *mode);
|
extern cfp *cfopen_read(const char *path, const char *mode);
|
||||||
extern cfp *cfopen_write(const char *path, const char *mode, int compression);
|
extern cfp *cfopen_write(const char *path, const char *mode,
|
||||||
|
const pg_compress_specification compression_spec);
|
||||||
extern int cfread(void *ptr, int size, cfp *fp);
|
extern int cfread(void *ptr, int size, cfp *fp);
|
||||||
extern int cfwrite(const void *ptr, int size, cfp *fp);
|
extern int cfwrite(const void *ptr, int size, cfp *fp);
|
||||||
extern int cfgetc(cfp *fp);
|
extern int cfgetc(cfp *fp);
|
||||||
|
|
|
@ -23,6 +23,7 @@
|
||||||
#ifndef PG_BACKUP_H
|
#ifndef PG_BACKUP_H
|
||||||
#define PG_BACKUP_H
|
#define PG_BACKUP_H
|
||||||
|
|
||||||
|
#include "common/compression.h"
|
||||||
#include "fe_utils/simple_list.h"
|
#include "fe_utils/simple_list.h"
|
||||||
#include "libpq-fe.h"
|
#include "libpq-fe.h"
|
||||||
|
|
||||||
|
@ -143,7 +144,8 @@ typedef struct _restoreOptions
|
||||||
|
|
||||||
int noDataForFailedTables;
|
int noDataForFailedTables;
|
||||||
int exit_on_error;
|
int exit_on_error;
|
||||||
int compression;
|
pg_compress_specification compression_spec; /* Specification for
|
||||||
|
* compression */
|
||||||
int suppressDumpWarnings; /* Suppress output of WARNING entries
|
int suppressDumpWarnings; /* Suppress output of WARNING entries
|
||||||
* to stderr */
|
* to stderr */
|
||||||
bool single_txn;
|
bool single_txn;
|
||||||
|
@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
|
||||||
|
|
||||||
/* Create a new archive */
|
/* Create a new archive */
|
||||||
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
|
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
|
||||||
const int compression, bool dosync, ArchiveMode mode,
|
const pg_compress_specification compression_spec,
|
||||||
|
bool dosync, ArchiveMode mode,
|
||||||
SetupWorkerPtrType setupDumpWorker);
|
SetupWorkerPtrType setupDumpWorker);
|
||||||
|
|
||||||
/* The --list option */
|
/* The --list option */
|
||||||
|
|
|
@ -70,7 +70,8 @@ typedef struct _parallelReadyList
|
||||||
|
|
||||||
|
|
||||||
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
|
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
|
||||||
const int compression, bool dosync, ArchiveMode mode,
|
const pg_compress_specification compression_spec,
|
||||||
|
bool dosync, ArchiveMode mode,
|
||||||
SetupWorkerPtrType setupWorkerPtr);
|
SetupWorkerPtrType setupWorkerPtr);
|
||||||
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
|
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
|
||||||
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
|
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
|
||||||
|
@ -98,7 +99,8 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
|
||||||
static int RestoringToDB(ArchiveHandle *AH);
|
static int RestoringToDB(ArchiveHandle *AH);
|
||||||
static void dump_lo_buf(ArchiveHandle *AH);
|
static void dump_lo_buf(ArchiveHandle *AH);
|
||||||
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
|
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
|
||||||
static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
|
static void SetOutput(ArchiveHandle *AH, const char *filename,
|
||||||
|
const pg_compress_specification compression_spec);
|
||||||
static OutputContext SaveOutput(ArchiveHandle *AH);
|
static OutputContext SaveOutput(ArchiveHandle *AH);
|
||||||
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
|
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
|
||||||
|
|
||||||
|
@ -239,12 +241,13 @@ setupRestoreWorker(Archive *AHX)
|
||||||
/* Public */
|
/* Public */
|
||||||
Archive *
|
Archive *
|
||||||
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
|
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
|
||||||
const int compression, bool dosync, ArchiveMode mode,
|
const pg_compress_specification compression_spec,
|
||||||
|
bool dosync, ArchiveMode mode,
|
||||||
SetupWorkerPtrType setupDumpWorker)
|
SetupWorkerPtrType setupDumpWorker)
|
||||||
|
|
||||||
{
|
{
|
||||||
ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
|
ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression_spec,
|
||||||
mode, setupDumpWorker);
|
dosync, mode, setupDumpWorker);
|
||||||
|
|
||||||
return (Archive *) AH;
|
return (Archive *) AH;
|
||||||
}
|
}
|
||||||
|
@ -254,7 +257,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
|
||||||
Archive *
|
Archive *
|
||||||
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
|
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
|
||||||
{
|
{
|
||||||
ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
|
ArchiveHandle *AH;
|
||||||
|
pg_compress_specification compression_spec = {0};
|
||||||
|
|
||||||
|
compression_spec.algorithm = PG_COMPRESSION_NONE;
|
||||||
|
AH = _allocAH(FileSpec, fmt, compression_spec, true,
|
||||||
|
archModeRead, setupRestoreWorker);
|
||||||
|
|
||||||
return (Archive *) AH;
|
return (Archive *) AH;
|
||||||
}
|
}
|
||||||
|
@ -384,7 +392,8 @@ RestoreArchive(Archive *AHX)
|
||||||
* Make sure we won't need (de)compression we haven't got
|
* Make sure we won't need (de)compression we haven't got
|
||||||
*/
|
*/
|
||||||
#ifndef HAVE_LIBZ
|
#ifndef HAVE_LIBZ
|
||||||
if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
|
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
|
||||||
|
AH->PrintTocDataPtr != NULL)
|
||||||
{
|
{
|
||||||
for (te = AH->toc->next; te != AH->toc; te = te->next)
|
for (te = AH->toc->next; te != AH->toc; te = te->next)
|
||||||
{
|
{
|
||||||
|
@ -459,8 +468,8 @@ RestoreArchive(Archive *AHX)
|
||||||
* Setup the output file if necessary.
|
* Setup the output file if necessary.
|
||||||
*/
|
*/
|
||||||
sav = SaveOutput(AH);
|
sav = SaveOutput(AH);
|
||||||
if (ropt->filename || ropt->compression)
|
if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE)
|
||||||
SetOutput(AH, ropt->filename, ropt->compression);
|
SetOutput(AH, ropt->filename, ropt->compression_spec);
|
||||||
|
|
||||||
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
|
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
|
||||||
|
|
||||||
|
@ -739,7 +748,7 @@ RestoreArchive(Archive *AHX)
|
||||||
*/
|
*/
|
||||||
AH->stage = STAGE_FINALIZING;
|
AH->stage = STAGE_FINALIZING;
|
||||||
|
|
||||||
if (ropt->filename || ropt->compression)
|
if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE)
|
||||||
RestoreOutput(AH, sav);
|
RestoreOutput(AH, sav);
|
||||||
|
|
||||||
if (ropt->useDB)
|
if (ropt->useDB)
|
||||||
|
@ -969,6 +978,8 @@ NewRestoreOptions(void)
|
||||||
opts->format = archUnknown;
|
opts->format = archUnknown;
|
||||||
opts->cparams.promptPassword = TRI_DEFAULT;
|
opts->cparams.promptPassword = TRI_DEFAULT;
|
||||||
opts->dumpSections = DUMP_UNSECTIONED;
|
opts->dumpSections = DUMP_UNSECTIONED;
|
||||||
|
opts->compression_spec.algorithm = PG_COMPRESSION_NONE;
|
||||||
|
opts->compression_spec.level = 0;
|
||||||
|
|
||||||
return opts;
|
return opts;
|
||||||
}
|
}
|
||||||
|
@ -1115,14 +1126,18 @@ PrintTOCSummary(Archive *AHX)
|
||||||
ArchiveHandle *AH = (ArchiveHandle *) AHX;
|
ArchiveHandle *AH = (ArchiveHandle *) AHX;
|
||||||
RestoreOptions *ropt = AH->public.ropt;
|
RestoreOptions *ropt = AH->public.ropt;
|
||||||
TocEntry *te;
|
TocEntry *te;
|
||||||
|
pg_compress_specification out_compression_spec = {0};
|
||||||
teSection curSection;
|
teSection curSection;
|
||||||
OutputContext sav;
|
OutputContext sav;
|
||||||
const char *fmtName;
|
const char *fmtName;
|
||||||
char stamp_str[64];
|
char stamp_str[64];
|
||||||
|
|
||||||
|
/* TOC is always uncompressed */
|
||||||
|
out_compression_spec.algorithm = PG_COMPRESSION_NONE;
|
||||||
|
|
||||||
sav = SaveOutput(AH);
|
sav = SaveOutput(AH);
|
||||||
if (ropt->filename)
|
if (ropt->filename)
|
||||||
SetOutput(AH, ropt->filename, 0 /* no compression */ );
|
SetOutput(AH, ropt->filename, out_compression_spec);
|
||||||
|
|
||||||
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
|
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
|
||||||
localtime(&AH->createDate)) == 0)
|
localtime(&AH->createDate)) == 0)
|
||||||
|
@ -1131,7 +1146,7 @@ PrintTOCSummary(Archive *AHX)
|
||||||
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
|
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
|
||||||
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
|
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
|
||||||
sanitize_line(AH->archdbname, false),
|
sanitize_line(AH->archdbname, false),
|
||||||
AH->tocCount, AH->compression);
|
AH->tocCount, AH->compression_spec.level);
|
||||||
|
|
||||||
switch (AH->format)
|
switch (AH->format)
|
||||||
{
|
{
|
||||||
|
@ -1485,7 +1500,8 @@ archprintf(Archive *AH, const char *fmt,...)
|
||||||
*******************************/
|
*******************************/
|
||||||
|
|
||||||
static void
|
static void
|
||||||
SetOutput(ArchiveHandle *AH, const char *filename, int compression)
|
SetOutput(ArchiveHandle *AH, const char *filename,
|
||||||
|
const pg_compress_specification compression_spec)
|
||||||
{
|
{
|
||||||
int fn;
|
int fn;
|
||||||
|
|
||||||
|
@ -1508,12 +1524,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
|
||||||
|
|
||||||
/* If compression explicitly requested, use gzopen */
|
/* If compression explicitly requested, use gzopen */
|
||||||
#ifdef HAVE_LIBZ
|
#ifdef HAVE_LIBZ
|
||||||
if (compression != 0)
|
if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
|
||||||
{
|
{
|
||||||
char fmode[14];
|
char fmode[14];
|
||||||
|
|
||||||
/* Don't use PG_BINARY_x since this is zlib */
|
/* Don't use PG_BINARY_x since this is zlib */
|
||||||
sprintf(fmode, "wb%d", compression);
|
sprintf(fmode, "wb%d", compression_spec.level);
|
||||||
if (fn >= 0)
|
if (fn >= 0)
|
||||||
AH->OF = gzdopen(dup(fn), fmode);
|
AH->OF = gzdopen(dup(fn), fmode);
|
||||||
else
|
else
|
||||||
|
@ -2198,7 +2214,8 @@ _discoverArchiveFormat(ArchiveHandle *AH)
|
||||||
*/
|
*/
|
||||||
static ArchiveHandle *
|
static ArchiveHandle *
|
||||||
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
|
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
|
||||||
const int compression, bool dosync, ArchiveMode mode,
|
const pg_compress_specification compression_spec,
|
||||||
|
bool dosync, ArchiveMode mode,
|
||||||
SetupWorkerPtrType setupWorkerPtr)
|
SetupWorkerPtrType setupWorkerPtr)
|
||||||
{
|
{
|
||||||
ArchiveHandle *AH;
|
ArchiveHandle *AH;
|
||||||
|
@ -2249,7 +2266,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
|
||||||
AH->toc->prev = AH->toc;
|
AH->toc->prev = AH->toc;
|
||||||
|
|
||||||
AH->mode = mode;
|
AH->mode = mode;
|
||||||
AH->compression = compression;
|
AH->compression_spec = compression_spec;
|
||||||
AH->dosync = dosync;
|
AH->dosync = dosync;
|
||||||
|
|
||||||
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
|
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
|
||||||
|
@ -2264,7 +2281,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
|
||||||
* Force stdin/stdout into binary mode if that is what we are using.
|
* Force stdin/stdout into binary mode if that is what we are using.
|
||||||
*/
|
*/
|
||||||
#ifdef WIN32
|
#ifdef WIN32
|
||||||
if ((fmt != archNull || compression != 0) &&
|
if ((fmt != archNull || compression_spec.algorithm != PG_COMPRESSION_NONE) &&
|
||||||
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
|
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
|
||||||
{
|
{
|
||||||
if (mode == archModeWrite)
|
if (mode == archModeWrite)
|
||||||
|
@ -3669,7 +3686,12 @@ WriteHead(ArchiveHandle *AH)
|
||||||
AH->WriteBytePtr(AH, AH->intSize);
|
AH->WriteBytePtr(AH, AH->intSize);
|
||||||
AH->WriteBytePtr(AH, AH->offSize);
|
AH->WriteBytePtr(AH, AH->offSize);
|
||||||
AH->WriteBytePtr(AH, AH->format);
|
AH->WriteBytePtr(AH, AH->format);
|
||||||
WriteInt(AH, AH->compression);
|
/*
|
||||||
|
* For now the compression type is implied by the level. This will need
|
||||||
|
* to change once support for more compression algorithms is added,
|
||||||
|
* requiring a format bump.
|
||||||
|
*/
|
||||||
|
WriteInt(AH, AH->compression_spec.level);
|
||||||
crtm = *localtime(&AH->createDate);
|
crtm = *localtime(&AH->createDate);
|
||||||
WriteInt(AH, crtm.tm_sec);
|
WriteInt(AH, crtm.tm_sec);
|
||||||
WriteInt(AH, crtm.tm_min);
|
WriteInt(AH, crtm.tm_min);
|
||||||
|
@ -3740,19 +3762,24 @@ ReadHead(ArchiveHandle *AH)
|
||||||
pg_fatal("expected format (%d) differs from format found in file (%d)",
|
pg_fatal("expected format (%d) differs from format found in file (%d)",
|
||||||
AH->format, fmt);
|
AH->format, fmt);
|
||||||
|
|
||||||
|
/* Guess the compression method based on the level */
|
||||||
|
AH->compression_spec.algorithm = PG_COMPRESSION_NONE;
|
||||||
if (AH->version >= K_VERS_1_2)
|
if (AH->version >= K_VERS_1_2)
|
||||||
{
|
{
|
||||||
if (AH->version < K_VERS_1_4)
|
if (AH->version < K_VERS_1_4)
|
||||||
AH->compression = AH->ReadBytePtr(AH);
|
AH->compression_spec.level = AH->ReadBytePtr(AH);
|
||||||
else
|
else
|
||||||
AH->compression = ReadInt(AH);
|
AH->compression_spec.level = ReadInt(AH);
|
||||||
|
|
||||||
|
if (AH->compression_spec.level != 0)
|
||||||
|
AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
AH->compression = Z_DEFAULT_COMPRESSION;
|
AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
|
||||||
|
|
||||||
#ifndef HAVE_LIBZ
|
#ifndef HAVE_LIBZ
|
||||||
if (AH->compression != 0)
|
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
|
||||||
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
|
pg_fatal("archive is compressed, but this installation does not support compression");
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
if (AH->version >= K_VERS_1_4)
|
if (AH->version >= K_VERS_1_4)
|
||||||
|
|
|
@ -331,14 +331,8 @@ struct _archiveHandle
|
||||||
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
|
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
|
||||||
|
|
||||||
struct _tocEntry *currToc; /* Used when dumping data */
|
struct _tocEntry *currToc; /* Used when dumping data */
|
||||||
int compression; /*---------
|
pg_compress_specification compression_spec; /* Requested specification for
|
||||||
* Compression requested on open().
|
* compression */
|
||||||
* Possible values for compression:
|
|
||||||
* -1 Z_DEFAULT_COMPRESSION
|
|
||||||
* 0 COMPRESSION_NONE
|
|
||||||
* 1-9 levels for gzip compression
|
|
||||||
*---------
|
|
||||||
*/
|
|
||||||
bool dosync; /* data requested to be synced on sight */
|
bool dosync; /* data requested to be synced on sight */
|
||||||
ArchiveMode mode; /* File mode - r or w */
|
ArchiveMode mode; /* File mode - r or w */
|
||||||
void *formatData; /* Header data specific to file format */
|
void *formatData; /* Header data specific to file format */
|
||||||
|
|
|
@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
|
||||||
_WriteByte(AH, BLK_DATA); /* Block type */
|
_WriteByte(AH, BLK_DATA); /* Block type */
|
||||||
WriteInt(AH, te->dumpId); /* For sanity check */
|
WriteInt(AH, te->dumpId); /* For sanity check */
|
||||||
|
|
||||||
ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
|
ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
|
||||||
|
|
||||||
WriteInt(AH, oid);
|
WriteInt(AH, oid);
|
||||||
|
|
||||||
ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
|
ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
|
||||||
static void
|
static void
|
||||||
_PrintData(ArchiveHandle *AH)
|
_PrintData(ArchiveHandle *AH)
|
||||||
{
|
{
|
||||||
ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
|
ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
|
|
|
@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
|
||||||
|
|
||||||
setFilePath(AH, fname, tctx->filename);
|
setFilePath(AH, fname, tctx->filename);
|
||||||
|
|
||||||
ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
|
ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
|
||||||
|
AH->compression_spec);
|
||||||
if (ctx->dataFH == NULL)
|
if (ctx->dataFH == NULL)
|
||||||
pg_fatal("could not open output file \"%s\": %m", fname);
|
pg_fatal("could not open output file \"%s\": %m", fname);
|
||||||
}
|
}
|
||||||
|
@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
|
||||||
if (AH->mode == archModeWrite)
|
if (AH->mode == archModeWrite)
|
||||||
{
|
{
|
||||||
cfp *tocFH;
|
cfp *tocFH;
|
||||||
|
pg_compress_specification compression_spec = {0};
|
||||||
char fname[MAXPGPATH];
|
char fname[MAXPGPATH];
|
||||||
|
|
||||||
setFilePath(AH, fname, "toc.dat");
|
setFilePath(AH, fname, "toc.dat");
|
||||||
|
@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
|
||||||
ctx->pstate = ParallelBackupStart(AH);
|
ctx->pstate = ParallelBackupStart(AH);
|
||||||
|
|
||||||
/* The TOC is always created uncompressed */
|
/* The TOC is always created uncompressed */
|
||||||
tocFH = cfopen_write(fname, PG_BINARY_W, 0);
|
compression_spec.algorithm = PG_COMPRESSION_NONE;
|
||||||
|
tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
|
||||||
if (tocFH == NULL)
|
if (tocFH == NULL)
|
||||||
pg_fatal("could not open output file \"%s\": %m", fname);
|
pg_fatal("could not open output file \"%s\": %m", fname);
|
||||||
ctx->dataFH = tocFH;
|
ctx->dataFH = tocFH;
|
||||||
|
@ -639,12 +642,14 @@ static void
|
||||||
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
|
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
|
||||||
{
|
{
|
||||||
lclContext *ctx = (lclContext *) AH->formatData;
|
lclContext *ctx = (lclContext *) AH->formatData;
|
||||||
|
pg_compress_specification compression_spec = {0};
|
||||||
char fname[MAXPGPATH];
|
char fname[MAXPGPATH];
|
||||||
|
|
||||||
setFilePath(AH, fname, "blobs.toc");
|
setFilePath(AH, fname, "blobs.toc");
|
||||||
|
|
||||||
/* The blob TOC file is never compressed */
|
/* The blob TOC file is never compressed */
|
||||||
ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
|
compression_spec.algorithm = PG_COMPRESSION_NONE;
|
||||||
|
ctx->blobsTocFH = cfopen_write(fname, "ab", compression_spec);
|
||||||
if (ctx->blobsTocFH == NULL)
|
if (ctx->blobsTocFH == NULL)
|
||||||
pg_fatal("could not open output file \"%s\": %m", fname);
|
pg_fatal("could not open output file \"%s\": %m", fname);
|
||||||
}
|
}
|
||||||
|
@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
|
||||||
|
|
||||||
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
|
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
|
||||||
|
|
||||||
ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
|
ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
|
||||||
|
|
||||||
if (ctx->dataFH == NULL)
|
if (ctx->dataFH == NULL)
|
||||||
pg_fatal("could not open output file \"%s\": %m", fname);
|
pg_fatal("could not open output file \"%s\": %m", fname);
|
||||||
|
|
|
@ -194,7 +194,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
|
||||||
* possible since gzdopen uses buffered IO which totally screws file
|
* possible since gzdopen uses buffered IO which totally screws file
|
||||||
* positioning.
|
* positioning.
|
||||||
*/
|
*/
|
||||||
if (AH->compression != 0)
|
if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
|
||||||
pg_fatal("compression is not supported by tar archive format");
|
pg_fatal("compression is not supported by tar archive format");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -328,7 +328,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (AH->compression == 0)
|
if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
|
||||||
tm->nFH = ctx->tarFH;
|
tm->nFH = ctx->tarFH;
|
||||||
else
|
else
|
||||||
pg_fatal("compression is not supported by tar archive format");
|
pg_fatal("compression is not supported by tar archive format");
|
||||||
|
@ -383,7 +383,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
|
||||||
|
|
||||||
umask(old_umask);
|
umask(old_umask);
|
||||||
|
|
||||||
if (AH->compression == 0)
|
if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
|
||||||
tm->nFH = tm->tmpFH;
|
tm->nFH = tm->tmpFH;
|
||||||
else
|
else
|
||||||
pg_fatal("compression is not supported by tar archive format");
|
pg_fatal("compression is not supported by tar archive format");
|
||||||
|
@ -401,7 +401,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
|
||||||
static void
|
static void
|
||||||
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
|
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
|
||||||
{
|
{
|
||||||
if (AH->compression != 0)
|
if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
|
||||||
pg_fatal("compression is not supported by tar archive format");
|
pg_fatal("compression is not supported by tar archive format");
|
||||||
|
|
||||||
if (th->mode == 'w')
|
if (th->mode == 'w')
|
||||||
|
@ -800,7 +800,6 @@ _CloseArchive(ArchiveHandle *AH)
|
||||||
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
|
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
|
||||||
ropt->filename = NULL;
|
ropt->filename = NULL;
|
||||||
ropt->dropSchema = 1;
|
ropt->dropSchema = 1;
|
||||||
ropt->compression = 0;
|
|
||||||
ropt->superuser = NULL;
|
ropt->superuser = NULL;
|
||||||
ropt->suppressDumpWarnings = true;
|
ropt->suppressDumpWarnings = true;
|
||||||
|
|
||||||
|
@ -888,7 +887,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
|
||||||
if (oid == 0)
|
if (oid == 0)
|
||||||
pg_fatal("invalid OID for large object (%u)", oid);
|
pg_fatal("invalid OID for large object (%u)", oid);
|
||||||
|
|
||||||
if (AH->compression != 0)
|
if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
|
||||||
pg_fatal("compression is not supported by tar archive format");
|
pg_fatal("compression is not supported by tar archive format");
|
||||||
|
|
||||||
sprintf(fname, "blob_%u.dat", oid);
|
sprintf(fname, "blob_%u.dat", oid);
|
||||||
|
|
|
@ -105,6 +105,8 @@ static Oid g_last_builtin_oid; /* value of the last builtin oid */
|
||||||
/* The specified names/patterns should to match at least one entity */
|
/* The specified names/patterns should to match at least one entity */
|
||||||
static int strict_names = 0;
|
static int strict_names = 0;
|
||||||
|
|
||||||
|
static pg_compress_algorithm compression_algorithm = PG_COMPRESSION_NONE;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Object inclusion/exclusion lists
|
* Object inclusion/exclusion lists
|
||||||
*
|
*
|
||||||
|
@ -340,10 +342,14 @@ main(int argc, char **argv)
|
||||||
const char *dumpsnapshot = NULL;
|
const char *dumpsnapshot = NULL;
|
||||||
char *use_role = NULL;
|
char *use_role = NULL;
|
||||||
int numWorkers = 1;
|
int numWorkers = 1;
|
||||||
int compressLevel = -1;
|
|
||||||
int plainText = 0;
|
int plainText = 0;
|
||||||
ArchiveFormat archiveFormat = archUnknown;
|
ArchiveFormat archiveFormat = archUnknown;
|
||||||
ArchiveMode archiveMode;
|
ArchiveMode archiveMode;
|
||||||
|
pg_compress_specification compression_spec = {0};
|
||||||
|
char *compression_detail = NULL;
|
||||||
|
char *compression_algorithm_str = "none";
|
||||||
|
char *error_detail = NULL;
|
||||||
|
bool user_compression_defined = false;
|
||||||
|
|
||||||
static DumpOptions dopt;
|
static DumpOptions dopt;
|
||||||
|
|
||||||
|
@ -561,10 +567,10 @@ main(int argc, char **argv)
|
||||||
dopt.aclsSkip = true;
|
dopt.aclsSkip = true;
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'Z': /* Compression Level */
|
case 'Z': /* Compression */
|
||||||
if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
|
parse_compress_options(optarg, &compression_algorithm_str,
|
||||||
&compressLevel))
|
&compression_detail);
|
||||||
exit_nicely(1);
|
user_compression_defined = true;
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 0:
|
case 0:
|
||||||
|
@ -687,22 +693,49 @@ main(int argc, char **argv)
|
||||||
if (archiveFormat == archNull)
|
if (archiveFormat == archNull)
|
||||||
plainText = 1;
|
plainText = 1;
|
||||||
|
|
||||||
/* Custom and directory formats are compressed by default, others not */
|
/*
|
||||||
if (compressLevel == -1)
|
* Compression options
|
||||||
|
*/
|
||||||
|
if (!parse_compress_algorithm(compression_algorithm_str,
|
||||||
|
&compression_algorithm))
|
||||||
|
pg_fatal("unrecognized compression algorithm: \"%s\"",
|
||||||
|
compression_algorithm_str);
|
||||||
|
|
||||||
|
parse_compress_specification(compression_algorithm, compression_detail,
|
||||||
|
&compression_spec);
|
||||||
|
error_detail = validate_compress_specification(&compression_spec);
|
||||||
|
if (error_detail != NULL)
|
||||||
|
pg_fatal("invalid compression specification: %s",
|
||||||
|
error_detail);
|
||||||
|
|
||||||
|
switch (compression_algorithm)
|
||||||
{
|
{
|
||||||
#ifdef HAVE_LIBZ
|
case PG_COMPRESSION_NONE:
|
||||||
if (archiveFormat == archCustom || archiveFormat == archDirectory)
|
/* fallthrough */
|
||||||
compressLevel = Z_DEFAULT_COMPRESSION;
|
case PG_COMPRESSION_GZIP:
|
||||||
else
|
break;
|
||||||
#endif
|
case PG_COMPRESSION_ZSTD:
|
||||||
compressLevel = 0;
|
pg_fatal("compression with %s is not yet supported", "ZSTD");
|
||||||
|
break;
|
||||||
|
case PG_COMPRESSION_LZ4:
|
||||||
|
pg_fatal("compression with %s is not yet supported", "LZ4");
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifndef HAVE_LIBZ
|
/*
|
||||||
if (compressLevel != 0)
|
* Custom and directory formats are compressed by default with gzip when
|
||||||
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
|
* available, not the others.
|
||||||
compressLevel = 0;
|
*/
|
||||||
|
if ((archiveFormat == archCustom || archiveFormat == archDirectory) &&
|
||||||
|
!user_compression_defined)
|
||||||
|
{
|
||||||
|
#ifdef HAVE_LIBZ
|
||||||
|
parse_compress_specification(PG_COMPRESSION_GZIP, NULL,
|
||||||
|
&compression_spec);
|
||||||
|
#else
|
||||||
|
/* Nothing to do in the default case */
|
||||||
#endif
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If emitting an archive format, we always want to emit a DATABASE item,
|
* If emitting an archive format, we always want to emit a DATABASE item,
|
||||||
|
@ -716,8 +749,8 @@ main(int argc, char **argv)
|
||||||
pg_fatal("parallel backup only supported by the directory format");
|
pg_fatal("parallel backup only supported by the directory format");
|
||||||
|
|
||||||
/* Open the output file */
|
/* Open the output file */
|
||||||
fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
|
fout = CreateArchive(filename, archiveFormat, compression_spec,
|
||||||
archiveMode, setupDumpWorker);
|
dosync, archiveMode, setupDumpWorker);
|
||||||
|
|
||||||
/* Make dump options accessible right away */
|
/* Make dump options accessible right away */
|
||||||
SetArchiveOptions(fout, &dopt, NULL);
|
SetArchiveOptions(fout, &dopt, NULL);
|
||||||
|
@ -948,10 +981,7 @@ main(int argc, char **argv)
|
||||||
ropt->sequence_data = dopt.sequence_data;
|
ropt->sequence_data = dopt.sequence_data;
|
||||||
ropt->binary_upgrade = dopt.binary_upgrade;
|
ropt->binary_upgrade = dopt.binary_upgrade;
|
||||||
|
|
||||||
if (compressLevel == -1)
|
ropt->compression_spec = compression_spec;
|
||||||
ropt->compression = 0;
|
|
||||||
else
|
|
||||||
ropt->compression = compressLevel;
|
|
||||||
|
|
||||||
ropt->suppressDumpWarnings = true; /* We've already shown them */
|
ropt->suppressDumpWarnings = true; /* We've already shown them */
|
||||||
|
|
||||||
|
@ -998,7 +1028,8 @@ help(const char *progname)
|
||||||
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
|
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
|
||||||
printf(_(" -v, --verbose verbose mode\n"));
|
printf(_(" -v, --verbose verbose mode\n"));
|
||||||
printf(_(" -V, --version output version information, then exit\n"));
|
printf(_(" -V, --version output version information, then exit\n"));
|
||||||
printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
|
printf(_(" -Z, --compress=METHOD[:LEVEL]\n"
|
||||||
|
" compress as specified\n"));
|
||||||
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
|
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
|
||||||
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
|
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
|
||||||
printf(_(" -?, --help show this help, then exit\n"));
|
printf(_(" -?, --help show this help, then exit\n"));
|
||||||
|
|
|
@ -121,24 +121,46 @@ command_fails_like(
|
||||||
'pg_restore: cannot specify both --single-transaction and multiple jobs');
|
'pg_restore: cannot specify both --single-transaction and multiple jobs');
|
||||||
|
|
||||||
command_fails_like(
|
command_fails_like(
|
||||||
[ 'pg_dump', '-Z', '-1' ],
|
[ 'pg_dump', '--compress', 'garbage' ],
|
||||||
qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
|
qr/\Qpg_dump: error: unrecognized compression algorithm/,
|
||||||
'pg_dump: -Z/--compress must be in range');
|
'pg_dump: invalid --compress');
|
||||||
|
|
||||||
|
command_fails_like(
|
||||||
|
[ 'pg_dump', '--compress', 'none:1' ],
|
||||||
|
qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/,
|
||||||
|
'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level'
|
||||||
|
);
|
||||||
|
|
||||||
|
|
||||||
if (check_pg_config("#define HAVE_LIBZ 1"))
|
if (check_pg_config("#define HAVE_LIBZ 1"))
|
||||||
{
|
{
|
||||||
|
command_fails_like(
|
||||||
|
[ 'pg_dump', '-Z', '15' ],
|
||||||
|
qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/,
|
||||||
|
'pg_dump: invalid compression specification: must be in range');
|
||||||
|
|
||||||
command_fails_like(
|
command_fails_like(
|
||||||
[ 'pg_dump', '--compress', '1', '--format', 'tar' ],
|
[ 'pg_dump', '--compress', '1', '--format', 'tar' ],
|
||||||
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
|
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
|
||||||
'pg_dump: compression is not supported by tar archive format');
|
'pg_dump: compression is not supported by tar archive format');
|
||||||
|
|
||||||
|
command_fails_like(
|
||||||
|
[ 'pg_dump', '-Z', 'gzip:nonInt' ],
|
||||||
|
qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/,
|
||||||
|
'pg_dump: invalid compression specification: must be an integer');
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
# --jobs > 1 forces an error with tar format.
|
# --jobs > 1 forces an error with tar format.
|
||||||
command_fails_like(
|
command_fails_like(
|
||||||
[ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
|
[ 'pg_dump', '--format', 'tar', '-j3' ],
|
||||||
qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
|
qr/\Qpg_dump: error: parallel backup only supported by the directory format\E/,
|
||||||
'pg_dump: warning: compression not available in this installation');
|
'pg_dump: warning: parallel backup not supported by tar format');
|
||||||
|
|
||||||
|
command_fails_like(
|
||||||
|
[ 'pg_dump', '-Z', 'gzip:nonInt', '--format', 'tar', '-j2' ],
|
||||||
|
qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option\E/,
|
||||||
|
'pg_dump: invalid compression specification: must be an integer');
|
||||||
}
|
}
|
||||||
|
|
||||||
command_fails_like(
|
command_fails_like(
|
||||||
|
|
|
@ -87,7 +87,7 @@ my %pgdump_runs = (
|
||||||
compile_option => 'gzip',
|
compile_option => 'gzip',
|
||||||
dump_cmd => [
|
dump_cmd => [
|
||||||
'pg_dump', '--jobs=2',
|
'pg_dump', '--jobs=2',
|
||||||
'--format=directory', '--compress=1',
|
'--format=directory', '--compress=gzip:1',
|
||||||
"--file=$tempdir/compression_gzip_dir", 'postgres',
|
"--file=$tempdir/compression_gzip_dir", 'postgres',
|
||||||
],
|
],
|
||||||
# Give coverage for manually compressed blob.toc files during
|
# Give coverage for manually compressed blob.toc files during
|
||||||
|
@ -200,6 +200,7 @@ my %pgdump_runs = (
|
||||||
# Do not use --no-sync to give test coverage for data sync.
|
# Do not use --no-sync to give test coverage for data sync.
|
||||||
defaults_custom_format => {
|
defaults_custom_format => {
|
||||||
test_key => 'defaults',
|
test_key => 'defaults',
|
||||||
|
compile_option => 'gzip',
|
||||||
dump_cmd => [
|
dump_cmd => [
|
||||||
'pg_dump', '-Fc', '-Z6',
|
'pg_dump', '-Fc', '-Z6',
|
||||||
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
|
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
|
||||||
|
|
|
@ -20,6 +20,10 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
|
||||||
# to define how each test should (or shouldn't) treat a result
|
# to define how each test should (or shouldn't) treat a result
|
||||||
# from a given run.
|
# from a given run.
|
||||||
#
|
#
|
||||||
|
# compile_option indicates if the commands run depend on a compilation
|
||||||
|
# option, if any. This can be used to control if tests should be
|
||||||
|
# skipped when a build dependency is not satisfied.
|
||||||
|
#
|
||||||
# test_key indicates that a given run should simply use the same
|
# test_key indicates that a given run should simply use the same
|
||||||
# set of like/unlike tests as another run, and which run that is.
|
# set of like/unlike tests as another run, and which run that is.
|
||||||
#
|
#
|
||||||
|
@ -90,6 +94,7 @@ my %pgdump_runs = (
|
||||||
},
|
},
|
||||||
defaults_custom_format => {
|
defaults_custom_format => {
|
||||||
test_key => 'defaults',
|
test_key => 'defaults',
|
||||||
|
compile_option => 'gzip',
|
||||||
dump_cmd => [
|
dump_cmd => [
|
||||||
'pg_dump', '--no-sync', '-Fc', '-Z6',
|
'pg_dump', '--no-sync', '-Fc', '-Z6',
|
||||||
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
|
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
|
||||||
|
@ -749,6 +754,8 @@ $node->start;
|
||||||
|
|
||||||
my $port = $node->port;
|
my $port = $node->port;
|
||||||
|
|
||||||
|
my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
|
||||||
|
|
||||||
#########################################
|
#########################################
|
||||||
# Set up schemas, tables, etc, to be dumped.
|
# Set up schemas, tables, etc, to be dumped.
|
||||||
|
|
||||||
|
@ -792,6 +799,15 @@ foreach my $run (sort keys %pgdump_runs)
|
||||||
|
|
||||||
my $test_key = $run;
|
my $test_key = $run;
|
||||||
|
|
||||||
|
# Skip command-level tests for gzip if there is no support for it.
|
||||||
|
if ( defined($pgdump_runs{$run}->{compile_option})
|
||||||
|
&& $pgdump_runs{$run}->{compile_option} eq 'gzip'
|
||||||
|
&& !$supports_gzip)
|
||||||
|
{
|
||||||
|
note "$run: skipped due to no gzip support";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
|
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
|
||||||
"$run: pg_dump runs");
|
"$run: pg_dump runs");
|
||||||
|
|
||||||
|
|
|
@ -428,7 +428,6 @@ CompiledExprState
|
||||||
CompositeIOData
|
CompositeIOData
|
||||||
CompositeTypeStmt
|
CompositeTypeStmt
|
||||||
CompoundAffixFlag
|
CompoundAffixFlag
|
||||||
CompressionAlgorithm
|
|
||||||
CompressionLocation
|
CompressionLocation
|
||||||
CompressorState
|
CompressorState
|
||||||
ComputeXidHorizonsResult
|
ComputeXidHorizonsResult
|
||||||
|
|
Loading…
Reference in New Issue