From e9537321a74a2b062c8f7a452314b4570913f780 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 11 Mar 2022 12:18:53 +0900 Subject: [PATCH] Add support for zstd with compression of full-page writes in WAL wal_compression gains a new value, "zstd", to allow the compression of full-page images using the compression method of the same name. Compression is done using the default level recommended by the library, as of ZSTD_CLEVEL_DEFAULT = 3. Some benchmarking has shown that it could make sense to use a level lower for the FPI compression, like 1 or 2, as the compression rate did not change much with a bit less CPU consumed, but any tests done would only cover few scenarios so it is hard to come to a clear conclusion. Anyway, there is no reason to not use the default level instead, which is the level recommended by the library so it should be fine for most cases. zstd outclasses easily pglz, and is better than LZ4 where one wants to have more compression at the cost of extra CPU but both are good enough in their own scenarios, so the choice between one or the other of these comes to a study of the workload patterns and the schema involved, mainly. This commit relies heavily on 4035cd5, that reshaped the code creating and restoring full-page writes to be aware of the compression type, making this integration straight-forward. This patch borrows some early work from Andrey Borodin, though the patch got a complete rewrite. Author: Justin Pryzby Discussion: https://postgr.es/m/20220222231948.GJ9008@telsasoft.com --- doc/src/sgml/config.sgml | 11 ++++--- doc/src/sgml/installation.sgml | 8 +++++ src/backend/access/transam/xloginsert.c | 32 ++++++++++++++++++- src/backend/access/transam/xlogreader.c | 20 ++++++++++++ src/backend/utils/misc/guc.c | 3 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/bin/pg_waldump/pg_waldump.c | 2 ++ src/include/access/xlog.h | 3 +- src/include/access/xlogrecord.h | 5 ++- 9 files changed, 78 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7ed8c82a9d..5612e80453 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3154,10 +3154,13 @@ include_dir 'conf.d' server compresses full page images written to WAL when is on or during a base backup. A compressed page image will be decompressed during WAL replay. - The supported methods are pglz and - lz4 (if PostgreSQL was - compiled with ). The default value is - off. Only superusers can change this setting. + The supported methods are pglz, + lz4 (if PostgreSQL + was compiled with ) and + zstd (if PostgreSQL + was compiled with ) and + The default value is off. + Only superusers can change this setting. diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 0f74252590..a239bbef2f 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -271,6 +271,14 @@ su - postgres + + + You need zstd, if you want to support + compression of data with this method; see + . + + + To build the PostgreSQL documentation, diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 83d40b55e6..f4eb54b63c 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -23,6 +23,10 @@ #include #endif +#ifdef USE_ZSTD +#include +#endif + #include "access/xact.h" #include "access/xlog.h" #include "access/xlog_internal.h" @@ -47,9 +51,16 @@ #define LZ4_MAX_BLCKSZ 0 #endif +#ifdef USE_ZSTD +#define ZSTD_MAX_BLCKSZ ZSTD_COMPRESSBOUND(BLCKSZ) +#else +#define ZSTD_MAX_BLCKSZ 0 +#endif + #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, LZ4_MAX_BLCKSZ) +/* Buffer size required to store a compressed version of backup block image */ +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, LZ4_MAX_BLCKSZ), ZSTD_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -698,6 +709,14 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, #endif break; + case WAL_COMPRESSION_ZSTD: +#ifdef USE_ZSTD + bimg.bimg_info |= BKPIMAGE_COMPRESS_ZSTD; +#else + elog(ERROR, "zstd is not supported by this build"); +#endif + break; + case WAL_COMPRESSION_NONE: Assert(false); /* cannot happen */ break; @@ -906,6 +925,17 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, #endif break; + case WAL_COMPRESSION_ZSTD: +#ifdef USE_ZSTD + len = ZSTD_compress(dest, COMPRESS_BUFSIZE, source, orig_len, + ZSTD_CLEVEL_DEFAULT); + if (ZSTD_isError(len)) + len = -1; /* failure */ +#else + elog(ERROR, "zstd is not supported by this build"); +#endif + break; + case WAL_COMPRESSION_NONE: Assert(false); /* cannot happen */ break; diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 35029cf97d..b7c06da255 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -21,6 +21,9 @@ #ifdef USE_LZ4 #include #endif +#ifdef USE_ZSTD +#include +#endif #include "access/transam.h" #include "access/xlog_internal.h" @@ -1618,6 +1621,23 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) "LZ4", block_id); return false; +#endif + } + else if ((bkpb->bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) + { +#ifdef USE_ZSTD + size_t decomp_result = ZSTD_decompress(tmp.data, + BLCKSZ - bkpb->hole_length, + ptr, bkpb->bimg_len); + + if (ZSTD_isError(decomp_result)) + decomp_success = false; +#else + report_invalid_record(record, "image at %X/%X compressed with %s not supported by build, block %d", + LSN_FORMAT_ARGS(record->ReadRecPtr), + "zstd", + block_id); + return false; #endif } else diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 6d11f9c71b..e7f0a380e6 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -550,6 +550,9 @@ static const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef USE_LZ4 {"lz4", WAL_COMPRESSION_LZ4, false}, +#endif +#ifdef USE_ZSTD + {"zstd", WAL_COMPRESSION_ZSTD, false}, #endif {"on", WAL_COMPRESSION_PGLZ, false}, {"off", WAL_COMPRESSION_NONE, false}, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 4a094bb38b..4cf5b26a36 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -220,7 +220,7 @@ #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_compression = off # enables compression of full-page writes; - # off, pglz, lz4, or on + # off, pglz, lz4, zstd, or on #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 2340dc247b..f128050b4e 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -562,6 +562,8 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) method = "pglz"; else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) method = "lz4"; + else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) + method = "zstd"; else method = "unknown"; diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 4b45ac64db..09f6464331 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -75,7 +75,8 @@ typedef enum WalCompression { WAL_COMPRESSION_NONE = 0, WAL_COMPRESSION_PGLZ, - WAL_COMPRESSION_LZ4 + WAL_COMPRESSION_LZ4, + WAL_COMPRESSION_ZSTD } WalCompression; /* Recovery states */ diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h index c1b1137aa7..052ac6817a 100644 --- a/src/include/access/xlogrecord.h +++ b/src/include/access/xlogrecord.h @@ -149,8 +149,11 @@ typedef struct XLogRecordBlockImageHeader /* compression methods supported */ #define BKPIMAGE_COMPRESS_PGLZ 0x04 #define BKPIMAGE_COMPRESS_LZ4 0x08 +#define BKPIMAGE_COMPRESS_ZSTD 0x10 + #define BKPIMAGE_COMPRESSED(info) \ - ((info & (BKPIMAGE_COMPRESS_PGLZ | BKPIMAGE_COMPRESS_LZ4)) != 0) + ((info & (BKPIMAGE_COMPRESS_PGLZ | BKPIMAGE_COMPRESS_LZ4 | \ + BKPIMAGE_COMPRESS_ZSTD)) != 0) /* * Extra header information used when page image has "hole" and