2020-04-20 20:37:38 +02:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* backup_manifest.c
|
|
|
|
* code for generating and sending a backup manifest
|
|
|
|
*
|
2021-01-02 19:06:25 +01:00
|
|
|
* Portions Copyright (c) 2010-2021, PostgreSQL Global Development Group
|
2020-04-20 20:37:38 +02:00
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
|
|
|
* src/backend/replication/backup_manifest.c
|
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include "access/timeline.h"
|
|
|
|
#include "libpq/libpq.h"
|
|
|
|
#include "libpq/pqformat.h"
|
|
|
|
#include "mb/pg_wchar.h"
|
|
|
|
#include "replication/backup_manifest.h"
|
2021-08-19 02:20:19 +02:00
|
|
|
#include "utils/builtins.h"
|
2020-04-20 20:37:38 +02:00
|
|
|
#include "utils/json.h"
|
|
|
|
|
2020-04-23 14:44:06 +02:00
|
|
|
static void AppendStringToManifest(backup_manifest_info *manifest, char *s);
|
|
|
|
|
2020-04-20 20:37:38 +02:00
|
|
|
/*
|
|
|
|
* Does the user want a backup manifest?
|
|
|
|
*
|
|
|
|
* It's simplest to always have a manifest_info object, so that we don't need
|
|
|
|
* checks for NULL pointers in too many places. However, if the user doesn't
|
|
|
|
* want a manifest, we set manifest->buffile to NULL.
|
|
|
|
*/
|
|
|
|
static inline bool
|
2020-04-23 14:44:06 +02:00
|
|
|
IsManifestEnabled(backup_manifest_info *manifest)
|
2020-04-20 20:37:38 +02:00
|
|
|
{
|
|
|
|
return (manifest->buffile != NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Convenience macro for appending data to the backup manifest.
|
|
|
|
*/
|
|
|
|
#define AppendToManifest(manifest, ...) \
|
|
|
|
{ \
|
|
|
|
char *_manifest_s = psprintf(__VA_ARGS__); \
|
|
|
|
AppendStringToManifest(manifest, _manifest_s); \
|
|
|
|
pfree(_manifest_s); \
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize state so that we can construct a backup manifest.
|
|
|
|
*
|
|
|
|
* NB: Although the checksum type for the data files is configurable, the
|
|
|
|
* checksum for the manifest itself always uses SHA-256. See comments in
|
|
|
|
* SendBackupManifest.
|
|
|
|
*/
|
|
|
|
void
|
2020-04-23 14:44:06 +02:00
|
|
|
InitializeBackupManifest(backup_manifest_info *manifest,
|
|
|
|
backup_manifest_option want_manifest,
|
|
|
|
pg_checksum_type manifest_checksum_type)
|
2020-04-20 20:37:38 +02:00
|
|
|
{
|
2020-11-12 02:56:33 +01:00
|
|
|
memset(manifest, 0, sizeof(backup_manifest_info));
|
|
|
|
manifest->checksum_type = manifest_checksum_type;
|
|
|
|
|
2020-04-20 20:37:38 +02:00
|
|
|
if (want_manifest == MANIFEST_OPTION_NO)
|
|
|
|
manifest->buffile = NULL;
|
|
|
|
else
|
2020-11-12 02:56:33 +01:00
|
|
|
{
|
2020-04-20 20:37:38 +02:00
|
|
|
manifest->buffile = BufFileCreateTemp(false);
|
Move SHA2 routines to a new generic API layer for crypto hashes
Two new routines to allocate a hash context and to free it are created,
as these become necessary for the goal behind this refactoring: switch
the all cryptohash implementations for OpenSSL to use EVP (for FIPS and
also because upstream does not recommend the use of low-level cryptohash
functions for 20 years). Note that OpenSSL hides the internals of
cryptohash contexts since 1.1.0, so it is necessary to leave the
allocation to OpenSSL itself, explaining the need for those two new
routines. This part is going to require more work to properly track
hash contexts with resource owners, but this not introduced here.
Still, this refactoring makes the move possible.
This reduces the number of routines for all SHA2 implementations from
twelve (SHA{224,256,386,512} with init, update and final calls) to five
(create, free, init, update and final calls) by incorporating the hash
type directly into the hash context data.
The new cryptohash routines are moved to a new file, called cryptohash.c
for the fallback implementations, with SHA2 specifics becoming a part
internal to src/common/. OpenSSL specifics are part of
cryptohash_openssl.c. This infrastructure is usable for more hash
types, like MD5 or HMAC.
Any code paths using the internal SHA2 routines are adapted to report
correctly errors, which are most of the changes of this commit. The
zones mostly impacted are checksum manifests, libpq and SCRAM.
Note that e21cbb4 was a first attempt to switch SHA2 to EVP, but it
lacked the refactoring needed for libpq, as done here.
This patch has been tested on Linux and Windows, with and without
OpenSSL, and down to 1.0.1, the oldest version supported on HEAD.
Author: Michael Paquier
Reviewed-by: Daniel Gustafsson
Discussion: https://postgr.es/m/20200924025314.GE7405@paquier.xyz
2020-12-02 02:37:20 +01:00
|
|
|
manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
|
|
|
|
if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
|
2022-01-14 03:25:39 +01:00
|
|
|
elog(ERROR, "failed to initialize checksum of backup manifest");
|
2020-11-12 02:56:33 +01:00
|
|
|
}
|
|
|
|
|
2020-04-20 20:37:38 +02:00
|
|
|
manifest->manifest_size = UINT64CONST(0);
|
|
|
|
manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
|
|
|
|
manifest->first_file = true;
|
|
|
|
manifest->still_checksumming = true;
|
|
|
|
|
|
|
|
if (want_manifest != MANIFEST_OPTION_NO)
|
|
|
|
AppendToManifest(manifest,
|
|
|
|
"{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
|
|
|
|
"\"Files\": [");
|
|
|
|
}
|
|
|
|
|
Move SHA2 routines to a new generic API layer for crypto hashes
Two new routines to allocate a hash context and to free it are created,
as these become necessary for the goal behind this refactoring: switch
the all cryptohash implementations for OpenSSL to use EVP (for FIPS and
also because upstream does not recommend the use of low-level cryptohash
functions for 20 years). Note that OpenSSL hides the internals of
cryptohash contexts since 1.1.0, so it is necessary to leave the
allocation to OpenSSL itself, explaining the need for those two new
routines. This part is going to require more work to properly track
hash contexts with resource owners, but this not introduced here.
Still, this refactoring makes the move possible.
This reduces the number of routines for all SHA2 implementations from
twelve (SHA{224,256,386,512} with init, update and final calls) to five
(create, free, init, update and final calls) by incorporating the hash
type directly into the hash context data.
The new cryptohash routines are moved to a new file, called cryptohash.c
for the fallback implementations, with SHA2 specifics becoming a part
internal to src/common/. OpenSSL specifics are part of
cryptohash_openssl.c. This infrastructure is usable for more hash
types, like MD5 or HMAC.
Any code paths using the internal SHA2 routines are adapted to report
correctly errors, which are most of the changes of this commit. The
zones mostly impacted are checksum manifests, libpq and SCRAM.
Note that e21cbb4 was a first attempt to switch SHA2 to EVP, but it
lacked the refactoring needed for libpq, as done here.
This patch has been tested on Linux and Windows, with and without
OpenSSL, and down to 1.0.1, the oldest version supported on HEAD.
Author: Michael Paquier
Reviewed-by: Daniel Gustafsson
Discussion: https://postgr.es/m/20200924025314.GE7405@paquier.xyz
2020-12-02 02:37:20 +01:00
|
|
|
/*
|
|
|
|
* Free resources assigned to a backup manifest constructed.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
FreeBackupManifest(backup_manifest_info *manifest)
|
|
|
|
{
|
|
|
|
pg_cryptohash_free(manifest->manifest_ctx);
|
|
|
|
manifest->manifest_ctx = NULL;
|
|
|
|
}
|
|
|
|
|
2020-04-20 20:37:38 +02:00
|
|
|
/*
|
|
|
|
* Add an entry to the backup manifest for a file.
|
|
|
|
*/
|
|
|
|
void
|
2020-04-23 14:44:06 +02:00
|
|
|
AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid,
|
|
|
|
const char *pathname, size_t size, pg_time_t mtime,
|
|
|
|
pg_checksum_context *checksum_ctx)
|
2020-04-20 20:37:38 +02:00
|
|
|
{
|
|
|
|
char pathbuf[MAXPGPATH];
|
|
|
|
int pathlen;
|
|
|
|
StringInfoData buf;
|
|
|
|
|
|
|
|
if (!IsManifestEnabled(manifest))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If this file is part of a tablespace, the pathname passed to this
|
|
|
|
* function will be relative to the tar file that contains it. We want the
|
|
|
|
* pathname relative to the data directory (ignoring the intermediate
|
|
|
|
* symlink traversal).
|
|
|
|
*/
|
|
|
|
if (spcoid != NULL)
|
|
|
|
{
|
|
|
|
snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
|
|
|
|
pathname);
|
|
|
|
pathname = pathbuf;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Each file's entry needs to be separated from any entry that follows by
|
|
|
|
* a comma, but there's no comma before the first one or after the last
|
|
|
|
* one. To make that work, adding a file to the manifest starts by
|
|
|
|
* terminating the most recently added line, with a comma if appropriate,
|
|
|
|
* but does not terminate the line inserted for this file.
|
|
|
|
*/
|
|
|
|
initStringInfo(&buf);
|
|
|
|
if (manifest->first_file)
|
|
|
|
{
|
2020-10-15 09:35:17 +02:00
|
|
|
appendStringInfoChar(&buf, '\n');
|
2020-04-20 20:37:38 +02:00
|
|
|
manifest->first_file = false;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
appendStringInfoString(&buf, ",\n");
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write the relative pathname to this file out to the manifest. The
|
|
|
|
* manifest is always stored in UTF-8, so we have to encode paths that are
|
|
|
|
* not valid in that encoding.
|
|
|
|
*/
|
|
|
|
pathlen = strlen(pathname);
|
|
|
|
if (!manifest->force_encode &&
|
|
|
|
pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
|
|
|
|
{
|
|
|
|
appendStringInfoString(&buf, "{ \"Path\": ");
|
|
|
|
escape_json(&buf, pathname);
|
|
|
|
appendStringInfoString(&buf, ", ");
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
|
2021-08-19 02:20:19 +02:00
|
|
|
enlargeStringInfo(&buf, 2 * pathlen);
|
|
|
|
buf.len += hex_encode(pathname, pathlen,
|
|
|
|
&buf.data[buf.len]);
|
2020-04-20 20:37:38 +02:00
|
|
|
appendStringInfoString(&buf, "\", ");
|
|
|
|
}
|
|
|
|
|
|
|
|
appendStringInfo(&buf, "\"Size\": %zu, ", size);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Convert last modification time to a string and append it to the
|
|
|
|
* manifest. Since it's not clear what time zone to use and since time
|
|
|
|
* zone definitions can change, possibly causing confusion, use GMT
|
|
|
|
* always.
|
|
|
|
*/
|
|
|
|
appendStringInfoString(&buf, "\"Last-Modified\": \"");
|
|
|
|
enlargeStringInfo(&buf, 128);
|
|
|
|
buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
|
|
|
|
pg_gmtime(&mtime));
|
2020-10-15 09:35:17 +02:00
|
|
|
appendStringInfoChar(&buf, '"');
|
2020-04-20 20:37:38 +02:00
|
|
|
|
|
|
|
/* Add checksum information. */
|
|
|
|
if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
|
|
|
|
{
|
|
|
|
uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
|
|
|
|
int checksumlen;
|
|
|
|
|
|
|
|
checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
|
Move SHA2 routines to a new generic API layer for crypto hashes
Two new routines to allocate a hash context and to free it are created,
as these become necessary for the goal behind this refactoring: switch
the all cryptohash implementations for OpenSSL to use EVP (for FIPS and
also because upstream does not recommend the use of low-level cryptohash
functions for 20 years). Note that OpenSSL hides the internals of
cryptohash contexts since 1.1.0, so it is necessary to leave the
allocation to OpenSSL itself, explaining the need for those two new
routines. This part is going to require more work to properly track
hash contexts with resource owners, but this not introduced here.
Still, this refactoring makes the move possible.
This reduces the number of routines for all SHA2 implementations from
twelve (SHA{224,256,386,512} with init, update and final calls) to five
(create, free, init, update and final calls) by incorporating the hash
type directly into the hash context data.
The new cryptohash routines are moved to a new file, called cryptohash.c
for the fallback implementations, with SHA2 specifics becoming a part
internal to src/common/. OpenSSL specifics are part of
cryptohash_openssl.c. This infrastructure is usable for more hash
types, like MD5 or HMAC.
Any code paths using the internal SHA2 routines are adapted to report
correctly errors, which are most of the changes of this commit. The
zones mostly impacted are checksum manifests, libpq and SCRAM.
Note that e21cbb4 was a first attempt to switch SHA2 to EVP, but it
lacked the refactoring needed for libpq, as done here.
This patch has been tested on Linux and Windows, with and without
OpenSSL, and down to 1.0.1, the oldest version supported on HEAD.
Author: Michael Paquier
Reviewed-by: Daniel Gustafsson
Discussion: https://postgr.es/m/20200924025314.GE7405@paquier.xyz
2020-12-02 02:37:20 +01:00
|
|
|
if (checksumlen < 0)
|
|
|
|
elog(ERROR, "could not finalize checksum of file \"%s\"",
|
|
|
|
pathname);
|
2020-04-20 20:37:38 +02:00
|
|
|
|
|
|
|
appendStringInfo(&buf,
|
|
|
|
", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
|
|
|
|
pg_checksum_type_name(checksum_ctx->type));
|
2021-08-19 02:20:19 +02:00
|
|
|
enlargeStringInfo(&buf, 2 * checksumlen);
|
|
|
|
buf.len += hex_encode((char *) checksumbuf, checksumlen,
|
|
|
|
&buf.data[buf.len]);
|
2020-10-15 09:35:17 +02:00
|
|
|
appendStringInfoChar(&buf, '"');
|
2020-04-20 20:37:38 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Close out the object. */
|
|
|
|
appendStringInfoString(&buf, " }");
|
|
|
|
|
|
|
|
/* OK, add it to the manifest. */
|
|
|
|
AppendStringToManifest(manifest, buf.data);
|
|
|
|
|
|
|
|
/* Avoid leaking memory. */
|
|
|
|
pfree(buf.data);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Add information about the WAL that will need to be replayed when restoring
|
|
|
|
* this backup to the manifest.
|
|
|
|
*/
|
|
|
|
void
|
2020-04-23 14:44:06 +02:00
|
|
|
AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
|
|
|
|
TimeLineID starttli, XLogRecPtr endptr,
|
|
|
|
TimeLineID endtli)
|
2020-04-20 20:37:38 +02:00
|
|
|
{
|
|
|
|
List *timelines;
|
|
|
|
ListCell *lc;
|
|
|
|
bool first_wal_range = true;
|
|
|
|
bool found_start_timeline = false;
|
|
|
|
|
|
|
|
if (!IsManifestEnabled(manifest))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Terminate the list of files. */
|
|
|
|
AppendStringToManifest(manifest, "\n],\n");
|
|
|
|
|
|
|
|
/* Read the timeline history for the ending timeline. */
|
|
|
|
timelines = readTimeLineHistory(endtli);
|
|
|
|
|
|
|
|
/* Start a list of LSN ranges. */
|
|
|
|
AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
|
|
|
|
|
|
|
|
foreach(lc, timelines)
|
|
|
|
{
|
|
|
|
TimeLineHistoryEntry *entry = lfirst(lc);
|
|
|
|
XLogRecPtr tl_beginptr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We only care about timelines that were active during the backup.
|
|
|
|
* Skip any that ended before the backup started. (Note that if
|
|
|
|
* entry->end is InvalidXLogRecPtr, it means that the timeline has not
|
|
|
|
* yet ended.)
|
|
|
|
*/
|
|
|
|
if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Because the timeline history file lists newer timelines before
|
|
|
|
* older ones, the first timeline we encounter that is new enough to
|
|
|
|
* matter ought to match the ending timeline of the backup.
|
|
|
|
*/
|
|
|
|
if (first_wal_range && endtli != entry->tli)
|
|
|
|
ereport(ERROR,
|
|
|
|
errmsg("expected end timeline %u but found timeline %u",
|
|
|
|
starttli, entry->tli));
|
|
|
|
|
2021-08-23 04:09:54 +02:00
|
|
|
/*
|
|
|
|
* If this timeline entry matches with the timeline on which the
|
|
|
|
* backup started, WAL needs to be checked from the start LSN of the
|
|
|
|
* backup. If this entry refers to a newer timeline, WAL needs to be
|
|
|
|
* checked since the beginning of this timeline, so use the LSN where
|
|
|
|
* the timeline began.
|
|
|
|
*/
|
|
|
|
if (starttli == entry->tli)
|
|
|
|
tl_beginptr = startptr;
|
2020-04-20 20:37:38 +02:00
|
|
|
else
|
|
|
|
{
|
2021-08-23 04:09:54 +02:00
|
|
|
tl_beginptr = entry->begin;
|
2020-04-20 20:37:38 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we reach a TLI that has no valid beginning LSN, there can't
|
|
|
|
* be any more timelines in the history after this point, so we'd
|
|
|
|
* better have arrived at the expected starting TLI. If not,
|
|
|
|
* something's gone horribly wrong.
|
|
|
|
*/
|
2021-08-23 04:09:54 +02:00
|
|
|
if (XLogRecPtrIsInvalid(entry->begin))
|
2020-04-20 20:37:38 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
errmsg("expected start timeline %u but found timeline %u",
|
|
|
|
starttli, entry->tli));
|
|
|
|
}
|
|
|
|
|
|
|
|
AppendToManifest(manifest,
|
|
|
|
"%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
|
|
|
|
first_wal_range ? "" : ",\n",
|
|
|
|
entry->tli,
|
2021-02-23 10:14:38 +01:00
|
|
|
LSN_FORMAT_ARGS(tl_beginptr),
|
|
|
|
LSN_FORMAT_ARGS(endptr));
|
2020-04-20 20:37:38 +02:00
|
|
|
|
|
|
|
if (starttli == entry->tli)
|
|
|
|
{
|
|
|
|
found_start_timeline = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
endptr = entry->begin;
|
|
|
|
first_wal_range = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The last entry in the timeline history for the ending timeline should
|
|
|
|
* be the ending timeline itself. Verify that this is what we observed.
|
|
|
|
*/
|
|
|
|
if (!found_start_timeline)
|
|
|
|
ereport(ERROR,
|
2020-09-09 17:53:39 +02:00
|
|
|
errmsg("start timeline %u not found in history of timeline %u",
|
2020-04-20 20:37:38 +02:00
|
|
|
starttli, endtli));
|
|
|
|
|
|
|
|
/* Terminate the list of WAL ranges. */
|
|
|
|
AppendStringToManifest(manifest, "\n],\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Finalize the backup manifest, and send it to the client.
|
|
|
|
*/
|
|
|
|
void
|
2020-04-23 14:44:06 +02:00
|
|
|
SendBackupManifest(backup_manifest_info *manifest)
|
2020-04-20 20:37:38 +02:00
|
|
|
{
|
|
|
|
StringInfoData protobuf;
|
|
|
|
uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
|
2021-08-19 02:20:19 +02:00
|
|
|
char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
|
2020-04-20 20:37:38 +02:00
|
|
|
size_t manifest_bytes_done = 0;
|
|
|
|
|
|
|
|
if (!IsManifestEnabled(manifest))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Append manifest checksum, so that the problems with the manifest itself
|
|
|
|
* can be detected.
|
|
|
|
*
|
|
|
|
* We always use SHA-256 for this, regardless of what algorithm is chosen
|
|
|
|
* for checksumming the files. If we ever want to make the checksum
|
|
|
|
* algorithm used for the manifest file variable, the client will need a
|
|
|
|
* way to figure out which algorithm to use as close to the beginning of
|
|
|
|
* the manifest file as possible, to avoid having to read the whole thing
|
|
|
|
* twice.
|
|
|
|
*/
|
|
|
|
manifest->still_checksumming = false;
|
Add result size as argument of pg_cryptohash_final() for overflow checks
With its current design, a careless use of pg_cryptohash_final() could
would result in an out-of-bound write in memory as the size of the
destination buffer to store the result digest is not known to the
cryptohash internals, without the caller knowing about that. This
commit adds a new argument to pg_cryptohash_final() to allow such sanity
checks, and implements such defenses.
The internals of SCRAM for HMAC could be tightened a bit more, but as
everything is based on SCRAM_KEY_LEN with uses particular to this code
there is no need to complicate its interface more than necessary, and
this comes back to the refactoring of HMAC in core. Except that, this
minimizes the uses of the existing DIGEST_LENGTH variables, relying
instead on sizeof() for the result sizes. In ossp-uuid, this also makes
the code more defensive, as it already relied on dce_uuid_t being at
least the size of a MD5 digest.
This is in philosophy similar to cfc40d3 for base64.c and aef8948 for
hex.c.
Reported-by: Ranier Vilela
Author: Michael Paquier, Ranier Vilela
Reviewed-by: Kyotaro Horiguchi
Discussion: https://postgr.es/m/CAEudQAoqEGmcff3J4sTSV-R_16Monuz-UpJFbf_dnVH=APr02Q@mail.gmail.com
2021-02-15 02:18:34 +01:00
|
|
|
if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
|
|
|
|
sizeof(checksumbuf)) < 0)
|
2022-01-14 03:25:39 +01:00
|
|
|
elog(ERROR, "failed to finalize checksum of backup manifest");
|
2020-04-20 20:37:38 +02:00
|
|
|
AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
|
2021-08-19 02:20:19 +02:00
|
|
|
|
|
|
|
hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
|
|
|
|
checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
|
|
|
|
|
2020-04-20 20:37:38 +02:00
|
|
|
AppendStringToManifest(manifest, checksumstringbuf);
|
|
|
|
AppendStringToManifest(manifest, "\"}\n");
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We've written all the data to the manifest file. Rewind the file so
|
|
|
|
* that we can read it all back.
|
|
|
|
*/
|
|
|
|
if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
2020-06-16 03:50:56 +02:00
|
|
|
errmsg("could not rewind temporary file")));
|
2020-04-20 20:37:38 +02:00
|
|
|
|
|
|
|
/* Send CopyOutResponse message */
|
|
|
|
pq_beginmessage(&protobuf, 'H');
|
|
|
|
pq_sendbyte(&protobuf, 0); /* overall format */
|
|
|
|
pq_sendint16(&protobuf, 0); /* natts */
|
|
|
|
pq_endmessage(&protobuf);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Send CopyData messages.
|
|
|
|
*
|
|
|
|
* We choose to read back the data from the temporary file in chunks of
|
|
|
|
* size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O
|
|
|
|
* size, so it seems to make sense to match that value here.
|
|
|
|
*/
|
|
|
|
while (manifest_bytes_done < manifest->manifest_size)
|
|
|
|
{
|
|
|
|
char manifestbuf[BLCKSZ];
|
|
|
|
size_t bytes_to_read;
|
|
|
|
size_t rc;
|
|
|
|
|
|
|
|
bytes_to_read = Min(sizeof(manifestbuf),
|
|
|
|
manifest->manifest_size - manifest_bytes_done);
|
|
|
|
rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read);
|
|
|
|
if (rc != bytes_to_read)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
2023-01-16 09:20:44 +01:00
|
|
|
errmsg("could not read from temporary file: read only %zu of %zu bytes",
|
|
|
|
rc, bytes_to_read)));
|
2020-04-20 20:37:38 +02:00
|
|
|
pq_putmessage('d', manifestbuf, bytes_to_read);
|
|
|
|
manifest_bytes_done += bytes_to_read;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* No more data, so send CopyDone message */
|
|
|
|
pq_putemptymessage('c');
|
|
|
|
|
|
|
|
/* Release resources */
|
|
|
|
BufFileClose(manifest->buffile);
|
|
|
|
}
|
2020-04-23 14:44:06 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Append a cstring to the manifest.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
AppendStringToManifest(backup_manifest_info *manifest, char *s)
|
|
|
|
{
|
|
|
|
int len = strlen(s);
|
|
|
|
|
|
|
|
Assert(manifest != NULL);
|
|
|
|
if (manifest->still_checksumming)
|
Move SHA2 routines to a new generic API layer for crypto hashes
Two new routines to allocate a hash context and to free it are created,
as these become necessary for the goal behind this refactoring: switch
the all cryptohash implementations for OpenSSL to use EVP (for FIPS and
also because upstream does not recommend the use of low-level cryptohash
functions for 20 years). Note that OpenSSL hides the internals of
cryptohash contexts since 1.1.0, so it is necessary to leave the
allocation to OpenSSL itself, explaining the need for those two new
routines. This part is going to require more work to properly track
hash contexts with resource owners, but this not introduced here.
Still, this refactoring makes the move possible.
This reduces the number of routines for all SHA2 implementations from
twelve (SHA{224,256,386,512} with init, update and final calls) to five
(create, free, init, update and final calls) by incorporating the hash
type directly into the hash context data.
The new cryptohash routines are moved to a new file, called cryptohash.c
for the fallback implementations, with SHA2 specifics becoming a part
internal to src/common/. OpenSSL specifics are part of
cryptohash_openssl.c. This infrastructure is usable for more hash
types, like MD5 or HMAC.
Any code paths using the internal SHA2 routines are adapted to report
correctly errors, which are most of the changes of this commit. The
zones mostly impacted are checksum manifests, libpq and SCRAM.
Note that e21cbb4 was a first attempt to switch SHA2 to EVP, but it
lacked the refactoring needed for libpq, as done here.
This patch has been tested on Linux and Windows, with and without
OpenSSL, and down to 1.0.1, the oldest version supported on HEAD.
Author: Michael Paquier
Reviewed-by: Daniel Gustafsson
Discussion: https://postgr.es/m/20200924025314.GE7405@paquier.xyz
2020-12-02 02:37:20 +01:00
|
|
|
{
|
|
|
|
if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
|
2022-01-14 03:25:39 +01:00
|
|
|
elog(ERROR, "failed to update checksum of backup manifest");
|
Move SHA2 routines to a new generic API layer for crypto hashes
Two new routines to allocate a hash context and to free it are created,
as these become necessary for the goal behind this refactoring: switch
the all cryptohash implementations for OpenSSL to use EVP (for FIPS and
also because upstream does not recommend the use of low-level cryptohash
functions for 20 years). Note that OpenSSL hides the internals of
cryptohash contexts since 1.1.0, so it is necessary to leave the
allocation to OpenSSL itself, explaining the need for those two new
routines. This part is going to require more work to properly track
hash contexts with resource owners, but this not introduced here.
Still, this refactoring makes the move possible.
This reduces the number of routines for all SHA2 implementations from
twelve (SHA{224,256,386,512} with init, update and final calls) to five
(create, free, init, update and final calls) by incorporating the hash
type directly into the hash context data.
The new cryptohash routines are moved to a new file, called cryptohash.c
for the fallback implementations, with SHA2 specifics becoming a part
internal to src/common/. OpenSSL specifics are part of
cryptohash_openssl.c. This infrastructure is usable for more hash
types, like MD5 or HMAC.
Any code paths using the internal SHA2 routines are adapted to report
correctly errors, which are most of the changes of this commit. The
zones mostly impacted are checksum manifests, libpq and SCRAM.
Note that e21cbb4 was a first attempt to switch SHA2 to EVP, but it
lacked the refactoring needed for libpq, as done here.
This patch has been tested on Linux and Windows, with and without
OpenSSL, and down to 1.0.1, the oldest version supported on HEAD.
Author: Michael Paquier
Reviewed-by: Daniel Gustafsson
Discussion: https://postgr.es/m/20200924025314.GE7405@paquier.xyz
2020-12-02 02:37:20 +01:00
|
|
|
}
|
2020-06-16 03:50:56 +02:00
|
|
|
BufFileWrite(manifest->buffile, s, len);
|
2020-04-23 14:44:06 +02:00
|
|
|
manifest->manifest_size += len;
|
|
|
|
}
|