diff --git a/src/common/parse_manifest.c b/src/common/parse_manifest.c index 40ec3b4f58..040c5597df 100644 --- a/src/common/parse_manifest.c +++ b/src/common/parse_manifest.c @@ -91,6 +91,13 @@ typedef struct char *manifest_checksum; } JsonManifestParseState; +typedef struct JsonManifestParseIncrementalState +{ + JsonLexContext lex; + JsonSemAction sem; + pg_cryptohash_ctx *manifest_ctx; +} JsonManifestParseIncrementalState; + static JsonParseErrorType json_manifest_object_start(void *state); static JsonParseErrorType json_manifest_object_end(void *state); static JsonParseErrorType json_manifest_array_start(void *state); @@ -104,7 +111,8 @@ static void json_manifest_finalize_system_identifier(JsonManifestParseState *par static void json_manifest_finalize_file(JsonManifestParseState *parse); static void json_manifest_finalize_wal_range(JsonManifestParseState *parse); static void verify_manifest_checksum(JsonManifestParseState *parse, - char *buffer, size_t size); + char *buffer, size_t size, + pg_cryptohash_ctx *incr_ctx); static void json_manifest_parse_failure(JsonManifestParseContext *context, char *msg); @@ -112,6 +120,90 @@ static int hexdecode_char(char c); static bool hexdecode_string(uint8 *result, char *input, int nbytes); static bool parse_xlogrecptr(XLogRecPtr *result, char *input); +/* + * Set up for incremental parsing of the manifest. + * + */ + +JsonManifestParseIncrementalState * +json_parse_manifest_incremental_init(JsonManifestParseContext *context) +{ + JsonManifestParseIncrementalState *incstate; + JsonManifestParseState *parse; + pg_cryptohash_ctx *manifest_ctx; + + incstate = palloc(sizeof(JsonManifestParseIncrementalState)); + parse = palloc(sizeof(JsonManifestParseState)); + + parse->context = context; + parse->state = JM_EXPECT_TOPLEVEL_START; + parse->saw_version_field = false; + + makeJsonLexContextIncremental(&(incstate->lex), PG_UTF8, true); + + incstate->sem.semstate = parse; + incstate->sem.object_start = json_manifest_object_start; + incstate->sem.object_end = json_manifest_object_end; + incstate->sem.array_start = json_manifest_array_start; + incstate->sem.array_end = json_manifest_array_end; + incstate->sem.object_field_start = json_manifest_object_field_start; + incstate->sem.object_field_end = NULL; + incstate->sem.array_element_start = NULL; + incstate->sem.array_element_end = NULL; + incstate->sem.scalar = json_manifest_scalar; + + manifest_ctx = pg_cryptohash_create(PG_SHA256); + if (manifest_ctx == NULL) + context->error_cb(context, "out of memory"); + if (pg_cryptohash_init(manifest_ctx) < 0) + context->error_cb(context, "could not initialize checksum of manifest"); + incstate->manifest_ctx = manifest_ctx; + + return incstate; +} + +/* + * parse the manifest in pieces. + * + * The caller must ensure that the final piece contains the final lines + * with the complete checksum. + */ + +void +json_parse_manifest_incremental_chunk( + JsonManifestParseIncrementalState *incstate, char *chunk, int size, + bool is_last) +{ + JsonParseErrorType res, + expected; + JsonManifestParseState *parse = incstate->sem.semstate; + JsonManifestParseContext *context = parse->context; + + res = pg_parse_json_incremental(&(incstate->lex), &(incstate->sem), + chunk, size, is_last); + + expected = is_last ? JSON_SUCCESS : JSON_INCOMPLETE; + + if (res != expected) + json_manifest_parse_failure(context, + json_errdetail(res, &(incstate->lex))); + + if (is_last && parse->state != JM_EXPECT_EOF) + json_manifest_parse_failure(context, "manifest ended unexpectedly"); + + if (!is_last) + { + if (pg_cryptohash_update(incstate->manifest_ctx, + (uint8 *) chunk, size) < 0) + context->error_cb(context, "could not update checksum of manifest"); + } + else + { + verify_manifest_checksum(parse, chunk, size, incstate->manifest_ctx); + } +} + + /* * Main entrypoint to parse a JSON-format backup manifest. * @@ -157,7 +249,7 @@ json_parse_manifest(JsonManifestParseContext *context, char *buffer, json_manifest_parse_failure(context, "manifest ended unexpectedly"); /* Verify the manifest checksum. */ - verify_manifest_checksum(&parse, buffer, size); + verify_manifest_checksum(&parse, buffer, size, NULL); freeJsonLexContext(lex); } @@ -390,6 +482,8 @@ json_manifest_object_field_start(void *state, char *fname, bool isnull) break; } + pfree(fname); + return JSON_SUCCESS; } @@ -698,10 +792,14 @@ json_manifest_finalize_wal_range(JsonManifestParseState *parse) * The last line of the manifest file is excluded from the manifest checksum, * because the last line is expected to contain the checksum that covers * the rest of the file. + * + * For an incremental parse, this will just be called on the last chunk of the + * manifest, and the cryptohash context paswed in. For a non-incremental + * parse incr_ctx will be NULL. */ static void verify_manifest_checksum(JsonManifestParseState *parse, char *buffer, - size_t size) + size_t size, pg_cryptohash_ctx *incr_ctx) { JsonManifestParseContext *context = parse->context; size_t i; @@ -736,11 +834,18 @@ verify_manifest_checksum(JsonManifestParseState *parse, char *buffer, "last line not newline-terminated"); /* Checksum the rest. */ - manifest_ctx = pg_cryptohash_create(PG_SHA256); - if (manifest_ctx == NULL) - context->error_cb(context, "out of memory"); - if (pg_cryptohash_init(manifest_ctx) < 0) - context->error_cb(context, "could not initialize checksum of manifest"); + if (incr_ctx == NULL) + { + manifest_ctx = pg_cryptohash_create(PG_SHA256); + if (manifest_ctx == NULL) + context->error_cb(context, "out of memory"); + if (pg_cryptohash_init(manifest_ctx) < 0) + context->error_cb(context, "could not initialize checksum of manifest"); + } + else + { + manifest_ctx = incr_ctx; + } if (pg_cryptohash_update(manifest_ctx, (uint8 *) buffer, penultimate_newline + 1) < 0) context->error_cb(context, "could not update checksum of manifest"); if (pg_cryptohash_final(manifest_ctx, manifest_checksum_actual, diff --git a/src/include/common/parse_manifest.h b/src/include/common/parse_manifest.h index 78b052c045..3aa594fcac 100644 --- a/src/include/common/parse_manifest.h +++ b/src/include/common/parse_manifest.h @@ -20,6 +20,7 @@ struct JsonManifestParseContext; typedef struct JsonManifestParseContext JsonManifestParseContext; +typedef struct JsonManifestParseIncrementalState JsonManifestParseIncrementalState; typedef void (*json_manifest_version_callback) (JsonManifestParseContext *, int manifest_version); @@ -48,5 +49,9 @@ struct JsonManifestParseContext extern void json_parse_manifest(JsonManifestParseContext *context, char *buffer, size_t size); +extern JsonManifestParseIncrementalState *json_parse_manifest_incremental_init(JsonManifestParseContext *context); +extern void json_parse_manifest_incremental_chunk( + JsonManifestParseIncrementalState *incstate, char *chunk, int size, + bool is_last); #endif