/*------------------------------------------------------------------------- * * pg_backup_archiver.h * * Private interface to the pg_dump archiver routines. * It is NOT intended that these routines be called by any * dumper directly. * * See the headers to pg_restore for more details. * * Copyright (c) 2000, Philip Warner * Rights are granted to use this software in any way so long * as this notice is not removed. * * The author is not responsible for loss or damages that may * result from its use. * * * IDENTIFICATION * src/bin/pg_dump/pg_backup_archiver.h * *------------------------------------------------------------------------- */ #ifndef __PG_BACKUP_ARCHIVE__ #define __PG_BACKUP_ARCHIVE__ #include #include "libpq-fe.h" #include "pg_backup.h" #include "pqexpbuffer.h" #define LOBBUFSIZE 16384 /* Data block types */ #define BLK_DATA 1 #define BLK_BLOBS 3 /* Encode version components into a convenient integer */ #define MAKE_ARCHIVE_VERSION(major, minor, rev) (((major) * 256 + (minor)) * 256 + (rev)) #define ARCHIVE_MAJOR(version) (((version) >> 16) & 255) #define ARCHIVE_MINOR(version) (((version) >> 8) & 255) #define ARCHIVE_REV(version) (((version) ) & 255) /* Historical version numbers (checked in code) */ #define K_VERS_1_0 MAKE_ARCHIVE_VERSION(1, 0, 0) #define K_VERS_1_2 MAKE_ARCHIVE_VERSION(1, 2, 0) /* Allow No ZLIB */ #define K_VERS_1_3 MAKE_ARCHIVE_VERSION(1, 3, 0) /* BLOBS */ #define K_VERS_1_4 MAKE_ARCHIVE_VERSION(1, 4, 0) /* Date & name in header */ #define K_VERS_1_5 MAKE_ARCHIVE_VERSION(1, 5, 0) /* Handle dependencies */ #define K_VERS_1_6 MAKE_ARCHIVE_VERSION(1, 6, 0) /* Schema field in TOCs */ #define K_VERS_1_7 MAKE_ARCHIVE_VERSION(1, 7, 0) /* File Offset size in * header */ #define K_VERS_1_8 MAKE_ARCHIVE_VERSION(1, 8, 0) /* change interpretation * of ID numbers and * dependencies */ #define K_VERS_1_9 MAKE_ARCHIVE_VERSION(1, 9, 0) /* add default_with_oids * tracking */ #define K_VERS_1_10 MAKE_ARCHIVE_VERSION(1, 10, 0) /* add tablespace */ #define K_VERS_1_11 MAKE_ARCHIVE_VERSION(1, 11, 0) /* add toc section * indicator */ #define K_VERS_1_12 MAKE_ARCHIVE_VERSION(1, 12, 0) /* add separate BLOB * entries */ #define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path * behavior */ #define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */ #define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add * compression_algorithm * in header */ #define K_VERS_1_16 MAKE_ARCHIVE_VERSION(1, 16, 0) /* BLOB METADATA entries * and multiple BLOBS, * relkind */ /* Current archive version number (the format we can output) */ #define K_VERS_MAJOR 1 #define K_VERS_MINOR 16 #define K_VERS_REV 0 #define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV) /* Newest format we can read */ #define K_VERS_MAX MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, 255) /* Flags to indicate disposition of offsets stored in files */ #define K_OFFSET_POS_NOT_SET 1 #define K_OFFSET_POS_SET 2 #define K_OFFSET_NO_DATA 3 /* * Special exit values from worker children. We reserve 0 for normal * success; 1 and other small values should be interpreted as crashes. */ #define WORKER_OK 0 #define WORKER_CREATE_DONE 10 #define WORKER_INHIBIT_DATA 11 #define WORKER_IGNORED_ERRORS 12 typedef struct _archiveHandle ArchiveHandle; typedef struct _tocEntry TocEntry; struct ParallelState; #define READ_ERROR_EXIT(fd) \ do { \ if (feof(fd)) \ pg_fatal("could not read from input file: end of file"); \ else \ pg_fatal("could not read from input file: %m"); \ } while (0) #define WRITE_ERROR_EXIT \ do { \ pg_fatal("could not write to output file: %m"); \ } while (0) typedef enum T_Action { ACT_DUMP, ACT_RESTORE, } T_Action; typedef void (*ClosePtrType) (ArchiveHandle *AH); typedef void (*ReopenPtrType) (ArchiveHandle *AH); typedef void (*ArchiveEntryPtrType) (ArchiveHandle *AH, TocEntry *te); typedef void (*StartDataPtrType) (ArchiveHandle *AH, TocEntry *te); typedef void (*WriteDataPtrType) (ArchiveHandle *AH, const void *data, size_t dLen); typedef void (*EndDataPtrType) (ArchiveHandle *AH, TocEntry *te); typedef void (*StartLOsPtrType) (ArchiveHandle *AH, TocEntry *te); typedef void (*StartLOPtrType) (ArchiveHandle *AH, TocEntry *te, Oid oid); typedef void (*EndLOPtrType) (ArchiveHandle *AH, TocEntry *te, Oid oid); typedef void (*EndLOsPtrType) (ArchiveHandle *AH, TocEntry *te); typedef int (*WriteBytePtrType) (ArchiveHandle *AH, const int i); typedef int (*ReadBytePtrType) (ArchiveHandle *AH); typedef void (*WriteBufPtrType) (ArchiveHandle *AH, const void *c, size_t len); typedef void (*ReadBufPtrType) (ArchiveHandle *AH, void *buf, size_t len); typedef void (*WriteExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); typedef void (*ReadExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); typedef void (*PrintExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te); typedef void (*PrintTocDataPtrType) (ArchiveHandle *AH, TocEntry *te); typedef void (*PrepParallelRestorePtrType) (ArchiveHandle *AH); typedef void (*ClonePtrType) (ArchiveHandle *AH); typedef void (*DeClonePtrType) (ArchiveHandle *AH); typedef int (*WorkerJobDumpPtrType) (ArchiveHandle *AH, TocEntry *te); typedef int (*WorkerJobRestorePtrType) (ArchiveHandle *AH, TocEntry *te); typedef size_t (*CustomOutPtrType) (ArchiveHandle *AH, const void *buf, size_t len); typedef enum { SQL_SCAN = 0, /* normal */ SQL_IN_SINGLE_QUOTE, /* '...' literal */ SQL_IN_DOUBLE_QUOTE, /* "..." identifier */ } sqlparseState; typedef struct { sqlparseState state; /* see above */ bool backSlash; /* next char is backslash quoted? */ PQExpBuffer curCmd; /* incomplete line (NULL if not created) */ } sqlparseInfo; typedef enum { STAGE_NONE = 0, STAGE_INITIALIZING, STAGE_PROCESSING, STAGE_FINALIZING, } ArchiverStage; typedef enum { OUTPUT_SQLCMDS = 0, /* emitting general SQL commands */ OUTPUT_COPYDATA, /* writing COPY data */ OUTPUT_OTHERDATA, /* writing data as INSERT commands */ } ArchiverOutput; /* * For historical reasons, ACL items are interspersed with everything else in * a dump file's TOC; typically they're right after the object they're for. * However, we need to restore data before ACLs, as otherwise a read-only * table (ie one where the owner has revoked her own INSERT privilege) causes * data restore failures. On the other hand, matview REFRESH commands should * come out after ACLs, as otherwise non-superuser-owned matviews might not * be able to execute. (If the permissions at the time of dumping would not * allow a REFRESH, too bad; we won't fix that for you.) We also want event * triggers to be restored after ACLs, so that they can't mess those up. * * These considerations force us to make three passes over the TOC, * restoring the appropriate subset of items in each pass. We assume that * the dependency sort resulted in an appropriate ordering of items within * each subset. * * XXX This mechanism should be superseded by tracking dependencies on ACLs * properly; but we'll still need it for old dump files even after that. */ typedef enum { RESTORE_PASS_MAIN = 0, /* Main pass (most TOC item types) */ RESTORE_PASS_ACL, /* ACL item types */ RESTORE_PASS_POST_ACL, /* Event trigger and matview refresh items */ #define RESTORE_PASS_LAST RESTORE_PASS_POST_ACL } RestorePass; #define REQ_SCHEMA 0x01 /* want schema */ #define REQ_DATA 0x02 /* want data */ #define REQ_SPECIAL 0x04 /* for special TOC entries */ struct _archiveHandle { Archive public; /* Public part of archive */ int version; /* Version of file */ char *archiveRemoteVersion; /* When reading an archive, the * version of the dumped DB */ char *archiveDumpVersion; /* When reading an archive, the version of * the dumper */ size_t intSize; /* Size of an integer in the archive */ size_t offSize; /* Size of a file offset in the archive - * Added V1.7 */ ArchiveFormat format; /* Archive format */ sqlparseInfo sqlparse; /* state for parsing INSERT data */ time_t createDate; /* Date archive created */ /* * Fields used when discovering archive format. For tar format, we load * the first block into the lookahead buffer, and verify that it looks * like a tar header. The tar module must then consume bytes from the * lookahead buffer before reading any more from the file. For custom * format, we load only the "PGDMP" marker into the buffer, and then set * readHeader after confirming it matches. The buffer is vestigial in * this case, as the subsequent code just checks readHeader and doesn't * examine the buffer. */ int readHeader; /* Set if we already read "PGDMP" marker */ char *lookahead; /* Buffer used when reading header to discover * format */ size_t lookaheadSize; /* Allocated size of buffer */ size_t lookaheadLen; /* Length of valid data in lookahead */ size_t lookaheadPos; /* Current read position in lookahead buffer */ ArchiveEntryPtrType ArchiveEntryPtr; /* Called for each metadata object */ StartDataPtrType StartDataPtr; /* Called when table data is about to be * dumped */ WriteDataPtrType WriteDataPtr; /* Called to send some table data to the * archive */ EndDataPtrType EndDataPtr; /* Called when table data dump is finished */ WriteBytePtrType WriteBytePtr; /* Write a byte to output */ ReadBytePtrType ReadBytePtr; /* Read a byte from an archive */ WriteBufPtrType WriteBufPtr; /* Write a buffer of output to the archive */ ReadBufPtrType ReadBufPtr; /* Read a buffer of input from the archive */ ClosePtrType ClosePtr; /* Close the archive */ ReopenPtrType ReopenPtr; /* Reopen the archive */ WriteExtraTocPtrType WriteExtraTocPtr; /* Write extra TOC entry data * associated with the current * archive format */ ReadExtraTocPtrType ReadExtraTocPtr; /* Read extra info associated with * archive format */ PrintExtraTocPtrType PrintExtraTocPtr; /* Extra TOC info for format */ PrintTocDataPtrType PrintTocDataPtr; StartLOsPtrType StartLOsPtr; EndLOsPtrType EndLOsPtr; StartLOPtrType StartLOPtr; EndLOPtrType EndLOPtr; SetupWorkerPtrType SetupWorkerPtr; WorkerJobDumpPtrType WorkerJobDumpPtr; WorkerJobRestorePtrType WorkerJobRestorePtr; PrepParallelRestorePtrType PrepParallelRestorePtr; ClonePtrType ClonePtr; /* Clone format-specific fields */ DeClonePtrType DeClonePtr; /* Clean up cloned fields */ CustomOutPtrType CustomOutPtr; /* Alternative script output routine */ /* Stuff for direct DB connection */ char *archdbname; /* DB name *read* from archive */ char *savedPassword; /* password for ropt->username, if known */ char *use_role; PGconn *connection; /* If connCancel isn't NULL, SIGINT handler will send a cancel */ PGcancel *volatile connCancel; int connectToDB; /* Flag to indicate if direct DB connection is * required */ ArchiverOutput outputKind; /* Flag for what we're currently writing */ bool pgCopyIn; /* Currently in libpq 'COPY IN' mode. */ int loFd; bool writingLO; int loCount; /* # of LOs restored */ char *fSpec; /* Archive File Spec */ FILE *FH; /* General purpose file handle */ void *OF; /* Output file */ struct _tocEntry *toc; /* Header of circular list of TOC entries */ int tocCount; /* Number of TOC entries */ DumpId maxDumpId; /* largest DumpId among all TOC entries */ /* arrays created after the TOC list is complete: */ struct _tocEntry **tocsByDumpId; /* TOCs indexed by dumpId */ DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ pg_compress_specification compression_spec; /* Requested specification for * compression */ bool dosync; /* data requested to be synced on sight */ DataDirSyncMethod sync_method; ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ /* these vars track state to avoid sending redundant SET commands */ char *currUser; /* current username, or NULL if unknown */ char *currSchema; /* current schema, or NULL */ char *currTablespace; /* current tablespace, or NULL */ char *currTableAm; /* current table access method, or NULL */ /* in --transaction-size mode, this counts objects emitted in cur xact */ int txnCount; void *lo_buf; size_t lo_buf_used; size_t lo_buf_size; int noTocComments; ArchiverStage stage; ArchiverStage lastErrorStage; RestorePass restorePass; /* used only during parallel restore */ struct _tocEntry *currentTE; struct _tocEntry *lastErrorTE; }; struct _tocEntry { struct _tocEntry *prev; struct _tocEntry *next; CatalogId catalogId; DumpId dumpId; teSection section; bool hadDumper; /* Archiver was passed a dumper routine (used * in restore) */ char *tag; /* index tag */ char *namespace; /* null or empty string if not in a schema */ char *tablespace; /* null if not in a tablespace; empty string * means use database default */ char *tableam; /* table access method, only for TABLE tags */ char relkind; /* relation kind, only for TABLE tags */ char *owner; char *desc; char *defn; char *dropStmt; char *copyStmt; DumpId *dependencies; /* dumpIds of objects this one depends on */ int nDeps; /* number of dependencies */ DataDumperPtr dataDumper; /* Routine to dump data for object */ const void *dataDumperArg; /* Arg for above routine */ void *formatData; /* TOC Entry data specific to file format */ /* working state while dumping/restoring */ pgoff_t dataLength; /* item's data size; 0 if none or unknown */ int reqs; /* do we need schema and/or data of object * (REQ_* bit mask) */ bool created; /* set for DATA member if TABLE was created */ /* working state (needed only for parallel restore) */ struct _tocEntry *pending_prev; /* list links for pending-items list; */ struct _tocEntry *pending_next; /* NULL if not in that list */ int depCount; /* number of dependencies not yet restored */ DumpId *revDeps; /* dumpIds of objects depending on this one */ int nRevDeps; /* number of such dependencies */ DumpId *lockDeps; /* dumpIds of objects this one needs lock on */ int nLockDeps; /* number of such dependencies */ }; extern int parallel_restore(ArchiveHandle *AH, TocEntry *te); extern void on_exit_close_archive(Archive *AHX); extern void warn_or_exit_horribly(ArchiveHandle *AH, const char *fmt,...) pg_attribute_printf(2, 3); /* Options for ArchiveEntry */ typedef struct _archiveOpts { const char *tag; const char *namespace; const char *tablespace; const char *tableam; char relkind; const char *owner; const char *description; teSection section; const char *createStmt; const char *dropStmt; const char *copyStmt; const DumpId *deps; int nDeps; DataDumperPtr dumpFn; const void *dumpArg; } ArchiveOpts; #define ARCHIVE_OPTS(...) &(ArchiveOpts){__VA_ARGS__} /* Called to add a TOC entry */ extern TocEntry *ArchiveEntry(Archive *AHX, CatalogId catalogId, DumpId dumpId, ArchiveOpts *opts); extern void WriteHead(ArchiveHandle *AH); extern void ReadHead(ArchiveHandle *AH); extern void WriteToc(ArchiveHandle *AH); extern void ReadToc(ArchiveHandle *AH); extern void WriteDataChunks(ArchiveHandle *AH, struct ParallelState *pstate); extern void WriteDataChunksForTocEntry(ArchiveHandle *AH, TocEntry *te); extern ArchiveHandle *CloneArchive(ArchiveHandle *AH); extern void DeCloneArchive(ArchiveHandle *AH); extern int TocIDRequired(ArchiveHandle *AH, DumpId id); TocEntry *getTocEntryByDumpId(ArchiveHandle *AH, DumpId id); extern bool checkSeek(FILE *fp); #define appendStringLiteralAHX(buf,str,AH) \ appendStringLiteral(buf, str, (AH)->public.encoding, (AH)->public.std_strings) #define appendByteaLiteralAHX(buf,str,len,AH) \ appendByteaLiteral(buf, str, len, (AH)->public.std_strings) /* * Mandatory routines for each supported format */ extern size_t WriteInt(ArchiveHandle *AH, int i); extern int ReadInt(ArchiveHandle *AH); extern char *ReadStr(ArchiveHandle *AH); extern size_t WriteStr(ArchiveHandle *AH, const char *c); int ReadOffset(ArchiveHandle *, pgoff_t *); size_t WriteOffset(ArchiveHandle *, pgoff_t, int); extern void StartRestoreLOs(ArchiveHandle *AH); extern void StartRestoreLO(ArchiveHandle *AH, Oid oid, bool drop); extern void EndRestoreLO(ArchiveHandle *AH, Oid oid); extern void EndRestoreLOs(ArchiveHandle *AH); extern void InitArchiveFmt_Custom(ArchiveHandle *AH); extern void InitArchiveFmt_Null(ArchiveHandle *AH); extern void InitArchiveFmt_Directory(ArchiveHandle *AH); extern void InitArchiveFmt_Tar(ArchiveHandle *AH); extern bool isValidTarHeader(char *header); extern void ReconnectToServer(ArchiveHandle *AH, const char *dbname); extern void IssueCommandPerBlob(ArchiveHandle *AH, TocEntry *te, const char *cmdBegin, const char *cmdEnd); extern void IssueACLPerBlob(ArchiveHandle *AH, TocEntry *te); extern void DropLOIfExists(ArchiveHandle *AH, Oid oid); void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH); int ahprintf(ArchiveHandle *AH, const char *fmt,...) pg_attribute_printf(2, 3); #endif