From e97512345419c35a33c07bde76ab3394652857f2 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Wed, 24 Apr 2002 02:21:04 +0000 Subject: [PATCH] Speed improvement for large object restore. Mario Weilguni --- src/bin/pg_dump/pg_backup_archiver.c | 45 +++++++++++++++++++++++----- src/bin/pg_dump/pg_backup_archiver.h | 6 +++- src/bin/pg_dump/pg_backup_custom.c | 8 ++++- src/bin/pg_dump/pg_backup_files.c | 8 ++++- src/bin/pg_dump/pg_backup_null.c | 3 +- src/bin/pg_dump/pg_backup_tar.c | 8 ++++- 6 files changed, 64 insertions(+), 14 deletions(-) diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 2c8ea7d828..8d2c2792bf 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -15,7 +15,7 @@ * * * IDENTIFICATION - * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.42 2002/02/11 00:18:20 tgl Exp $ + * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.43 2002/04/24 02:21:04 momjian Exp $ * * Modifications - 28-Jun-2000 - pjw@rhyme.com.au * @@ -819,6 +819,9 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid) AH->createdBlobXref = 1; } + /* Initialize the LO Buffer */ + AH->lo_buf_used = 0; + /* * Start long-running TXs if necessary */ @@ -848,6 +851,19 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid) void EndRestoreBlob(ArchiveHandle *AH, Oid oid) { + if(AH->lo_buf_used > 0) { + /* Write remaining bytes from the LO buffer */ + int res; + res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf, AH->lo_buf_used); + + ahlog(AH, 5, "wrote remaining %d bytes of large object data (result = %d)\n", + (int)AH->lo_buf_used, res); + if (res != AH->lo_buf_used) + die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n", + res, AH->lo_buf_used); + AH->lo_buf_used = 0; + } + lo_close(AH->connection, AH->loFd); AH->writingBlob = 0; @@ -1228,14 +1244,27 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH) if (AH->writingBlob) { - res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb); - ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n", - (int) (size * nmemb), res); - if (res != size * nmemb) - die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n", - res, (int) (size * nmemb)); + if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) { + /* Split LO buffer */ + int remaining = AH->lo_buf_size - AH->lo_buf_used; + int slack = nmemb * size - remaining; - return res; + memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining); + res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size); + ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n", + AH->lo_buf_size, res); + if (res != AH->lo_buf_size) + die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n", + res, AH->lo_buf_size); + memcpy(AH->lo_buf, ptr + remaining, slack); + AH->lo_buf_used = slack; + } else { + /* LO Buffer is still large enough, buffer it */ + memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb); + AH->lo_buf_used += size * nmemb; + } + + return size * nmemb; } else if (AH->gzOut) { diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index cb7671ca69..0fbce03f52 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -17,7 +17,7 @@ * * * IDENTIFICATION - * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.41 2001/11/05 17:46:30 momjian Exp $ + * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.42 2002/04/24 02:21:04 momjian Exp $ * * Modifications - 28-Jun-2000 - pjw@rhyme.com.au * - Initial version. @@ -41,6 +41,7 @@ #include #include "pqexpbuffer.h" +#define LOBBUFSIZE 32768 #ifdef HAVE_LIBZ #include @@ -240,6 +241,9 @@ typedef struct _archiveHandle RestoreOptions *ropt; /* Used to check restore options in * ahwrite etc */ + void *lo_buf; + int lo_buf_used; + int lo_buf_size; } ArchiveHandle; typedef struct _tocEntry diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 314e90d4df..93e58378f4 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -19,7 +19,7 @@ * * * IDENTIFICATION - * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.17 2001/11/27 23:48:12 tgl Exp $ + * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.18 2002/04/24 02:21:04 momjian Exp $ * * Modifications - 28-Jun-2000 - pjw@rhyme.com.au * @@ -153,6 +153,12 @@ InitArchiveFmt_Custom(ArchiveHandle *AH) if (ctx->zp == NULL) die_horribly(AH, modulename, "out of memory\n"); + /* Initialize LO buffering */ + AH->lo_buf_size = LOBBUFSIZE; + AH->lo_buf = (void *)malloc(LOBBUFSIZE); + if(AH->lo_buf == NULL) + die_horribly(AH, modulename, "out of memory\n"); + /* * zlibOutSize is the buffer size we tell zlib it can output to. We * actually allocate one extra byte because some routines want to diff --git a/src/bin/pg_dump/pg_backup_files.c b/src/bin/pg_dump/pg_backup_files.c index dfc1fdd2a1..33e49b2d38 100644 --- a/src/bin/pg_dump/pg_backup_files.c +++ b/src/bin/pg_dump/pg_backup_files.c @@ -20,7 +20,7 @@ * * * IDENTIFICATION - * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.14 2001/10/25 05:49:52 momjian Exp $ + * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.15 2002/04/24 02:21:04 momjian Exp $ * * Modifications - 28-Jun-2000 - pjw@rhyme.com.au * @@ -113,6 +113,12 @@ InitArchiveFmt_Files(ArchiveHandle *AH) AH->formatData = (void *) ctx; ctx->filePos = 0; + /* Initialize LO buffering */ + AH->lo_buf_size = LOBBUFSIZE; + AH->lo_buf = (void *)malloc(LOBBUFSIZE); + if(AH->lo_buf == NULL) + die_horribly(AH, modulename, "out of memory\n"); + /* * Now open the TOC file */ diff --git a/src/bin/pg_dump/pg_backup_null.c b/src/bin/pg_dump/pg_backup_null.c index ee3b85eb4c..af8f52a7c2 100644 --- a/src/bin/pg_dump/pg_backup_null.c +++ b/src/bin/pg_dump/pg_backup_null.c @@ -17,7 +17,7 @@ * * * IDENTIFICATION - * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.7 2001/06/27 21:21:37 petere Exp $ + * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.8 2002/04/24 02:21:04 momjian Exp $ * * Modifications - 09-Jul-2000 - pjw@rhyme.com.au * @@ -64,7 +64,6 @@ InitArchiveFmt_Null(ArchiveHandle *AH) */ if (AH->mode == archModeRead) die_horribly(AH, NULL, "this format cannot be read\n"); - } /* diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 557ccbecbd..efd8a03013 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -16,7 +16,7 @@ * * * IDENTIFICATION - * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.20 2001/10/28 06:25:58 momjian Exp $ + * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.21 2002/04/24 02:21:04 momjian Exp $ * * Modifications - 28-Jun-2000 - pjw@rhyme.com.au * @@ -157,6 +157,12 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) ctx = (lclContext *) malloc(sizeof(lclContext)); AH->formatData = (void *) ctx; ctx->filePos = 0; + + /* Initialize LO buffering */ + AH->lo_buf_size = LOBBUFSIZE; + AH->lo_buf = (void *)malloc(LOBBUFSIZE); + if(AH->lo_buf == NULL) + die_horribly(AH, modulename, "out of memory\n"); /* * Now open the TOC file