mirror of
https://git.postgresql.org/git/postgresql.git
synced 2024-10-02 10:51:19 +02:00
Further hacking on performance of COPY OUT. It seems that fwrite()'s
per-call overhead is quite significant, at least on Linux: whatever it's doing is more than just shoving the bytes into a buffer. Buffering the data so we can call fwrite() just once per row seems to be a win.
This commit is contained in:
parent
223ae6957f
commit
4d63e26774
@ -8,7 +8,7 @@
|
|||||||
*
|
*
|
||||||
*
|
*
|
||||||
* IDENTIFICATION
|
* IDENTIFICATION
|
||||||
* $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.265 2006/05/25 18:42:17 tgl Exp $
|
* $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.266 2006/05/26 22:50:02 tgl Exp $
|
||||||
*
|
*
|
||||||
*-------------------------------------------------------------------------
|
*-------------------------------------------------------------------------
|
||||||
*/
|
*/
|
||||||
@ -95,7 +95,8 @@ typedef struct CopyStateData
|
|||||||
/* low-level state data */
|
/* low-level state data */
|
||||||
CopyDest copy_dest; /* type of copy source/destination */
|
CopyDest copy_dest; /* type of copy source/destination */
|
||||||
FILE *copy_file; /* used if copy_dest == COPY_FILE */
|
FILE *copy_file; /* used if copy_dest == COPY_FILE */
|
||||||
StringInfo fe_msgbuf; /* used if copy_dest == COPY_NEW_FE */
|
StringInfo fe_msgbuf; /* used for all dests during COPY TO, only
|
||||||
|
* for dest == COPY_NEW_FE in COPY FROM */
|
||||||
bool fe_copy; /* true for all FE copy dests */
|
bool fe_copy; /* true for all FE copy dests */
|
||||||
bool fe_eof; /* true if detected end of copy data */
|
bool fe_eof; /* true if detected end of copy data */
|
||||||
EolType eol_type; /* EOL type of input */
|
EolType eol_type; /* EOL type of input */
|
||||||
@ -287,7 +288,6 @@ SendCopyBegin(CopyState cstate)
|
|||||||
pq_sendint(&buf, format, 2); /* per-column formats */
|
pq_sendint(&buf, format, 2); /* per-column formats */
|
||||||
pq_endmessage(&buf);
|
pq_endmessage(&buf);
|
||||||
cstate->copy_dest = COPY_NEW_FE;
|
cstate->copy_dest = COPY_NEW_FE;
|
||||||
cstate->fe_msgbuf = makeStringInfo();
|
|
||||||
}
|
}
|
||||||
else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2)
|
else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2)
|
||||||
{
|
{
|
||||||
@ -364,23 +364,16 @@ SendCopyEnd(CopyState cstate)
|
|||||||
{
|
{
|
||||||
if (cstate->copy_dest == COPY_NEW_FE)
|
if (cstate->copy_dest == COPY_NEW_FE)
|
||||||
{
|
{
|
||||||
if (cstate->binary)
|
/* Shouldn't have any unsent data */
|
||||||
{
|
Assert(cstate->fe_msgbuf->len == 0);
|
||||||
/* Need to flush out file trailer word */
|
|
||||||
CopySendEndOfRow(cstate);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
/* Shouldn't have any unsent data */
|
|
||||||
Assert(cstate->fe_msgbuf->len == 0);
|
|
||||||
}
|
|
||||||
/* Send Copy Done message */
|
/* Send Copy Done message */
|
||||||
pq_putemptymessage('c');
|
pq_putemptymessage('c');
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
/* The FE/BE protocol uses \n as newline for all platforms */
|
CopySendData(cstate, "\\.", 2);
|
||||||
CopySendData(cstate, "\\.\n", 3);
|
/* Need to flush out the trailer (this also appends a newline) */
|
||||||
|
CopySendEndOfRow(cstate);
|
||||||
pq_endcopyout(false);
|
pq_endcopyout(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -390,6 +383,7 @@ SendCopyEnd(CopyState cstate)
|
|||||||
* CopySendString does the same for null-terminated strings
|
* CopySendString does the same for null-terminated strings
|
||||||
* CopySendChar does the same for single characters
|
* CopySendChar does the same for single characters
|
||||||
* CopySendEndOfRow does the appropriate thing at end of each data row
|
* CopySendEndOfRow does the appropriate thing at end of each data row
|
||||||
|
* (data is not actually flushed except by CopySendEndOfRow)
|
||||||
*
|
*
|
||||||
* NB: no data conversion is applied by these functions
|
* NB: no data conversion is applied by these functions
|
||||||
*----------
|
*----------
|
||||||
@ -397,46 +391,26 @@ SendCopyEnd(CopyState cstate)
|
|||||||
static void
|
static void
|
||||||
CopySendData(CopyState cstate, void *databuf, int datasize)
|
CopySendData(CopyState cstate, void *databuf, int datasize)
|
||||||
{
|
{
|
||||||
switch (cstate->copy_dest)
|
appendBinaryStringInfo(cstate->fe_msgbuf, (char *) databuf, datasize);
|
||||||
{
|
|
||||||
case COPY_FILE:
|
|
||||||
fwrite(databuf, datasize, 1, cstate->copy_file);
|
|
||||||
if (ferror(cstate->copy_file))
|
|
||||||
ereport(ERROR,
|
|
||||||
(errcode_for_file_access(),
|
|
||||||
errmsg("could not write to COPY file: %m")));
|
|
||||||
break;
|
|
||||||
case COPY_OLD_FE:
|
|
||||||
if (pq_putbytes((char *) databuf, datasize))
|
|
||||||
{
|
|
||||||
/* no hope of recovering connection sync, so FATAL */
|
|
||||||
ereport(FATAL,
|
|
||||||
(errcode(ERRCODE_CONNECTION_FAILURE),
|
|
||||||
errmsg("connection lost during COPY to stdout")));
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case COPY_NEW_FE:
|
|
||||||
appendBinaryStringInfo(cstate->fe_msgbuf,
|
|
||||||
(char *) databuf, datasize);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
CopySendString(CopyState cstate, const char *str)
|
CopySendString(CopyState cstate, const char *str)
|
||||||
{
|
{
|
||||||
CopySendData(cstate, (void *) str, strlen(str));
|
appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
CopySendChar(CopyState cstate, char c)
|
CopySendChar(CopyState cstate, char c)
|
||||||
{
|
{
|
||||||
CopySendData(cstate, &c, 1);
|
appendStringInfoCharMacro(cstate->fe_msgbuf, c);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
CopySendEndOfRow(CopyState cstate)
|
CopySendEndOfRow(CopyState cstate)
|
||||||
{
|
{
|
||||||
|
StringInfo fe_msgbuf = cstate->fe_msgbuf;
|
||||||
|
|
||||||
switch (cstate->copy_dest)
|
switch (cstate->copy_dest)
|
||||||
{
|
{
|
||||||
case COPY_FILE:
|
case COPY_FILE:
|
||||||
@ -449,24 +423,40 @@ CopySendEndOfRow(CopyState cstate)
|
|||||||
CopySendString(cstate, "\r\n");
|
CopySendString(cstate, "\r\n");
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
(void) fwrite(fe_msgbuf->data, fe_msgbuf->len,
|
||||||
|
1, cstate->copy_file);
|
||||||
|
if (ferror(cstate->copy_file))
|
||||||
|
ereport(ERROR,
|
||||||
|
(errcode_for_file_access(),
|
||||||
|
errmsg("could not write to COPY file: %m")));
|
||||||
break;
|
break;
|
||||||
case COPY_OLD_FE:
|
case COPY_OLD_FE:
|
||||||
/* The FE/BE protocol uses \n as newline for all platforms */
|
/* The FE/BE protocol uses \n as newline for all platforms */
|
||||||
if (!cstate->binary)
|
if (!cstate->binary)
|
||||||
CopySendChar(cstate, '\n');
|
CopySendChar(cstate, '\n');
|
||||||
|
|
||||||
|
if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
|
||||||
|
{
|
||||||
|
/* no hope of recovering connection sync, so FATAL */
|
||||||
|
ereport(FATAL,
|
||||||
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
||||||
|
errmsg("connection lost during COPY to stdout")));
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
case COPY_NEW_FE:
|
case COPY_NEW_FE:
|
||||||
/* The FE/BE protocol uses \n as newline for all platforms */
|
/* The FE/BE protocol uses \n as newline for all platforms */
|
||||||
if (!cstate->binary)
|
if (!cstate->binary)
|
||||||
CopySendChar(cstate, '\n');
|
CopySendChar(cstate, '\n');
|
||||||
|
|
||||||
/* Dump the accumulated row as one CopyData message */
|
/* Dump the accumulated row as one CopyData message */
|
||||||
(void) pq_putmessage('d', cstate->fe_msgbuf->data,
|
(void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
|
||||||
cstate->fe_msgbuf->len);
|
|
||||||
/* Reset fe_msgbuf to empty */
|
|
||||||
cstate->fe_msgbuf->len = 0;
|
|
||||||
cstate->fe_msgbuf->data[0] = '\0';
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Reset fe_msgbuf to empty */
|
||||||
|
fe_msgbuf->len = 0;
|
||||||
|
fe_msgbuf->data[0] = '\0';
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -1237,6 +1227,9 @@ CopyTo(CopyState cstate)
|
|||||||
attr_count = list_length(cstate->attnumlist);
|
attr_count = list_length(cstate->attnumlist);
|
||||||
null_print_client = cstate->null_print; /* default */
|
null_print_client = cstate->null_print; /* default */
|
||||||
|
|
||||||
|
/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
|
||||||
|
cstate->fe_msgbuf = makeStringInfo();
|
||||||
|
|
||||||
/* Get info about the columns we need to process. */
|
/* Get info about the columns we need to process. */
|
||||||
out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
|
out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
|
||||||
force_quote = (bool *) palloc(num_phys_attrs * sizeof(bool));
|
force_quote = (bool *) palloc(num_phys_attrs * sizeof(bool));
|
||||||
@ -1423,6 +1416,8 @@ CopyTo(CopyState cstate)
|
|||||||
{
|
{
|
||||||
/* Generate trailer for a binary copy */
|
/* Generate trailer for a binary copy */
|
||||||
CopySendInt16(cstate, -1);
|
CopySendInt16(cstate, -1);
|
||||||
|
/* Need to flush out the trailer */
|
||||||
|
CopySendEndOfRow(cstate);
|
||||||
}
|
}
|
||||||
|
|
||||||
MemoryContextDelete(mycontext);
|
MemoryContextDelete(mycontext);
|
||||||
|
Loading…
Reference in New Issue
Block a user