Revert "Permit dump/reload of not-too-large >1GB tuples"

This reverts commit 646655d264.
Per Tom Lane, changing the definition of StringInfoData amounts to an
ABI break, which is unacceptable in back branches.
This commit is contained in:
Alvaro Herrera 2016-12-06 12:36:44 -03:00
parent 8606271640
commit f858524ee4
4 changed files with 22 additions and 74 deletions

View File

@ -741,9 +741,7 @@ heap_form_tuple(TupleDesc tupleDescriptor,
* Allocate and zero the space needed. Note that the tuple body and * Allocate and zero the space needed. Note that the tuple body and
* HeapTupleData management structure are allocated in one chunk. * HeapTupleData management structure are allocated in one chunk.
*/ */
tuple = MemoryContextAllocExtended(CurrentMemoryContext, tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len);
HEAPTUPLESIZE + len,
MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO);
tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE); tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
/* /*

View File

@ -400,7 +400,7 @@ ReceiveCopyBegin(CopyState cstate)
pq_sendint(&buf, format, 2); /* per-column formats */ pq_sendint(&buf, format, 2); /* per-column formats */
pq_endmessage(&buf); pq_endmessage(&buf);
cstate->copy_dest = COPY_NEW_FE; cstate->copy_dest = COPY_NEW_FE;
cstate->fe_msgbuf = makeLongStringInfo(); cstate->fe_msgbuf = makeStringInfo();
} }
else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2) else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2)
{ {
@ -1865,7 +1865,7 @@ CopyTo(CopyState cstate)
cstate->null_print_client = cstate->null_print; /* default */ cstate->null_print_client = cstate->null_print; /* default */
/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */ /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
cstate->fe_msgbuf = makeLongStringInfo(); cstate->fe_msgbuf = makeStringInfo();
/* Get info about the columns we need to process. */ /* Get info about the columns we need to process. */
cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
@ -2681,8 +2681,8 @@ BeginCopyFrom(Relation rel,
cstate->cur_attval = NULL; cstate->cur_attval = NULL;
/* Set up variables to avoid per-attribute overhead. */ /* Set up variables to avoid per-attribute overhead. */
initLongStringInfo(&cstate->attribute_buf); initStringInfo(&cstate->attribute_buf);
initLongStringInfo(&cstate->line_buf); initStringInfo(&cstate->line_buf);
cstate->line_buf_converted = false; cstate->line_buf_converted = false;
cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1); cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
cstate->raw_buf_index = cstate->raw_buf_len = 0; cstate->raw_buf_index = cstate->raw_buf_len = 0;

View File

@ -4,8 +4,7 @@
* *
* StringInfo provides an indefinitely-extensible string data type. * StringInfo provides an indefinitely-extensible string data type.
* It can be used to buffer either ordinary C strings (null-terminated text) * It can be used to buffer either ordinary C strings (null-terminated text)
* or arbitrary binary data. All storage is allocated with palloc() and * or arbitrary binary data. All storage is allocated with palloc().
* friends.
* *
* Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) 1994, Regents of the University of California
@ -37,29 +36,11 @@ makeStringInfo(void)
return res; return res;
} }
/*
* makeLongStringInfo
*
* Same as makeStringInfo, for larger strings.
*/
StringInfo
makeLongStringInfo(void)
{
StringInfo res;
res = (StringInfo) palloc(sizeof(StringInfoData));
initLongStringInfo(res);
return res;
}
/* /*
* initStringInfo * initStringInfo
* *
* Initialize a StringInfoData struct (with previously undefined contents) * Initialize a StringInfoData struct (with previously undefined contents)
* to describe an empty string; don't enable long strings yet. * to describe an empty string.
*/ */
void void
initStringInfo(StringInfo str) initStringInfo(StringInfo str)
@ -68,22 +49,9 @@ initStringInfo(StringInfo str)
str->data = (char *) palloc(size); str->data = (char *) palloc(size);
str->maxlen = size; str->maxlen = size;
str->long_ok = false;
resetStringInfo(str); resetStringInfo(str);
} }
/*
* initLongStringInfo
*
* Same as initStringInfo, plus enable long strings.
*/
void
initLongStringInfo(StringInfo str)
{
initStringInfo(str);
str->long_ok = true;
}
/* /*
* resetStringInfo * resetStringInfo
* *
@ -174,7 +142,7 @@ appendStringInfoVA(StringInfo str, const char *fmt, va_list args)
/* /*
* Return pvsnprintf's estimate of the space needed. (Although this is * Return pvsnprintf's estimate of the space needed. (Although this is
* given as a size_t, we know it will fit in int because it's not more * given as a size_t, we know it will fit in int because it's not more
* than either MaxAllocSize or half an int's width.) * than MaxAllocSize.)
*/ */
return (int) nprinted; return (int) nprinted;
} }
@ -276,17 +244,7 @@ appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
void void
enlargeStringInfo(StringInfo str, int needed) enlargeStringInfo(StringInfo str, int needed)
{ {
Size newlen; int newlen;
Size limit;
/*
* Determine the upper size limit. Because of overflow concerns outside
* of this module, we limit ourselves to 4-byte signed integer range,
* even for "long_ok" strings.
*/
limit = str->long_ok ?
(((Size) 1) << (sizeof(int32) * 8 - 1)) - 1 :
MaxAllocSize;
/* /*
* Guard against out-of-range "needed" values. Without this, we can get * Guard against out-of-range "needed" values. Without this, we can get
@ -294,7 +252,7 @@ enlargeStringInfo(StringInfo str, int needed)
*/ */
if (needed < 0) /* should not happen */ if (needed < 0) /* should not happen */
elog(ERROR, "invalid string enlargement request size: %d", needed); elog(ERROR, "invalid string enlargement request size: %d", needed);
if (((Size) needed) >= (limit - (Size) str->len)) if (((Size) needed) >= (MaxAllocSize - (Size) str->len))
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("out of memory"), errmsg("out of memory"),
@ -303,7 +261,7 @@ enlargeStringInfo(StringInfo str, int needed)
needed += str->len + 1; /* total space required now */ needed += str->len + 1; /* total space required now */
/* Because of the above test, we now have needed <= limit */ /* Because of the above test, we now have needed <= MaxAllocSize */
if (needed <= str->maxlen) if (needed <= str->maxlen)
return; /* got enough space already */ return; /* got enough space already */
@ -318,14 +276,14 @@ enlargeStringInfo(StringInfo str, int needed)
newlen = 2 * newlen; newlen = 2 * newlen;
/* /*
* Clamp to the limit in case we went past it. Note we are assuming here * Clamp to MaxAllocSize in case we went past it. Note we are assuming
* that limit <= INT_MAX/2, else the above loop could overflow. We will * here that MaxAllocSize <= INT_MAX/2, else the above loop could
* still have newlen >= needed. * overflow. We will still have newlen >= needed.
*/ */
if (newlen > limit) if (newlen > (int) MaxAllocSize)
newlen = limit; newlen = (int) MaxAllocSize;
str->data = (char *) repalloc_huge(str->data, (Size) newlen); str->data = (char *) repalloc(str->data, newlen);
str->maxlen = newlen; str->maxlen = newlen;
} }

View File

@ -30,8 +30,6 @@
* cursor is initialized to zero by makeStringInfo or initStringInfo, * cursor is initialized to zero by makeStringInfo or initStringInfo,
* but is not otherwise touched by the stringinfo.c routines. * but is not otherwise touched by the stringinfo.c routines.
* Some routines use it to scan through a StringInfo. * Some routines use it to scan through a StringInfo.
* long_ok whether this StringInfo can allocate more than MaxAllocSize
* bytes (but still up to 2GB).
*------------------------- *-------------------------
*/ */
typedef struct StringInfoData typedef struct StringInfoData
@ -40,7 +38,6 @@ typedef struct StringInfoData
int len; int len;
int maxlen; int maxlen;
int cursor; int cursor;
bool long_ok;
} StringInfoData; } StringInfoData;
typedef StringInfoData *StringInfo; typedef StringInfoData *StringInfo;
@ -49,11 +46,11 @@ typedef StringInfoData *StringInfo;
/*------------------------ /*------------------------
* There are two ways to create a StringInfo object initially: * There are two ways to create a StringInfo object initially:
* *
* StringInfo stringptr = makeStringInfo(); // or makeLongStringInfo(); * StringInfo stringptr = makeStringInfo();
* Both the StringInfoData and the data buffer are palloc'd. * Both the StringInfoData and the data buffer are palloc'd.
* *
* StringInfoData string; * StringInfoData string;
* initStringInfo(&string); // or initLongStringInfo(); * initStringInfo(&string);
* The data buffer is palloc'd but the StringInfoData is just local. * The data buffer is palloc'd but the StringInfoData is just local.
* This is the easiest approach for a StringInfo object that will * This is the easiest approach for a StringInfo object that will
* only live as long as the current routine. * only live as long as the current routine.
@ -70,26 +67,21 @@ typedef StringInfoData *StringInfo;
/*------------------------ /*------------------------
* makeStringInfo * makeStringInfo
* makeLongStringInfo * Create an empty 'StringInfoData' & return a pointer to it.
* Create an empty 'StringInfoData' & return a pointer to it. The former
* allows up to 1 GB in size, per palloc(); the latter allows up to 2 GB.
*/ */
extern StringInfo makeStringInfo(void); extern StringInfo makeStringInfo(void);
extern StringInfo makeLongStringInfo(void);
/*------------------------ /*------------------------
* initStringInfo * initStringInfo
* initLongStringInfo
* Initialize a StringInfoData struct (with previously undefined contents) * Initialize a StringInfoData struct (with previously undefined contents)
* to describe an empty string. Size limits as above. * to describe an empty string.
*/ */
extern void initStringInfo(StringInfo str); extern void initStringInfo(StringInfo str);
extern void initLongStringInfo(StringInfo str);
/*------------------------ /*------------------------
* resetStringInfo * resetStringInfo
* Clears the current content of the StringInfo, if any. The * Clears the current content of the StringInfo, if any. The
* StringInfo remains valid. The long_ok flag is not reset. * StringInfo remains valid.
*/ */
extern void resetStringInfo(StringInfo str); extern void resetStringInfo(StringInfo str);