Compare commits

...

4 Commits

Author SHA1 Message Date
Tom Lane 4fe2aa7656 Reduce memory leakage in initdb.
While testing commit 3e51b278d, I noted that initdb leaks about a
megabyte worth of data due to the sloppy bookkeeping in its
string-manipulating code.  That's not a huge amount on modern machines,
but it's still kind of annoying, and it's easy to fix by recognizing
that we might as well treat these arrays of strings as
modifiable-in-place.  There's no caller that cares about preserving
the old state of the array after replace_token or replace_guc_value.

With this fix, valgrind sees only a few hundred bytes leaked during
an initdb run.

Discussion: https://postgr.es/m/2844176.1674681919@sss.pgh.pa.us
2023-03-22 14:28:45 -04:00
Tom Lane 3e51b278db Add "-c name=value" switch to initdb.
This option, or its long form --set, sets the GUC "name" to "value".
The setting applies in the bootstrap and standalone servers run by
initdb, and is also written into the generated postgresql.conf.

This can save an extra editing step when creating a new cluster,
but the real use-case is for coping with situations where the
bootstrap server fails to start due to environmental issues;
for example, if it's necessary to force huge_pages to off.

Discussion: https://postgr.es/m/2844176.1674681919@sss.pgh.pa.us
2023-03-22 13:49:05 -04:00
Andres Freund 5df319f3d5 Fix memory leak and inefficiency in CREATE DATABASE ... STRATEGY WAL_LOG
RelationCopyStorageUsingBuffer() did not free the strategies used to access
the source / target relation. They memory was released at the end of the
transaction, but when using a template database with a lot of relations, the
temporary leak can become big prohibitively big.

RelationCopyStorageUsingBuffer() acquired the buffer for the target relation
with RBM_NORMAL, therefore requiring a read of a block guaranteed to be
zero. Use RBM_ZERO_AND_LOCK instead.

Reviewed-by: Robert Haas <robertmhaas@gmail.com>
Discussion: https://postgr.es/m/20230321070113.o2vqqxogjykwgfrr@awork3.anarazel.de
Backpatch: 15-, where STRATEGY WAL_LOG was introduced
2023-03-22 09:20:34 -07:00
Robert Haas bbc1376b39 Teach verify_heapam() to validate update chains within a page.
Prior to this commit, we only consider each tuple or line pointer
on the page in isolation, but now we can do some validation of a line
pointer against its successor. For example, a redirect line pointer
shouldn't point to another redirect line pointer, and if a tuple
is HOT-updated, the result should be a heap-only tuple.

Himanshu Upadhyaya and Robert Haas, reviewed by Aleksander Alekseev,
Andres Freund, and Peter Geoghegan.
2023-03-22 08:48:54 -04:00
6 changed files with 918 additions and 161 deletions

View File

@ -150,7 +150,9 @@ typedef struct HeapCheckContext
} HeapCheckContext;
/* Internal implementation */
static void check_tuple(HeapCheckContext *ctx);
static void check_tuple(HeapCheckContext *ctx,
bool *xmin_commit_status_ok,
XidCommitStatus *xmin_commit_status);
static void check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
ToastedAttribute *ta, int32 *expected_chunk_seq,
uint32 extsize);
@ -160,7 +162,9 @@ static void check_toasted_attribute(HeapCheckContext *ctx,
ToastedAttribute *ta);
static bool check_tuple_header(HeapCheckContext *ctx);
static bool check_tuple_visibility(HeapCheckContext *ctx);
static bool check_tuple_visibility(HeapCheckContext *ctx,
bool *xmin_commit_status_ok,
XidCommitStatus *xmin_commit_status);
static void report_corruption(HeapCheckContext *ctx, char *msg);
static void report_toast_corruption(HeapCheckContext *ctx,
@ -399,9 +403,16 @@ verify_heapam(PG_FUNCTION_ARGS)
for (ctx.blkno = first_block; ctx.blkno <= last_block; ctx.blkno++)
{
OffsetNumber maxoff;
OffsetNumber predecessor[MaxOffsetNumber];
OffsetNumber successor[MaxOffsetNumber];
bool lp_valid[MaxOffsetNumber];
bool xmin_commit_status_ok[MaxOffsetNumber];
XidCommitStatus xmin_commit_status[MaxOffsetNumber];
CHECK_FOR_INTERRUPTS();
memset(predecessor, 0, sizeof(OffsetNumber) * MaxOffsetNumber);
/* Optionally skip over all-frozen or all-visible blocks */
if (skip_option != SKIP_PAGES_NONE)
{
@ -433,6 +444,12 @@ verify_heapam(PG_FUNCTION_ARGS)
for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
ctx.offnum = OffsetNumberNext(ctx.offnum))
{
BlockNumber nextblkno;
OffsetNumber nextoffnum;
successor[ctx.offnum] = InvalidOffsetNumber;
lp_valid[ctx.offnum] = false;
xmin_commit_status_ok[ctx.offnum] = false;
ctx.itemid = PageGetItemId(ctx.page, ctx.offnum);
/* Skip over unused/dead line pointers */
@ -469,6 +486,14 @@ verify_heapam(PG_FUNCTION_ARGS)
report_corruption(&ctx,
psprintf("line pointer redirection to unused item at offset %u",
(unsigned) rdoffnum));
/*
* Record the fact that this line pointer has passed basic
* sanity checking, and also the offset number to which it
* points.
*/
lp_valid[ctx.offnum] = true;
successor[ctx.offnum] = rdoffnum;
continue;
}
@ -502,11 +527,237 @@ verify_heapam(PG_FUNCTION_ARGS)
}
/* It should be safe to examine the tuple's header, at least */
lp_valid[ctx.offnum] = true;
ctx.tuphdr = (HeapTupleHeader) PageGetItem(ctx.page, ctx.itemid);
ctx.natts = HeapTupleHeaderGetNatts(ctx.tuphdr);
/* Ok, ready to check this next tuple */
check_tuple(&ctx);
check_tuple(&ctx,
&xmin_commit_status_ok[ctx.offnum],
&xmin_commit_status[ctx.offnum]);
/*
* If the CTID field of this tuple seems to point to another tuple
* on the same page, record that tuple as the successor of this
* one.
*/
nextblkno = ItemPointerGetBlockNumber(&(ctx.tuphdr)->t_ctid);
nextoffnum = ItemPointerGetOffsetNumber(&(ctx.tuphdr)->t_ctid);
if (nextblkno == ctx.blkno && nextoffnum != ctx.offnum)
successor[ctx.offnum] = nextoffnum;
}
/*
* Update chain validation. Check each line pointer that's got a valid
* successor against that successor.
*/
ctx.attnum = -1;
for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
ctx.offnum = OffsetNumberNext(ctx.offnum))
{
ItemId curr_lp;
ItemId next_lp;
HeapTupleHeader curr_htup;
HeapTupleHeader next_htup;
TransactionId curr_xmin;
TransactionId curr_xmax;
TransactionId next_xmin;
OffsetNumber nextoffnum = successor[ctx.offnum];
/*
* The current line pointer may not have a successor, either
* because it's not valid or because it didn't point to anything.
* In either case, we have to give up.
*
* If the current line pointer does point to something, it's
* possible that the target line pointer isn't valid. We have to
* give up in that case, too.
*/
if (nextoffnum == InvalidOffsetNumber || !lp_valid[nextoffnum])
continue;
/* We have two valid line pointers that we can examine. */
curr_lp = PageGetItemId(ctx.page, ctx.offnum);
next_lp = PageGetItemId(ctx.page, nextoffnum);
/* Handle the cases where the current line pointer is a redirect. */
if (ItemIdIsRedirected(curr_lp))
{
/* Can't redirect to another redirect. */
if (ItemIdIsRedirected(next_lp))
{
report_corruption(&ctx,
psprintf("redirected line pointer points to another redirected line pointer at offset %u",
(unsigned) nextoffnum));
continue;
}
/* Can only redirect to a HOT tuple. */
next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
if (!HeapTupleHeaderIsHeapOnly(next_htup))
{
report_corruption(&ctx,
psprintf("redirected line pointer points to a non-heap-only tuple at offset %u",
(unsigned) nextoffnum));
}
/*
* Redirects are created by updates, so successor should be
* the result of an update.
*/
if ((next_htup->t_infomask & HEAP_UPDATED) == 0)
{
report_corruption(&ctx,
psprintf("redirected line pointer points to a non-heap-updated tuple at offset %u",
(unsigned) nextoffnum));
}
/* HOT chains should not intersect. */
if (predecessor[nextoffnum] != InvalidOffsetNumber)
{
report_corruption(&ctx,
psprintf("redirect line pointer points to offset %u, but offset %u also points there",
(unsigned) nextoffnum, (unsigned) predecessor[nextoffnum]));
continue;
}
/*
* This redirect and the tuple to which it points seem to be
* part of an update chain.
*/
predecessor[nextoffnum] = ctx.offnum;
continue;
}
/*
* If the next line pointer is a redirect, or if it's a tuple
* but the XMAX of this tuple doesn't match the XMIN of the next
* tuple, then the two aren't part of the same update chain and
* there is nothing more to do.
*/
if (ItemIdIsRedirected(next_lp))
continue;
curr_htup = (HeapTupleHeader) PageGetItem(ctx.page, curr_lp);
curr_xmax = HeapTupleHeaderGetUpdateXid(curr_htup);
next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
next_xmin = HeapTupleHeaderGetXmin(next_htup);
if (!TransactionIdIsValid(curr_xmax) ||
!TransactionIdEquals(curr_xmax, next_xmin))
continue;
/* HOT chains should not intersect. */
if (predecessor[nextoffnum] != InvalidOffsetNumber)
{
report_corruption(&ctx,
psprintf("tuple points to new version at offset %u, but offset %u also points there",
(unsigned) nextoffnum, (unsigned) predecessor[nextoffnum]));
continue;
}
/*
* This tuple and the tuple to which it points seem to be part
* of an update chain.
*/
predecessor[nextoffnum] = ctx.offnum;
/*
* If the current tuple is marked as HOT-updated, then the next
* tuple should be marked as a heap-only tuple. Conversely, if the
* current tuple isn't marked as HOT-updated, then the next tuple
* shouldn't be marked as a heap-only tuple.
*/
if (!HeapTupleHeaderIsHotUpdated(curr_htup) &&
HeapTupleHeaderIsHeapOnly(next_htup))
{
report_corruption(&ctx,
psprintf("non-heap-only update produced a heap-only tuple at offset %u",
(unsigned) nextoffnum));
}
if (HeapTupleHeaderIsHotUpdated(curr_htup) &&
!HeapTupleHeaderIsHeapOnly(next_htup))
{
report_corruption(&ctx,
psprintf("heap-only update produced a non-heap only tuple at offset %u",
(unsigned) nextoffnum));
}
/*
* If the current tuple's xmin is still in progress but the
* successor tuple's xmin is committed, that's corruption.
*
* NB: We recheck the commit status of the current tuple's xmin
* here, because it might have committed after we checked it and
* before we checked the commit status of the successor tuple's
* xmin. This should be safe because the xmin itself can't have
* changed, only its commit status.
*/
curr_xmin = HeapTupleHeaderGetXmin(curr_htup);
if (xmin_commit_status_ok[ctx.offnum] &&
xmin_commit_status[ctx.offnum] == XID_IN_PROGRESS &&
xmin_commit_status_ok[nextoffnum] &&
xmin_commit_status[nextoffnum] == XID_COMMITTED &&
TransactionIdIsInProgress(curr_xmin))
{
report_corruption(&ctx,
psprintf("tuple with in-progress xmin %u was updated to produce a tuple at offset %u with committed xmin %u",
(unsigned) curr_xmin,
(unsigned) ctx.offnum,
(unsigned) next_xmin));
}
/*
* If the current tuple's xmin is aborted but the successor tuple's
* xmin is in-progress or committed, that's corruption.
*/
if (xmin_commit_status_ok[ctx.offnum] &&
xmin_commit_status[ctx.offnum] == XID_ABORTED &&
xmin_commit_status_ok[nextoffnum])
{
if (xmin_commit_status[nextoffnum] == XID_IN_PROGRESS)
report_corruption(&ctx,
psprintf("tuple with aborted xmin %u was updated to produce a tuple at offset %u with in-progress xmin %u",
(unsigned) curr_xmin,
(unsigned) ctx.offnum,
(unsigned) next_xmin));
else if (xmin_commit_status[nextoffnum] == XID_COMMITTED)
report_corruption(&ctx,
psprintf("tuple with aborted xmin %u was updated to produce a tuple at offset %u with committed xmin %u",
(unsigned) curr_xmin,
(unsigned) ctx.offnum,
(unsigned) next_xmin));
}
}
/*
* An update chain can start either with a non-heap-only tuple or with
* a redirect line pointer, but not with a heap-only tuple.
*
* (This check is in a separate loop because we need the predecessor
* array to be fully populated before we can perform it.)
*/
for (ctx.offnum = FirstOffsetNumber;
ctx.offnum <= maxoff;
ctx.offnum = OffsetNumberNext(ctx.offnum))
{
if (xmin_commit_status_ok[ctx.offnum] &&
(xmin_commit_status[ctx.offnum] == XID_COMMITTED ||
xmin_commit_status[ctx.offnum] == XID_IN_PROGRESS) &&
predecessor[ctx.offnum] == InvalidOffsetNumber)
{
ItemId curr_lp;
curr_lp = PageGetItemId(ctx.page, ctx.offnum);
if (!ItemIdIsRedirected(curr_lp))
{
HeapTupleHeader curr_htup;
curr_htup = (HeapTupleHeader)
PageGetItem(ctx.page, curr_lp);
if (HeapTupleHeaderIsHeapOnly(curr_htup))
report_corruption(&ctx,
psprintf("tuple is root of chain but is marked as heap-only tuple"));
}
}
}
/* clean up */
@ -638,6 +889,7 @@ check_tuple_header(HeapCheckContext *ctx)
{
HeapTupleHeader tuphdr = ctx->tuphdr;
uint16 infomask = tuphdr->t_infomask;
TransactionId curr_xmax = HeapTupleHeaderGetUpdateXid(tuphdr);
bool result = true;
unsigned expected_hoff;
@ -663,6 +915,19 @@ check_tuple_header(HeapCheckContext *ctx)
*/
}
if (!TransactionIdIsValid(curr_xmax) &&
HeapTupleHeaderIsHotUpdated(tuphdr))
{
report_corruption(ctx,
psprintf("tuple has been HOT updated, but xmax is 0"));
/*
* As above, even though this shouldn't happen, it's not sufficient
* justification for skipping further checks, we should still be able
* to perform sensibly.
*/
}
if (infomask & HEAP_HASNULL)
expected_hoff = MAXALIGN(SizeofHeapTupleHeader + BITMAPLEN(ctx->natts));
else
@ -718,9 +983,14 @@ check_tuple_header(HeapCheckContext *ctx)
* Returns true if the tuple itself should be checked, false otherwise. Sets
* ctx->tuple_could_be_pruned if the tuple -- and thus also any associated
* TOAST tuples -- are eligible for pruning.
*
* Sets *xmin_commit_status_ok to true if the commit status of xmin is known
* and false otherwise. If it's set to true, then also set *xid_commit_status
* to the actual commit status.
*/
static bool
check_tuple_visibility(HeapCheckContext *ctx)
check_tuple_visibility(HeapCheckContext *ctx, bool *xmin_commit_status_ok,
XidCommitStatus *xmin_commit_status)
{
TransactionId xmin;
TransactionId xvac;
@ -731,13 +1001,17 @@ check_tuple_visibility(HeapCheckContext *ctx)
HeapTupleHeader tuphdr = ctx->tuphdr;
ctx->tuple_could_be_pruned = true; /* have not yet proven otherwise */
*xmin_commit_status_ok = false; /* have not yet proven otherwise */
/* If xmin is normal, it should be within valid range */
xmin = HeapTupleHeaderGetXmin(tuphdr);
switch (get_xid_status(xmin, ctx, &xmin_status))
{
case XID_INVALID:
break;
case XID_BOUNDS_OK:
*xmin_commit_status_ok = true;
*xmin_commit_status = xmin_status;
break;
case XID_IN_FUTURE:
report_corruption(ctx,
@ -1515,9 +1789,13 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
/*
* Check the current tuple as tracked in ctx, recording any corruption found in
* ctx->tupstore.
*
* We return some information about the status of xmin to aid in validating
* update chains.
*/
static void
check_tuple(HeapCheckContext *ctx)
check_tuple(HeapCheckContext *ctx, bool *xmin_commit_status_ok,
XidCommitStatus *xmin_commit_status)
{
/*
* Check various forms of tuple header corruption, and if the header is
@ -1531,7 +1809,8 @@ check_tuple(HeapCheckContext *ctx)
* cannot assume our relation description matches the tuple structure, and
* therefore cannot check it.
*/
if (!check_tuple_visibility(ctx))
if (!check_tuple_visibility(ctx, xmin_commit_status_ok,
xmin_commit_status))
return;
/*

View File

@ -464,6 +464,23 @@ PostgreSQL documentation
Other, less commonly used, options are also available:
<variablelist>
<varlistentry id="app-initdb-option-set">
<term><option>-c <replaceable>name</replaceable>=<replaceable>value</replaceable></option></term>
<term><option>--set <replaceable>name</replaceable>=<replaceable>value</replaceable></option></term>
<listitem>
<para>
Forcibly set the server parameter <replaceable>name</replaceable>
to <replaceable>value</replaceable> during <command>initdb</command>,
and also install that setting in the
generated <filename>postgresql.conf</filename> file,
so that it will apply during future server runs.
This option can be given more than once to set several parameters.
It is primarily useful when the environment is such that the server
will not start at all using the default parameters.
</para>
</listitem>
</varlistentry>
<varlistentry id="app-initdb-option-debug">
<term><option>-d</option></term>
<term><option>--debug</option></term>

View File

@ -3833,11 +3833,9 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator,
LockBuffer(srcBuf, BUFFER_LOCK_SHARE);
srcPage = BufferGetPage(srcBuf);
/* Use P_NEW to extend the destination relation. */
dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum, blkno,
RBM_NORMAL, bstrategy_dst,
RBM_ZERO_AND_LOCK, bstrategy_dst,
permanent);
LockBuffer(dstBuf, BUFFER_LOCK_EXCLUSIVE);
dstPage = BufferGetPage(dstBuf);
START_CRIT_SECTION();
@ -3855,6 +3853,9 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator,
UnlockReleaseBuffer(dstBuf);
UnlockReleaseBuffer(srcBuf);
}
FreeAccessStrategy(bstrategy_src);
FreeAccessStrategy(bstrategy_dst);
}
/* ---------------------------------------------------------------------

View File

@ -85,6 +85,13 @@
/* Ideally this would be in a .h file, but it hardly seems worth the trouble */
extern const char *select_default_timezone(const char *share_path);
/* simple list of strings */
typedef struct _stringlist
{
char *str;
struct _stringlist *next;
} _stringlist;
static const char *const auth_methods_host[] = {
"trust", "reject", "scram-sha-256", "md5", "password", "ident", "radius",
#ifdef ENABLE_GSS
@ -150,6 +157,8 @@ static char *pwfilename = NULL;
static char *superuser_password = NULL;
static const char *authmethodhost = NULL;
static const char *authmethodlocal = NULL;
static _stringlist *extra_guc_names = NULL;
static _stringlist *extra_guc_values = NULL;
static bool debug = false;
static bool noclean = false;
static bool noinstructions = false;
@ -250,7 +259,10 @@ static char backend_exec[MAXPGPATH];
static char **replace_token(char **lines,
const char *token, const char *replacement);
static char **replace_guc_value(char **lines,
const char *guc_name, const char *guc_value,
bool mark_as_comment);
static bool guc_value_requires_quotes(const char *guc_value);
static char **readfile(const char *path);
static void writefile(char *path, char **lines);
static FILE *popen_check(const char *command, const char *mode);
@ -261,6 +273,7 @@ static void check_input(char *path);
static void write_version_file(const char *extrapath);
static void set_null_conf(void);
static void test_config_settings(void);
static bool test_specific_config_settings(int test_conns, int test_buffs);
static void setup_config(void);
static void bootstrap_template1(void);
static void setup_auth(FILE *cmdfd);
@ -368,43 +381,56 @@ escape_quotes_bki(const char *src)
}
/*
* make a copy of the array of lines, with token replaced by replacement
* Add an item at the end of a stringlist.
*/
static void
add_stringlist_item(_stringlist **listhead, const char *str)
{
_stringlist *newentry = pg_malloc(sizeof(_stringlist));
_stringlist *oldentry;
newentry->str = pg_strdup(str);
newentry->next = NULL;
if (*listhead == NULL)
*listhead = newentry;
else
{
for (oldentry = *listhead; oldentry->next; oldentry = oldentry->next)
/* skip */ ;
oldentry->next = newentry;
}
}
/*
* Modify the array of lines, replacing "token" by "replacement"
* the first time it occurs on each line.
*
* The array must be a malloc'd array of individually malloc'd strings.
* We free any discarded strings.
*
* This does most of what sed was used for in the shell script, but
* doesn't need any regexp stuff.
*/
static char **
replace_token(char **lines, const char *token, const char *replacement)
{
int numlines = 1;
int i;
char **result;
int toklen,
replen,
diff;
for (i = 0; lines[i]; i++)
numlines++;
result = (char **) pg_malloc(numlines * sizeof(char *));
toklen = strlen(token);
replen = strlen(replacement);
diff = replen - toklen;
for (i = 0; i < numlines; i++)
for (int i = 0; lines[i]; i++)
{
char *where;
char *newline;
int pre;
/* just copy pointer if NULL or no change needed */
if (lines[i] == NULL || (where = strstr(lines[i], token)) == NULL)
{
result[i] = lines[i];
/* nothing to do if no change needed */
if ((where = strstr(lines[i], token)) == NULL)
continue;
}
/* if we get here a change is needed - set up new line */
@ -418,14 +444,170 @@ replace_token(char **lines, const char *token, const char *replacement)
strcpy(newline + pre + replen, lines[i] + pre + toklen);
result[i] = newline;
free(lines[i]);
lines[i] = newline;
}
return result;
return lines;
}
/*
* Modify the array of lines, replacing the possibly-commented-out
* assignment of parameter guc_name with a live assignment of guc_value.
* The value will be suitably quoted.
*
* If mark_as_comment is true, the replacement line is prefixed with '#'.
* This is used for fixing up cases where the effective default might not
* match what is in postgresql.conf.sample.
*
* We assume there's at most one matching assignment. If we find no match,
* append a new line with the desired assignment.
*
* The array must be a malloc'd array of individually malloc'd strings.
* We free any discarded strings.
*/
static char **
replace_guc_value(char **lines, const char *guc_name, const char *guc_value,
bool mark_as_comment)
{
int namelen = strlen(guc_name);
PQExpBuffer newline = createPQExpBuffer();
int i;
/* prepare the replacement line, except for possible comment and newline */
if (mark_as_comment)
appendPQExpBufferChar(newline, '#');
appendPQExpBuffer(newline, "%s = ", guc_name);
if (guc_value_requires_quotes(guc_value))
appendPQExpBuffer(newline, "'%s'", escape_quotes(guc_value));
else
appendPQExpBufferStr(newline, guc_value);
for (i = 0; lines[i]; i++)
{
const char *where;
/*
* Look for a line assigning to guc_name. Typically it will be
* preceded by '#', but that might not be the case if a -c switch
* overrides a previous assignment. We allow leading whitespace too,
* although normally there wouldn't be any.
*/
where = lines[i];
while (*where == '#' || isspace((unsigned char) *where))
where++;
if (strncmp(where, guc_name, namelen) != 0)
continue;
where += namelen;
while (isspace((unsigned char) *where))
where++;
if (*where != '=')
continue;
/* found it -- append the original comment if any */
where = strrchr(where, '#');
if (where)
{
/*
* We try to preserve original indentation, which is tedious.
* oldindent and newindent are measured in de-tab-ified columns.
*/
const char *ptr;
int oldindent = 0;
int newindent;
for (ptr = lines[i]; ptr < where; ptr++)
{
if (*ptr == '\t')
oldindent += 8 - (oldindent % 8);
else
oldindent++;
}
/* ignore the possibility of tabs in guc_value */
newindent = newline->len;
/* append appropriate tabs and spaces, forcing at least one */
oldindent = Max(oldindent, newindent + 1);
while (newindent < oldindent)
{
int newindent_if_tab = newindent + 8 - (newindent % 8);
if (newindent_if_tab <= oldindent)
{
appendPQExpBufferChar(newline, '\t');
newindent = newindent_if_tab;
}
else
{
appendPQExpBufferChar(newline, ' ');
newindent++;
}
}
/* and finally append the old comment */
appendPQExpBufferStr(newline, where);
/* we'll have appended the original newline; don't add another */
}
else
appendPQExpBufferChar(newline, '\n');
free(lines[i]);
lines[i] = newline->data;
break; /* assume there's only one match */
}
if (lines[i] == NULL)
{
/*
* No match, so append a new entry. (We rely on the bootstrap server
* to complain if it's not a valid GUC name.)
*/
appendPQExpBufferChar(newline, '\n');
lines = pg_realloc_array(lines, char *, i + 2);
lines[i++] = newline->data;
lines[i] = NULL; /* keep the array null-terminated */
}
free(newline); /* but don't free newline->data */
return lines;
}
/*
* Decide if we should quote a replacement GUC value. We aren't too tense
* here, but we'd like to avoid quoting simple identifiers and numbers
* with units, which are common cases.
*/
static bool
guc_value_requires_quotes(const char *guc_value)
{
/* Don't use <ctype.h> macros here, they might accept too much */
#define LETTERS "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
#define DIGITS "0123456789"
if (*guc_value == '\0')
return true; /* empty string must be quoted */
if (strchr(LETTERS, *guc_value))
{
if (strspn(guc_value, LETTERS DIGITS) == strlen(guc_value))
return false; /* it's an identifier */
return true; /* nope */
}
if (strchr(DIGITS, *guc_value))
{
/* skip over digits */
guc_value += strspn(guc_value, DIGITS);
/* there can be zero or more unit letters after the digits */
if (strspn(guc_value, LETTERS) == strlen(guc_value))
return false; /* it's a number, possibly with units */
return true; /* nope */
}
return true; /* all else must be quoted */
}
/*
* get the lines from a text file
*
* The result is a malloc'd array of individually malloc'd strings.
*/
static char **
readfile(const char *path)
@ -468,6 +650,9 @@ readfile(const char *path)
/*
* write an array of lines to a file
*
* "lines" must be a malloc'd array of individually malloc'd strings.
* All that data is freed here.
*
* This is only used to write text files. Use fopen "w" not PG_BINARY_W
* so that the resulting configuration files are nicely editable on Windows.
*/
@ -487,6 +672,7 @@ writefile(char *path, char **lines)
}
if (fclose(out_file))
pg_fatal("could not close file \"%s\": %m", path);
free(lines);
}
/*
@ -881,11 +1067,9 @@ test_config_settings(void)
400, 300, 200, 100, 50
};
char cmd[MAXPGPATH];
const int connslen = sizeof(trial_conns) / sizeof(int);
const int bufslen = sizeof(trial_bufs) / sizeof(int);
int i,
status,
test_conns,
test_buffs,
ok_buffers = 0;
@ -911,19 +1095,7 @@ test_config_settings(void)
test_conns = trial_conns[i];
test_buffs = MIN_BUFS_FOR_CONNS(test_conns);
snprintf(cmd, sizeof(cmd),
"\"%s\" --check %s %s "
"-c max_connections=%d "
"-c shared_buffers=%d "
"-c dynamic_shared_memory_type=%s "
"< \"%s\" > \"%s\" 2>&1",
backend_exec, boot_options, extra_options,
test_conns, test_buffs,
dynamic_shared_memory_type,
DEVNULL, DEVNULL);
fflush(NULL);
status = system(cmd);
if (status == 0)
if (test_specific_config_settings(test_conns, test_buffs))
{
ok_buffers = test_buffs;
break;
@ -948,19 +1120,7 @@ test_config_settings(void)
break;
}
snprintf(cmd, sizeof(cmd),
"\"%s\" --check %s %s "
"-c max_connections=%d "
"-c shared_buffers=%d "
"-c dynamic_shared_memory_type=%s "
"< \"%s\" > \"%s\" 2>&1",
backend_exec, boot_options, extra_options,
n_connections, test_buffs,
dynamic_shared_memory_type,
DEVNULL, DEVNULL);
fflush(NULL);
status = system(cmd);
if (status == 0)
if (test_specific_config_settings(n_connections, test_buffs))
break;
}
n_buffers = test_buffs;
@ -976,6 +1136,48 @@ test_config_settings(void)
printf("%s\n", default_timezone ? default_timezone : "GMT");
}
/*
* Test a specific combination of configuration settings.
*/
static bool
test_specific_config_settings(int test_conns, int test_buffs)
{
PQExpBuffer cmd = createPQExpBuffer();
_stringlist *gnames,
*gvalues;
int status;
/* Set up the test postmaster invocation */
printfPQExpBuffer(cmd,
"\"%s\" --check %s %s "
"-c max_connections=%d "
"-c shared_buffers=%d "
"-c dynamic_shared_memory_type=%s",
backend_exec, boot_options, extra_options,
test_conns, test_buffs,
dynamic_shared_memory_type);
/* Add any user-given setting overrides */
for (gnames = extra_guc_names, gvalues = extra_guc_values;
gnames != NULL; /* assume lists have the same length */
gnames = gnames->next, gvalues = gvalues->next)
{
appendPQExpBuffer(cmd, " -c %s=", gnames->str);
appendShellString(cmd, gvalues->str);
}
appendPQExpBuffer(cmd,
" < \"%s\" > \"%s\" 2>&1",
DEVNULL, DEVNULL);
fflush(NULL);
status = system(cmd->data);
destroyPQExpBuffer(cmd);
return (status == 0);
}
/*
* Calculate the default wal_size with a "pretty" unit.
*/
@ -1002,7 +1204,8 @@ setup_config(void)
char **conflines;
char repltok[MAXPGPATH];
char path[MAXPGPATH];
char *autoconflines[3];
_stringlist *gnames,
*gvalues;
fputs(_("creating configuration files ... "), stdout);
fflush(stdout);
@ -1011,120 +1214,116 @@ setup_config(void)
conflines = readfile(conf_file);
snprintf(repltok, sizeof(repltok), "max_connections = %d", n_connections);
conflines = replace_token(conflines, "#max_connections = 100", repltok);
snprintf(repltok, sizeof(repltok), "%d", n_connections);
conflines = replace_guc_value(conflines, "max_connections",
repltok, false);
if ((n_buffers * (BLCKSZ / 1024)) % 1024 == 0)
snprintf(repltok, sizeof(repltok), "shared_buffers = %dMB",
snprintf(repltok, sizeof(repltok), "%dMB",
(n_buffers * (BLCKSZ / 1024)) / 1024);
else
snprintf(repltok, sizeof(repltok), "shared_buffers = %dkB",
snprintf(repltok, sizeof(repltok), "%dkB",
n_buffers * (BLCKSZ / 1024));
conflines = replace_token(conflines, "#shared_buffers = 128MB", repltok);
conflines = replace_guc_value(conflines, "shared_buffers",
repltok, false);
snprintf(repltok, sizeof(repltok), "#unix_socket_directories = '%s'",
DEFAULT_PGSOCKET_DIR);
conflines = replace_token(conflines, "#unix_socket_directories = '/tmp'",
repltok);
/*
* Hack: don't replace the LC_XXX GUCs when their value is 'C', because
* replace_guc_value will decide not to quote that, which looks strange.
*/
if (strcmp(lc_messages, "C") != 0)
conflines = replace_guc_value(conflines, "lc_messages",
lc_messages, false);
#if DEF_PGPORT != 5432
snprintf(repltok, sizeof(repltok), "#port = %d", DEF_PGPORT);
conflines = replace_token(conflines, "#port = 5432", repltok);
#endif
if (strcmp(lc_monetary, "C") != 0)
conflines = replace_guc_value(conflines, "lc_monetary",
lc_monetary, false);
/* set default max_wal_size and min_wal_size */
snprintf(repltok, sizeof(repltok), "min_wal_size = %s",
pretty_wal_size(DEFAULT_MIN_WAL_SEGS));
conflines = replace_token(conflines, "#min_wal_size = 80MB", repltok);
if (strcmp(lc_numeric, "C") != 0)
conflines = replace_guc_value(conflines, "lc_numeric",
lc_numeric, false);
snprintf(repltok, sizeof(repltok), "max_wal_size = %s",
pretty_wal_size(DEFAULT_MAX_WAL_SEGS));
conflines = replace_token(conflines, "#max_wal_size = 1GB", repltok);
snprintf(repltok, sizeof(repltok), "lc_messages = '%s'",
escape_quotes(lc_messages));
conflines = replace_token(conflines, "#lc_messages = 'C'", repltok);
snprintf(repltok, sizeof(repltok), "lc_monetary = '%s'",
escape_quotes(lc_monetary));
conflines = replace_token(conflines, "#lc_monetary = 'C'", repltok);
snprintf(repltok, sizeof(repltok), "lc_numeric = '%s'",
escape_quotes(lc_numeric));
conflines = replace_token(conflines, "#lc_numeric = 'C'", repltok);
snprintf(repltok, sizeof(repltok), "lc_time = '%s'",
escape_quotes(lc_time));
conflines = replace_token(conflines, "#lc_time = 'C'", repltok);
if (strcmp(lc_time, "C") != 0)
conflines = replace_guc_value(conflines, "lc_time",
lc_time, false);
switch (locale_date_order(lc_time))
{
case DATEORDER_YMD:
strcpy(repltok, "datestyle = 'iso, ymd'");
strcpy(repltok, "iso, ymd");
break;
case DATEORDER_DMY:
strcpy(repltok, "datestyle = 'iso, dmy'");
strcpy(repltok, "iso, dmy");
break;
case DATEORDER_MDY:
default:
strcpy(repltok, "datestyle = 'iso, mdy'");
strcpy(repltok, "iso, mdy");
break;
}
conflines = replace_token(conflines, "#datestyle = 'iso, mdy'", repltok);
conflines = replace_guc_value(conflines, "datestyle",
repltok, false);
snprintf(repltok, sizeof(repltok),
"default_text_search_config = 'pg_catalog.%s'",
escape_quotes(default_text_search_config));
conflines = replace_token(conflines,
"#default_text_search_config = 'pg_catalog.simple'",
repltok);
snprintf(repltok, sizeof(repltok), "pg_catalog.%s",
default_text_search_config);
conflines = replace_guc_value(conflines, "default_text_search_config",
repltok, false);
if (default_timezone)
{
snprintf(repltok, sizeof(repltok), "timezone = '%s'",
escape_quotes(default_timezone));
conflines = replace_token(conflines, "#timezone = 'GMT'", repltok);
snprintf(repltok, sizeof(repltok), "log_timezone = '%s'",
escape_quotes(default_timezone));
conflines = replace_token(conflines, "#log_timezone = 'GMT'", repltok);
conflines = replace_guc_value(conflines, "timezone",
default_timezone, false);
conflines = replace_guc_value(conflines, "log_timezone",
default_timezone, false);
}
snprintf(repltok, sizeof(repltok), "dynamic_shared_memory_type = %s",
dynamic_shared_memory_type);
conflines = replace_token(conflines, "#dynamic_shared_memory_type = posix",
repltok);
conflines = replace_guc_value(conflines, "dynamic_shared_memory_type",
dynamic_shared_memory_type, false);
/*
* Fix up various entries to match the true compile-time defaults. Since
* these are indeed defaults, keep the postgresql.conf lines commented.
*/
conflines = replace_guc_value(conflines, "unix_socket_directories",
DEFAULT_PGSOCKET_DIR, true);
conflines = replace_guc_value(conflines, "port",
DEF_PGPORT_STR, true);
conflines = replace_guc_value(conflines, "min_wal_size",
pretty_wal_size(DEFAULT_MIN_WAL_SEGS), true);
conflines = replace_guc_value(conflines, "max_wal_size",
pretty_wal_size(DEFAULT_MAX_WAL_SEGS), true);
#if DEFAULT_BACKEND_FLUSH_AFTER > 0
snprintf(repltok, sizeof(repltok), "#backend_flush_after = %dkB",
snprintf(repltok, sizeof(repltok), "%dkB",
DEFAULT_BACKEND_FLUSH_AFTER * (BLCKSZ / 1024));
conflines = replace_token(conflines, "#backend_flush_after = 0",
repltok);
conflines = replace_guc_value(conflines, "backend_flush_after",
repltok, true);
#endif
#if DEFAULT_BGWRITER_FLUSH_AFTER > 0
snprintf(repltok, sizeof(repltok), "#bgwriter_flush_after = %dkB",
snprintf(repltok, sizeof(repltok), "%dkB",
DEFAULT_BGWRITER_FLUSH_AFTER * (BLCKSZ / 1024));
conflines = replace_token(conflines, "#bgwriter_flush_after = 0",
repltok);
conflines = replace_guc_value(conflines, "bgwriter_flush_after",
repltok, true);
#endif
#if DEFAULT_CHECKPOINT_FLUSH_AFTER > 0
snprintf(repltok, sizeof(repltok), "#checkpoint_flush_after = %dkB",
snprintf(repltok, sizeof(repltok), "%dkB",
DEFAULT_CHECKPOINT_FLUSH_AFTER * (BLCKSZ / 1024));
conflines = replace_token(conflines, "#checkpoint_flush_after = 0",
repltok);
conflines = replace_guc_value(conflines, "checkpoint_flush_after",
repltok, true);
#endif
#ifndef USE_PREFETCH
conflines = replace_token(conflines,
"#effective_io_concurrency = 1",
"#effective_io_concurrency = 0");
conflines = replace_guc_value(conflines, "effective_io_concurrency",
"0", true);
#endif
#ifdef WIN32
conflines = replace_token(conflines,
"#update_process_title = on",
"#update_process_title = off");
conflines = replace_guc_value(conflines, "update_process_title",
"off", true);
#endif
/*
@ -1136,9 +1335,8 @@ setup_config(void)
(strcmp(authmethodhost, "md5") == 0 &&
strcmp(authmethodlocal, "scram-sha-256") != 0))
{
conflines = replace_token(conflines,
"#password_encryption = scram-sha-256",
"password_encryption = md5");
conflines = replace_guc_value(conflines, "password_encryption",
"md5", false);
}
/*
@ -1149,35 +1347,42 @@ setup_config(void)
*/
if (pg_dir_create_mode == PG_DIR_MODE_GROUP)
{
conflines = replace_token(conflines,
"#log_file_mode = 0600",
"log_file_mode = 0640");
conflines = replace_guc_value(conflines, "log_file_mode",
"0640", false);
}
/*
* Now replace anything that's overridden via -c switches.
*/
for (gnames = extra_guc_names, gvalues = extra_guc_values;
gnames != NULL; /* assume lists have the same length */
gnames = gnames->next, gvalues = gvalues->next)
{
conflines = replace_guc_value(conflines, gnames->str,
gvalues->str, false);
}
/* ... and write out the finished postgresql.conf file */
snprintf(path, sizeof(path), "%s/postgresql.conf", pg_data);
writefile(path, conflines);
if (chmod(path, pg_file_create_mode) != 0)
pg_fatal("could not change permissions of \"%s\": %m", path);
/*
* create the automatic configuration file to store the configuration
* parameters set by ALTER SYSTEM command. The parameters present in this
* file will override the value of parameters that exists before parse of
* this file.
*/
autoconflines[0] = pg_strdup("# Do not edit this file manually!\n");
autoconflines[1] = pg_strdup("# It will be overwritten by the ALTER SYSTEM command.\n");
autoconflines[2] = NULL;
/* postgresql.auto.conf */
conflines = pg_malloc_array(char *, 3);
conflines[0] = pg_strdup("# Do not edit this file manually!\n");
conflines[1] = pg_strdup("# It will be overwritten by the ALTER SYSTEM command.\n");
conflines[2] = NULL;
sprintf(path, "%s/postgresql.auto.conf", pg_data);
writefile(path, autoconflines);
writefile(path, conflines);
if (chmod(path, pg_file_create_mode) != 0)
pg_fatal("could not change permissions of \"%s\": %m", path);
free(conflines);
/* pg_hba.conf */
@ -1245,7 +1450,6 @@ setup_config(void)
if (chmod(path, pg_file_create_mode) != 0)
pg_fatal("could not change permissions of \"%s\": %m", path);
free(conflines);
/* pg_ident.conf */
@ -1257,8 +1461,6 @@ setup_config(void)
if (chmod(path, pg_file_create_mode) != 0)
pg_fatal("could not change permissions of \"%s\": %m", path);
free(conflines);
check_ok();
}
@ -2183,6 +2385,7 @@ usage(const char *progname)
printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n"));
printf(_(" --wal-segsize=SIZE size of WAL segments, in megabytes\n"));
printf(_("\nLess commonly used options:\n"));
printf(_(" -c, --set NAME=VALUE override default setting for server parameter\n"));
printf(_(" -d, --debug generate lots of debugging output\n"));
printf(_(" --discard-caches set debug_discard_caches=1\n"));
printf(_(" -L DIRECTORY where to find the input files\n"));
@ -2819,6 +3022,7 @@ main(int argc, char *argv[])
{"nosync", no_argument, NULL, 'N'}, /* for backwards compatibility */
{"no-sync", no_argument, NULL, 'N'},
{"no-instructions", no_argument, NULL, 13},
{"set", required_argument, NULL, 'c'},
{"sync-only", no_argument, NULL, 'S'},
{"waldir", required_argument, NULL, 'X'},
{"wal-segsize", required_argument, NULL, 12},
@ -2869,7 +3073,8 @@ main(int argc, char *argv[])
/* process command-line options */
while ((c = getopt_long(argc, argv, "A:dD:E:gkL:nNsST:U:WX:", long_options, &option_index)) != -1)
while ((c = getopt_long(argc, argv, "A:c:dD:E:gkL:nNsST:U:WX:",
long_options, &option_index)) != -1)
{
switch (c)
{
@ -2892,6 +3097,24 @@ main(int argc, char *argv[])
case 11:
authmethodhost = pg_strdup(optarg);
break;
case 'c':
{
char *buf = pg_strdup(optarg);
char *equals = strchr(buf, '=');
if (!equals)
{
pg_log_error("-c %s requires a value", buf);
pg_log_error_hint("Try \"%s --help\" for more information.",
progname);
exit(1);
}
*equals++ = '\0'; /* terminate variable name */
add_stringlist_item(&extra_guc_names, buf);
add_stringlist_item(&extra_guc_values, equals);
pfree(buf);
}
break;
case 'D':
pg_data = pg_strdup(optarg);
break;

View File

@ -48,7 +48,13 @@ mkdir $datadir;
local (%ENV) = %ENV;
delete $ENV{TZ};
command_ok([ 'initdb', '-N', '-T', 'german', '-X', $xlogdir, $datadir ],
# while we are here, also exercise -T and -c options
command_ok(
[
'initdb', '-N', '-T', 'german', '-c',
'default_text_search_config=german',
'-X', $xlogdir, $datadir
],
'successful creation');
# Permissions on PGDATA should be default
@ -142,4 +148,7 @@ command_fails(
],
'fails for invalid option combination');
command_fails([ 'initdb', '--no-sync', '--set', 'foo=bar', "$tempdir/dataX" ],
'fails for invalid --set option');
done_testing();

View File

@ -174,12 +174,16 @@ sub write_tuple
# Set umask so test directories and files are created with default permissions
umask(0077);
my $pred_xmax;
my $pred_posid;
my $aborted_xid;
# Set up the node. Once we create and corrupt the table,
# autovacuum workers visiting the table could crash the backend.
# Disable autovacuum so that won't happen.
my $node = PostgreSQL::Test::Cluster->new('test');
$node->init;
$node->append_conf('postgresql.conf', 'autovacuum=off');
$node->append_conf('postgresql.conf', 'max_prepared_transactions=10');
# Start the node and load the extensions. We depend on both
# amcheck and pageinspect for this test.
@ -216,8 +220,15 @@ my $rel = $node->safe_psql('postgres',
qq(SELECT pg_relation_filepath('public.test')));
my $relpath = "$pgdata/$rel";
# Insert data and freeze public.test
my $ROWCOUNT = 17;
# Initial setup for the public.test table.
# $ROWCOUNT is the total number of rows that we expect to insert into the page.
# $ROWCOUNT_BASIC is the number of those rows that are related to basic
# tuple validation, rather than update chain validation.
my $ROWCOUNT = 44;
my $ROWCOUNT_BASIC = 16;
# First insert data needed for tests unrelated to update chain validation.
# Then freeze the page. These tuples are at offset numbers 1 to 16.
$node->safe_psql(
'postgres', qq(
INSERT INTO public.test (a, b, c)
@ -225,10 +236,86 @@ $node->safe_psql(
x'DEADF9F9DEADF9F9'::bigint,
'abcdefg',
repeat('w', 10000)
FROM generate_series(1, $ROWCOUNT);
FROM generate_series(1, $ROWCOUNT_BASIC);
VACUUM FREEZE public.test;)
);
# Create some simple HOT update chains for line pointer validation. After
# the page is HOT pruned, we'll have two redirects line pointers each pointing
# to a tuple. We'll then change the second redirect to point to the same
# tuple as the first one and verify that we can detect corruption.
$node->safe_psql(
'postgres', qq(
INSERT INTO public.test (a, b, c)
VALUES ( x'DEADF9F9DEADF9F9'::bigint, 'abcdefg',
generate_series(1,2)); -- offset numbers 17 and 18
UPDATE public.test SET c = 'a' WHERE c = '1'; -- offset number 19
UPDATE public.test SET c = 'a' WHERE c = '2'; -- offset number 20
));
# Create some more HOT update chains.
$node->safe_psql(
'postgres', qq(
INSERT INTO public.test (a, b, c)
VALUES ( x'DEADF9F9DEADF9F9'::bigint, 'abcdefg',
generate_series(3,6)); -- offset numbers 21 through 24
UPDATE public.test SET c = 'a' WHERE c = '3'; -- offset number 25
UPDATE public.test SET c = 'a' WHERE c = '4'; -- offset number 26
));
# Negative test case of HOT-pruning with aborted tuple.
$node->safe_psql(
'postgres', qq(
BEGIN;
UPDATE public.test SET c = 'a' WHERE c = '5'; -- offset number 27
ABORT;
VACUUM FREEZE public.test;
));
# Next update on any tuple will be stored at the same place of tuple inserted
# by aborted transaction. This should not cause the table to appear corrupt.
$node->safe_psql(
'postgres', qq(
UPDATE public.test SET c = 'a' WHERE c = '6'; -- offset number 27 again
VACUUM FREEZE public.test;
));
# Data for HOT chain validation, so not calling VACUUM FREEZE.
$node->safe_psql(
'postgres', qq(
INSERT INTO public.test (a, b, c)
VALUES ( x'DEADF9F9DEADF9F9'::bigint, 'abcdefg',
generate_series(7,15)); -- offset numbers 28 to 36
UPDATE public.test SET c = 'a' WHERE c = '7'; -- offset number 37
UPDATE public.test SET c = 'a' WHERE c = '10'; -- offset number 38
UPDATE public.test SET c = 'a' WHERE c = '11'; -- offset number 39
UPDATE public.test SET c = 'a' WHERE c = '12'; -- offset number 40
UPDATE public.test SET c = 'a' WHERE c = '13'; -- offset number 41
UPDATE public.test SET c = 'a' WHERE c = '14'; -- offset number 42
UPDATE public.test SET c = 'a' WHERE c = '15'; -- offset number 43
));
# Need one aborted transaction to test corruption in HOT chains.
$node->safe_psql(
'postgres', qq(
BEGIN;
UPDATE public.test SET c = 'a' WHERE c = '9'; -- offset number 44
ABORT;
));
# Need one in-progress transaction to test few corruption in HOT chains.
# We are creating PREPARE TRANSACTION here as these will not be aborted
# even if we stop the node.
$node->safe_psql(
'postgres', qq(
BEGIN;
PREPARE TRANSACTION 'in_progress_tx';
));
my $in_progress_xid = $node->safe_psql(
'postgres', qq(
SELECT transaction FROM pg_prepared_xacts;
));
my $relfrozenxid = $node->safe_psql('postgres',
q(select relfrozenxid from pg_class where relname = 'test'));
my $datfrozenxid = $node->safe_psql('postgres',
@ -245,11 +332,12 @@ if ($datfrozenxid <= 3 || $datfrozenxid >= $relfrozenxid)
exit;
}
# Find where each of the tuples is located on the page.
# Find where each of the tuples is located on the page. If a particular
# line pointer is a redirect rather than a tuple, we record the offset as -1.
my @lp_off = split '\n', $node->safe_psql(
'postgres', qq(
select lp_off from heap_page_items(get_raw_page('test', 'main', 0))
where lp <= $ROWCOUNT
SELECT CASE WHEN lp_flags = 2 THEN -1 ELSE lp_off END
FROM heap_page_items(get_raw_page('test', 'main', 0))
)
);
is(scalar @lp_off, $ROWCOUNT, "acquired row offsets");
@ -268,6 +356,7 @@ for (my $tupidx = 0; $tupidx < $ROWCOUNT; $tupidx++)
{
my $offnum = $tupidx + 1; # offnum is 1-based, not zero-based
my $offset = $lp_off[$tupidx];
next if $offset == -1; # ignore redirect line pointers
my $tup = read_tuple($file, $offset);
# Sanity-check that the data appears on the page where we expect.
@ -280,7 +369,7 @@ for (my $tupidx = 0; $tupidx < $ROWCOUNT; $tupidx++)
$node->clean_node;
plan skip_all =>
sprintf(
"Page layout differs from our expectations: expected (%x, %x, \"%s\"), got (%x, %x, \"%s\")",
"Page layout of index %d differs from our expectations: expected (%x, %x, \"%s\"), got (%x, %x, \"%s\")", $tupidx,
0xDEADF9F9, 0xDEADF9F9, "abcdefg", $a_1, $a_2, $b);
exit;
}
@ -315,6 +404,9 @@ use constant HEAP_XMAX_INVALID => 0x0800;
use constant HEAP_NATTS_MASK => 0x07FF;
use constant HEAP_XMAX_IS_MULTI => 0x1000;
use constant HEAP_KEYS_UPDATED => 0x2000;
use constant HEAP_HOT_UPDATED => 0x4000;
use constant HEAP_ONLY_TUPLE => 0x8000;
use constant HEAP_UPDATED => 0x2000;
# Helper function to generate a regular expression matching the header we
# expect verify_heapam() to return given which fields we expect to be non-null.
@ -346,9 +438,11 @@ for (my $tupidx = 0; $tupidx < $ROWCOUNT; $tupidx++)
{
my $offnum = $tupidx + 1; # offnum is 1-based, not zero-based
my $offset = $lp_off[$tupidx];
my $tup = read_tuple($file, $offset);
my $header = header(0, $offnum, undef);
# Read tuple, if there is one.
my $tup = $offset == -1 ? undef : read_tuple($file, $offset);
if ($offnum == 1)
{
# Corruptly set xmin < relfrozenxid
@ -361,7 +455,7 @@ for (my $tupidx = 0; $tupidx < $ROWCOUNT; $tupidx++)
push @expected,
qr/${header}xmin $xmin precedes relation freeze threshold 0:\d+/;
}
if ($offnum == 2)
elsif ($offnum == 2)
{
# Corruptly set xmin < datfrozenxid
my $xmin = 3;
@ -521,7 +615,137 @@ for (my $tupidx = 0; $tupidx < $ROWCOUNT; $tupidx++)
push @expected,
qr/${$header}xmin ${xmin} equals or exceeds next valid transaction ID 0:\d+/;
}
write_tuple($file, $offset, $tup);
elsif ($offnum == 17)
{
# at offnum 19 we will unset HEAP_ONLY_TUPLE and HEAP_UPDATED flags.
die "offnum $offnum should be a redirect" if defined $tup;
push @expected,
qr/${header}redirected line pointer points to a non-heap-only tuple at offset \d+/;
push @expected,
qr/${header}redirected line pointer points to a non-heap-updated tuple at offset \d+/;
}
elsif ($offnum == 18)
{
# rewrite line pointer with lp_off = 17, lp_flags = 2, lp_len = 0.
die "offnum $offnum should be a redirect" if defined $tup;
sysseek($file, 92, 0) or BAIL_OUT("sysseek failed: $!");
syswrite($file,
pack("L", $ENDIANNESS eq 'little' ? 0x00010011 : 0x11000100))
or BAIL_OUT("syswrite failed: $!");
push @expected,
qr/${header}redirected line pointer points to another redirected line pointer at offset \d+/;
}
elsif ($offnum == 19)
{
# unset HEAP_ONLY_TUPLE and HEAP_UPDATED flag, so that update chain
# validation will complain about offset 17
$tup->{t_infomask2} &= ~HEAP_ONLY_TUPLE;
$tup->{t_infomask} &= ~HEAP_UPDATED;
}
elsif ($offnum == 22)
{
# rewrite line pointer with lp.off = 25, lp_flags = 2, lp_len = 0
sysseek($file, 108, 0) or BAIL_OUT("sysseek failed: $!");
syswrite($file,
pack("L", $ENDIANNESS eq 'little' ? 0x00010019 : 0x19000100))
or BAIL_OUT("syswrite failed: $!");
push @expected,
qr/${header}redirect line pointer points to offset \d+, but offset \d+ also points there/;
}
elsif ($offnum == 28)
{
$tup->{t_infomask2} &= ~HEAP_HOT_UPDATED;
push @expected,
qr/${header}non-heap-only update produced a heap-only tuple at offset \d+/;
# Save these values so we can insert them into the tuple at offnum 29.
$pred_xmax = $tup->{t_xmax};
$pred_posid = $tup->{ip_posid};
}
elsif ($offnum == 29)
{
# Copy these values from the tuple at offset 28.
$tup->{t_xmax} = $pred_xmax;
$tup->{ip_posid} = $pred_posid;
push @expected,
qr/${header}tuple points to new version at offset \d+, but offset \d+ also points there/;
}
elsif ($offnum == 30)
{
# Save xid, so we can insert into into tuple at offset 31.
$aborted_xid = $tup->{t_xmax};
}
elsif ($offnum == 31)
{
# Set xmin to xmax of tuple at offset 30.
$tup->{t_xmin} = $aborted_xid;
$tup->{t_infomask} &= ~HEAP_XMIN_COMMITTED;
push @expected,
qr/${header}tuple with aborted xmin \d+ was updated to produce a tuple at offset \d+ with committed xmin \d+/;
}
elsif ($offnum == 32)
{
$tup->{t_infomask2} |= HEAP_ONLY_TUPLE;
push @expected,
qr/${header}tuple is root of chain but is marked as heap-only tuple/;
}
elsif ($offnum == 33)
{
# Tuple at offset 40 is the successor of this one; we'll corrupt it to
# be non-heap-only.
push @expected,
qr/${header}heap-only update produced a non-heap only tuple at offset \d+/;
}
elsif ($offnum == 34)
{
$tup->{t_xmax} = 0;
push @expected,
qr/${header}tuple has been HOT updated, but xmax is 0/;
}
elsif ($offnum == 35)
{
$tup->{t_xmin} = $in_progress_xid;
$tup->{t_infomask} &= ~HEAP_XMIN_COMMITTED;
push @expected,
qr/${header}tuple with in-progress xmin \d+ was updated to produce a tuple at offset \d+ with committed xmin \d+/;
}
elsif ($offnum == 36)
{
# Tuple at offset 43 is the successor of this one; we'll corrupt it to
# have xmin = $in_progress_xid. By setting the xmax of this tuple to
# the same value, we make it look like an update chain with an
# in-progress XID following a committed one.
$tup->{t_xmin} = $aborted_xid;
$tup->{t_xmax} = $in_progress_xid;
$tup->{t_infomask} &= ~HEAP_XMIN_COMMITTED;
push @expected,
qr/${header}tuple with aborted xmin \d+ was updated to produce a tuple at offset \d+ with in-progress xmin \d+/;
}
elsif ($offnum == 40)
{
# Tuple at offset 33 is the predecessor of this one; the error will
# be reported there.
$tup->{t_infomask2} &= ~HEAP_ONLY_TUPLE;
}
elsif ($offnum == 43)
{
# Tuple at offset 36 is the predecessor of this one; the error will
# be reported there.
$tup->{t_xmin} = $in_progress_xid;
$tup->{t_infomask} &= ~HEAP_XMIN_COMMITTED;
}
else
{
# The tests for update chain validation end up creating a bunch of
# tuples that aren't corrupted in any way e.g. because only one of
# the two tuples in the update chain needs to be corrupted for the
# test, or because one update chain is being made to erroneously
# point into the middle of another that has nothing wrong with it.
# In all such cases we need not write the tuple back to the file.
next;
}
write_tuple($file, $offset, $tup) if defined $tup;
}
close($file)
or BAIL_OUT("close failed: $!");
@ -532,6 +756,10 @@ $node->start;
$node->command_checks_all(
[ 'pg_amcheck', '--no-dependent-indexes', '-p', $port, 'postgres' ],
2, [@expected], [], 'Expected corruption message output');
$node->safe_psql(
'postgres', qq(
COMMIT PREPARED 'in_progress_tx';
));
$node->teardown_node;
$node->clean_node;