2010-12-29 12:48:53 +01:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* reinit.c
|
|
|
|
* Reinitialization of unlogged relations
|
|
|
|
*
|
2023-01-02 21:00:37 +01:00
|
|
|
* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
|
2010-12-29 12:48:53 +01:00
|
|
|
* Portions Copyright (c) 1994, Regents of the University of California
|
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
|
|
|
* src/backend/storage/file/reinit.c
|
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include <unistd.h>
|
|
|
|
|
2013-02-22 02:46:17 +01:00
|
|
|
#include "common/relpath.h"
|
Report progress of startup operations that take a long time.
Users sometimes get concerned whe they start the server and it
emits a few messages and then doesn't emit any more messages for
a long time. Generally, what's happening is either that the
system is taking a long time to apply WAL, or it's taking a
long time to reset unlogged relations, or it's taking a long
time to fsync the data directory, but it's not easy to tell
which is the case.
To fix that, add a new 'log_startup_progress_interval' setting,
by default 10s. When an operation that is known to be potentially
long-running takes more than this amount of time, we'll log a
status update each time this interval elapses.
To avoid undesirable log chatter, don't log anything about WAL
replay when in standby mode.
Nitin Jadhav and Robert Haas, reviewed by Amul Sul, Bharath
Rupireddy, Justin Pryzby, Michael Paquier, and Álvaro Herrera.
Discussion: https://postgr.es/m/CA+TgmoaHQrgDFOBwgY16XCoMtXxsrVGFB2jNCvb7-ubuEe1MGg@mail.gmail.com
Discussion: https://postgr.es/m/CAMm1aWaHF7VE69572_OLQ+MgpT5RUiUDgF1x5RrtkJBLdpRj3Q@mail.gmail.com
2021-10-25 17:51:57 +02:00
|
|
|
#include "postmaster/startup.h"
|
2010-12-29 12:48:53 +01:00
|
|
|
#include "storage/copydir.h"
|
|
|
|
#include "storage/fd.h"
|
|
|
|
#include "storage/reinit.h"
|
|
|
|
#include "utils/hsearch.h"
|
|
|
|
#include "utils/memutils.h"
|
|
|
|
|
|
|
|
static void ResetUnloggedRelationsInTablespaceDir(const char *tsdirname,
|
|
|
|
int op);
|
|
|
|
static void ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname,
|
|
|
|
int op);
|
|
|
|
|
|
|
|
typedef struct
|
|
|
|
{
|
2022-09-28 15:45:27 +02:00
|
|
|
Oid reloid; /* hash key */
|
2010-12-29 12:48:53 +01:00
|
|
|
} unlogged_relation_entry;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reset unlogged relations from before the last restart.
|
|
|
|
*
|
|
|
|
* If op includes UNLOGGED_RELATION_CLEANUP, we remove all forks of any
|
|
|
|
* relation with an "init" fork, except for the "init" fork itself.
|
|
|
|
*
|
|
|
|
* If op includes UNLOGGED_RELATION_INIT, we copy the "init" fork to the main
|
|
|
|
* fork.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
ResetUnloggedRelations(int op)
|
|
|
|
{
|
2017-04-11 20:13:31 +02:00
|
|
|
char temp_path[MAXPGPATH + 10 + sizeof(TABLESPACE_VERSION_DIRECTORY)];
|
2010-12-29 12:48:53 +01:00
|
|
|
DIR *spc_dir;
|
|
|
|
struct dirent *spc_de;
|
|
|
|
MemoryContext tmpctx,
|
|
|
|
oldctx;
|
|
|
|
|
|
|
|
/* Log it. */
|
2011-07-19 06:50:10 +02:00
|
|
|
elog(DEBUG1, "resetting unlogged relations: cleanup %d init %d",
|
|
|
|
(op & UNLOGGED_RELATION_CLEANUP) != 0,
|
|
|
|
(op & UNLOGGED_RELATION_INIT) != 0);
|
2010-12-29 12:48:53 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Just to be sure we don't leak any memory, let's create a temporary
|
|
|
|
* memory context for this operation.
|
|
|
|
*/
|
|
|
|
tmpctx = AllocSetContextCreate(CurrentMemoryContext,
|
|
|
|
"ResetUnloggedRelations",
|
Add macros to make AllocSetContextCreate() calls simpler and safer.
I found that half a dozen (nearly 5%) of our AllocSetContextCreate calls
had typos in the context-sizing parameters. While none of these led to
especially significant problems, they did create minor inefficiencies,
and it's now clear that expecting people to copy-and-paste those calls
accurately is not a great idea. Let's reduce the risk of future errors
by introducing single macros that encapsulate the common use-cases.
Three such macros are enough to cover all but two special-purpose contexts;
those two calls can be left as-is, I think.
While this patch doesn't in itself improve matters for third-party
extensions, it doesn't break anything for them either, and they can
gradually adopt the simplified notation over time.
In passing, change TopMemoryContext to use the default allocation
parameters. Formerly it could only be extended 8K at a time. That was
probably reasonable when this code was written; but nowadays we create
many more contexts than we did then, so that it's not unusual to have a
couple hundred K in TopMemoryContext, even without considering various
dubious code that sticks other things there. There seems no good reason
not to let it use growing blocks like most other contexts.
Back-patch to 9.6, mostly because that's still close enough to HEAD that
it's easy to do so, and keeping the branches in sync can be expected to
avoid some future back-patching pain. The bugs fixed by these changes
don't seem to be significant enough to justify fixing them further back.
Discussion: <21072.1472321324@sss.pgh.pa.us>
2016-08-27 23:50:38 +02:00
|
|
|
ALLOCSET_DEFAULT_SIZES);
|
2010-12-29 12:48:53 +01:00
|
|
|
oldctx = MemoryContextSwitchTo(tmpctx);
|
|
|
|
|
Report progress of startup operations that take a long time.
Users sometimes get concerned whe they start the server and it
emits a few messages and then doesn't emit any more messages for
a long time. Generally, what's happening is either that the
system is taking a long time to apply WAL, or it's taking a
long time to reset unlogged relations, or it's taking a long
time to fsync the data directory, but it's not easy to tell
which is the case.
To fix that, add a new 'log_startup_progress_interval' setting,
by default 10s. When an operation that is known to be potentially
long-running takes more than this amount of time, we'll log a
status update each time this interval elapses.
To avoid undesirable log chatter, don't log anything about WAL
replay when in standby mode.
Nitin Jadhav and Robert Haas, reviewed by Amul Sul, Bharath
Rupireddy, Justin Pryzby, Michael Paquier, and Álvaro Herrera.
Discussion: https://postgr.es/m/CA+TgmoaHQrgDFOBwgY16XCoMtXxsrVGFB2jNCvb7-ubuEe1MGg@mail.gmail.com
Discussion: https://postgr.es/m/CAMm1aWaHF7VE69572_OLQ+MgpT5RUiUDgF1x5RrtkJBLdpRj3Q@mail.gmail.com
2021-10-25 17:51:57 +02:00
|
|
|
/* Prepare to report progress resetting unlogged relations. */
|
|
|
|
begin_startup_progress_phase();
|
|
|
|
|
2010-12-29 12:48:53 +01:00
|
|
|
/*
|
|
|
|
* First process unlogged files in pg_default ($PGDATA/base)
|
|
|
|
*/
|
|
|
|
ResetUnloggedRelationsInTablespaceDir("base", op);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Cycle through directories for all non-default tablespaces.
|
|
|
|
*/
|
|
|
|
spc_dir = AllocateDir("pg_tblspc");
|
|
|
|
|
|
|
|
while ((spc_de = ReadDir(spc_dir, "pg_tblspc")) != NULL)
|
|
|
|
{
|
|
|
|
if (strcmp(spc_de->d_name, ".") == 0 ||
|
|
|
|
strcmp(spc_de->d_name, "..") == 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
snprintf(temp_path, sizeof(temp_path), "pg_tblspc/%s/%s",
|
|
|
|
spc_de->d_name, TABLESPACE_VERSION_DIRECTORY);
|
|
|
|
ResetUnloggedRelationsInTablespaceDir(temp_path, op);
|
|
|
|
}
|
|
|
|
|
|
|
|
FreeDir(spc_dir);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Restore memory context.
|
|
|
|
*/
|
|
|
|
MemoryContextSwitchTo(oldctx);
|
|
|
|
MemoryContextDelete(tmpctx);
|
|
|
|
}
|
|
|
|
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
/*
|
|
|
|
* Process one tablespace directory for ResetUnloggedRelations
|
|
|
|
*/
|
2010-12-29 12:48:53 +01:00
|
|
|
static void
|
|
|
|
ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
|
|
|
|
{
|
|
|
|
DIR *ts_dir;
|
|
|
|
struct dirent *de;
|
2017-04-11 20:13:31 +02:00
|
|
|
char dbspace_path[MAXPGPATH * 2];
|
2010-12-29 12:48:53 +01:00
|
|
|
|
|
|
|
ts_dir = AllocateDir(tsdirname);
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we get ENOENT on a tablespace directory, log it and return. This
|
|
|
|
* can happen if a previous DROP TABLESPACE crashed between removing the
|
|
|
|
* tablespace directory and removing the symlink in pg_tblspc. We don't
|
|
|
|
* really want to prevent database startup in that scenario, so let it
|
|
|
|
* pass instead. Any other type of error will be reported by ReadDir
|
|
|
|
* (causing a startup failure).
|
|
|
|
*/
|
|
|
|
if (ts_dir == NULL && errno == ENOENT)
|
2010-12-29 12:48:53 +01:00
|
|
|
{
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
ereport(LOG,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not open directory \"%s\": %m",
|
|
|
|
tsdirname)));
|
2010-12-29 12:48:53 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
while ((de = ReadDir(ts_dir, tsdirname)) != NULL)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We're only interested in the per-database directories, which have
|
|
|
|
* numeric names. Note that this code will also (properly) ignore "."
|
|
|
|
* and "..".
|
|
|
|
*/
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
if (strspn(de->d_name, "0123456789") != strlen(de->d_name))
|
2010-12-29 12:48:53 +01:00
|
|
|
continue;
|
|
|
|
|
|
|
|
snprintf(dbspace_path, sizeof(dbspace_path), "%s/%s",
|
|
|
|
tsdirname, de->d_name);
|
Report progress of startup operations that take a long time.
Users sometimes get concerned whe they start the server and it
emits a few messages and then doesn't emit any more messages for
a long time. Generally, what's happening is either that the
system is taking a long time to apply WAL, or it's taking a
long time to reset unlogged relations, or it's taking a long
time to fsync the data directory, but it's not easy to tell
which is the case.
To fix that, add a new 'log_startup_progress_interval' setting,
by default 10s. When an operation that is known to be potentially
long-running takes more than this amount of time, we'll log a
status update each time this interval elapses.
To avoid undesirable log chatter, don't log anything about WAL
replay when in standby mode.
Nitin Jadhav and Robert Haas, reviewed by Amul Sul, Bharath
Rupireddy, Justin Pryzby, Michael Paquier, and Álvaro Herrera.
Discussion: https://postgr.es/m/CA+TgmoaHQrgDFOBwgY16XCoMtXxsrVGFB2jNCvb7-ubuEe1MGg@mail.gmail.com
Discussion: https://postgr.es/m/CAMm1aWaHF7VE69572_OLQ+MgpT5RUiUDgF1x5RrtkJBLdpRj3Q@mail.gmail.com
2021-10-25 17:51:57 +02:00
|
|
|
|
|
|
|
if (op & UNLOGGED_RELATION_INIT)
|
|
|
|
ereport_startup_progress("resetting unlogged relations (init), elapsed time: %ld.%02d s, current path: %s",
|
|
|
|
dbspace_path);
|
|
|
|
else if (op & UNLOGGED_RELATION_CLEANUP)
|
|
|
|
ereport_startup_progress("resetting unlogged relations (cleanup), elapsed time: %ld.%02d s, current path: %s",
|
|
|
|
dbspace_path);
|
|
|
|
|
2010-12-29 12:48:53 +01:00
|
|
|
ResetUnloggedRelationsInDbspaceDir(dbspace_path, op);
|
|
|
|
}
|
|
|
|
|
|
|
|
FreeDir(ts_dir);
|
|
|
|
}
|
|
|
|
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
/*
|
|
|
|
* Process one per-dbspace directory for ResetUnloggedRelations
|
|
|
|
*/
|
2010-12-29 12:48:53 +01:00
|
|
|
static void
|
|
|
|
ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
|
|
|
|
{
|
|
|
|
DIR *dbspace_dir;
|
|
|
|
struct dirent *de;
|
2017-04-11 20:13:31 +02:00
|
|
|
char rm_path[MAXPGPATH * 2];
|
2010-12-29 12:48:53 +01:00
|
|
|
|
|
|
|
/* Caller must specify at least one operation. */
|
|
|
|
Assert((op & (UNLOGGED_RELATION_CLEANUP | UNLOGGED_RELATION_INIT)) != 0);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Cleanup is a two-pass operation. First, we go through and identify all
|
|
|
|
* the files with init forks. Then, we go through again and nuke
|
|
|
|
* everything with the same OID except the init fork.
|
|
|
|
*/
|
|
|
|
if ((op & UNLOGGED_RELATION_CLEANUP) != 0)
|
|
|
|
{
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
HTAB *hash;
|
2010-12-29 12:48:53 +01:00
|
|
|
HASHCTL ctl;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It's possible that someone could create a ton of unlogged relations
|
|
|
|
* in the same database & tablespace, so we'd better use a hash table
|
|
|
|
* rather than an array or linked list to keep track of which files
|
|
|
|
* need to be reset. Otherwise, this cleanup operation would be
|
|
|
|
* O(n^2).
|
|
|
|
*/
|
2022-09-28 15:45:27 +02:00
|
|
|
ctl.keysize = sizeof(Oid);
|
2010-12-29 12:48:53 +01:00
|
|
|
ctl.entrysize = sizeof(unlogged_relation_entry);
|
Improve hash_create()'s API for some added robustness.
Invent a new flag bit HASH_STRINGS to specify C-string hashing, which
was formerly the default; and add assertions insisting that exactly
one of the bits HASH_STRINGS, HASH_BLOBS, and HASH_FUNCTION be set.
This is in hopes of preventing recurrences of the type of oversight
fixed in commit a1b8aa1e4 (i.e., mistakenly omitting HASH_BLOBS).
Also, when HASH_STRINGS is specified, insist that the keysize be
more than 8 bytes. This is a heuristic, but it should catch
accidental use of HASH_STRINGS for integer or pointer keys.
(Nearly all existing use-cases set the keysize to NAMEDATALEN or
more, so there's little reason to think this restriction should
be problematic.)
Tweak hash_create() to insist that the HASH_ELEM flag be set, and
remove the defaults it had for keysize and entrysize. Since those
defaults were undocumented and basically useless, no callers
omitted HASH_ELEM anyway.
Also, remove memset's zeroing the HASHCTL parameter struct from
those callers that had one. This has never been really necessary,
and while it wasn't a bad coding convention it was confusing that
some callers did it and some did not. We might as well save a few
cycles by standardizing on "not".
Also improve the documentation for hash_create().
In passing, improve reinit.c's usage of a hash table by storing
the key as a binary Oid rather than a string; and, since that's
a temporary hash table, allocate it in CurrentMemoryContext for
neatness.
Discussion: https://postgr.es/m/590625.1607878171@sss.pgh.pa.us
2020-12-15 17:38:53 +01:00
|
|
|
ctl.hcxt = CurrentMemoryContext;
|
2022-09-28 15:45:27 +02:00
|
|
|
hash = hash_create("unlogged relation OIDs", 32, &ctl,
|
Improve hash_create()'s API for some added robustness.
Invent a new flag bit HASH_STRINGS to specify C-string hashing, which
was formerly the default; and add assertions insisting that exactly
one of the bits HASH_STRINGS, HASH_BLOBS, and HASH_FUNCTION be set.
This is in hopes of preventing recurrences of the type of oversight
fixed in commit a1b8aa1e4 (i.e., mistakenly omitting HASH_BLOBS).
Also, when HASH_STRINGS is specified, insist that the keysize be
more than 8 bytes. This is a heuristic, but it should catch
accidental use of HASH_STRINGS for integer or pointer keys.
(Nearly all existing use-cases set the keysize to NAMEDATALEN or
more, so there's little reason to think this restriction should
be problematic.)
Tweak hash_create() to insist that the HASH_ELEM flag be set, and
remove the defaults it had for keysize and entrysize. Since those
defaults were undocumented and basically useless, no callers
omitted HASH_ELEM anyway.
Also, remove memset's zeroing the HASHCTL parameter struct from
those callers that had one. This has never been really necessary,
and while it wasn't a bad coding convention it was confusing that
some callers did it and some did not. We might as well save a few
cycles by standardizing on "not".
Also improve the documentation for hash_create().
In passing, improve reinit.c's usage of a hash table by storing
the key as a binary Oid rather than a string; and, since that's
a temporary hash table, allocate it in CurrentMemoryContext for
neatness.
Discussion: https://postgr.es/m/590625.1607878171@sss.pgh.pa.us
2020-12-15 17:38:53 +01:00
|
|
|
HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
|
2010-12-29 12:48:53 +01:00
|
|
|
|
|
|
|
/* Scan the directory. */
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
dbspace_dir = AllocateDir(dbspacedirname);
|
2010-12-29 12:48:53 +01:00
|
|
|
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
|
|
|
|
{
|
|
|
|
ForkNumber forkNum;
|
2022-07-28 16:08:10 +02:00
|
|
|
int relnumchars;
|
2010-12-29 12:48:53 +01:00
|
|
|
unlogged_relation_entry ent;
|
|
|
|
|
|
|
|
/* Skip anything that doesn't look like a relation data file. */
|
2022-07-28 16:08:10 +02:00
|
|
|
if (!parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
|
2010-12-29 12:48:53 +01:00
|
|
|
&forkNum))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Also skip it unless this is the init fork. */
|
|
|
|
if (forkNum != INIT_FORKNUM)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
2022-09-28 15:45:27 +02:00
|
|
|
* Put the OID portion of the name into the hash table, if it
|
|
|
|
* isn't already.
|
2010-12-29 12:48:53 +01:00
|
|
|
*/
|
2022-09-28 15:45:27 +02:00
|
|
|
ent.reloid = atooid(de->d_name);
|
Improve hash_create()'s API for some added robustness.
Invent a new flag bit HASH_STRINGS to specify C-string hashing, which
was formerly the default; and add assertions insisting that exactly
one of the bits HASH_STRINGS, HASH_BLOBS, and HASH_FUNCTION be set.
This is in hopes of preventing recurrences of the type of oversight
fixed in commit a1b8aa1e4 (i.e., mistakenly omitting HASH_BLOBS).
Also, when HASH_STRINGS is specified, insist that the keysize be
more than 8 bytes. This is a heuristic, but it should catch
accidental use of HASH_STRINGS for integer or pointer keys.
(Nearly all existing use-cases set the keysize to NAMEDATALEN or
more, so there's little reason to think this restriction should
be problematic.)
Tweak hash_create() to insist that the HASH_ELEM flag be set, and
remove the defaults it had for keysize and entrysize. Since those
defaults were undocumented and basically useless, no callers
omitted HASH_ELEM anyway.
Also, remove memset's zeroing the HASHCTL parameter struct from
those callers that had one. This has never been really necessary,
and while it wasn't a bad coding convention it was confusing that
some callers did it and some did not. We might as well save a few
cycles by standardizing on "not".
Also improve the documentation for hash_create().
In passing, improve reinit.c's usage of a hash table by storing
the key as a binary Oid rather than a string; and, since that's
a temporary hash table, allocate it in CurrentMemoryContext for
neatness.
Discussion: https://postgr.es/m/590625.1607878171@sss.pgh.pa.us
2020-12-15 17:38:53 +01:00
|
|
|
(void) hash_search(hash, &ent, HASH_ENTER, NULL);
|
2010-12-29 12:48:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Done with the first pass. */
|
|
|
|
FreeDir(dbspace_dir);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we didn't find any init forks, there's no point in continuing;
|
|
|
|
* we can bail out now.
|
|
|
|
*/
|
|
|
|
if (hash_get_num_entries(hash) == 0)
|
|
|
|
{
|
|
|
|
hash_destroy(hash);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
* Now, make a second pass and remove anything that matches.
|
2010-12-29 12:48:53 +01:00
|
|
|
*/
|
|
|
|
dbspace_dir = AllocateDir(dbspacedirname);
|
|
|
|
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
|
|
|
|
{
|
|
|
|
ForkNumber forkNum;
|
2022-07-28 16:08:10 +02:00
|
|
|
int relnumchars;
|
2010-12-29 12:48:53 +01:00
|
|
|
unlogged_relation_entry ent;
|
|
|
|
|
|
|
|
/* Skip anything that doesn't look like a relation data file. */
|
2022-07-28 16:08:10 +02:00
|
|
|
if (!parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
|
2010-12-29 12:48:53 +01:00
|
|
|
&forkNum))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* We never remove the init fork. */
|
|
|
|
if (forkNum == INIT_FORKNUM)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
2022-09-28 15:45:27 +02:00
|
|
|
* See whether the OID portion of the name shows up in the hash
|
|
|
|
* table. If so, nuke it!
|
2010-12-29 12:48:53 +01:00
|
|
|
*/
|
2022-09-28 15:45:27 +02:00
|
|
|
ent.reloid = atooid(de->d_name);
|
Improve hash_create()'s API for some added robustness.
Invent a new flag bit HASH_STRINGS to specify C-string hashing, which
was formerly the default; and add assertions insisting that exactly
one of the bits HASH_STRINGS, HASH_BLOBS, and HASH_FUNCTION be set.
This is in hopes of preventing recurrences of the type of oversight
fixed in commit a1b8aa1e4 (i.e., mistakenly omitting HASH_BLOBS).
Also, when HASH_STRINGS is specified, insist that the keysize be
more than 8 bytes. This is a heuristic, but it should catch
accidental use of HASH_STRINGS for integer or pointer keys.
(Nearly all existing use-cases set the keysize to NAMEDATALEN or
more, so there's little reason to think this restriction should
be problematic.)
Tweak hash_create() to insist that the HASH_ELEM flag be set, and
remove the defaults it had for keysize and entrysize. Since those
defaults were undocumented and basically useless, no callers
omitted HASH_ELEM anyway.
Also, remove memset's zeroing the HASHCTL parameter struct from
those callers that had one. This has never been really necessary,
and while it wasn't a bad coding convention it was confusing that
some callers did it and some did not. We might as well save a few
cycles by standardizing on "not".
Also improve the documentation for hash_create().
In passing, improve reinit.c's usage of a hash table by storing
the key as a binary Oid rather than a string; and, since that's
a temporary hash table, allocate it in CurrentMemoryContext for
neatness.
Discussion: https://postgr.es/m/590625.1607878171@sss.pgh.pa.us
2020-12-15 17:38:53 +01:00
|
|
|
if (hash_search(hash, &ent, HASH_FIND, NULL))
|
2010-12-29 12:48:53 +01:00
|
|
|
{
|
|
|
|
snprintf(rm_path, sizeof(rm_path), "%s/%s",
|
|
|
|
dbspacedirname, de->d_name);
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
if (unlink(rm_path) < 0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not remove file \"%s\": %m",
|
|
|
|
rm_path)));
|
2010-12-29 12:48:53 +01:00
|
|
|
else
|
|
|
|
elog(DEBUG2, "unlinked file \"%s\"", rm_path);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Cleanup is complete. */
|
|
|
|
FreeDir(dbspace_dir);
|
|
|
|
hash_destroy(hash);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialization happens after cleanup is complete: we copy each init
|
|
|
|
* fork file to the corresponding main fork file. Note that if we are
|
|
|
|
* asked to do both cleanup and init, we may never get here: if the
|
|
|
|
* cleanup code determines that there are no init forks in this dbspace,
|
|
|
|
* it will return before we get to this point.
|
|
|
|
*/
|
|
|
|
if ((op & UNLOGGED_RELATION_INIT) != 0)
|
|
|
|
{
|
|
|
|
/* Scan the directory. */
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
dbspace_dir = AllocateDir(dbspacedirname);
|
2010-12-29 12:48:53 +01:00
|
|
|
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
|
|
|
|
{
|
|
|
|
ForkNumber forkNum;
|
2022-07-28 16:08:10 +02:00
|
|
|
int relnumchars;
|
2022-09-28 15:45:27 +02:00
|
|
|
char relnumbuf[OIDCHARS + 1];
|
2017-04-11 20:13:31 +02:00
|
|
|
char srcpath[MAXPGPATH * 2];
|
2010-12-29 12:48:53 +01:00
|
|
|
char dstpath[MAXPGPATH];
|
|
|
|
|
|
|
|
/* Skip anything that doesn't look like a relation data file. */
|
2022-07-28 16:08:10 +02:00
|
|
|
if (!parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
|
2010-12-29 12:48:53 +01:00
|
|
|
&forkNum))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Also skip it unless this is the init fork. */
|
|
|
|
if (forkNum != INIT_FORKNUM)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Construct source pathname. */
|
|
|
|
snprintf(srcpath, sizeof(srcpath), "%s/%s",
|
|
|
|
dbspacedirname, de->d_name);
|
|
|
|
|
|
|
|
/* Construct destination pathname. */
|
2022-07-28 16:08:10 +02:00
|
|
|
memcpy(relnumbuf, de->d_name, relnumchars);
|
|
|
|
relnumbuf[relnumchars] = '\0';
|
2010-12-29 12:48:53 +01:00
|
|
|
snprintf(dstpath, sizeof(dstpath), "%s/%s%s",
|
2022-07-28 16:08:10 +02:00
|
|
|
dbspacedirname, relnumbuf, de->d_name + relnumchars + 1 +
|
2010-12-29 12:48:53 +01:00
|
|
|
strlen(forkNames[INIT_FORKNUM]));
|
|
|
|
|
|
|
|
/* OK, we're ready to perform the actual copy. */
|
|
|
|
elog(DEBUG2, "copying %s to %s", srcpath, dstpath);
|
|
|
|
copy_file(srcpath, dstpath);
|
|
|
|
}
|
|
|
|
|
|
|
|
FreeDir(dbspace_dir);
|
2014-11-14 18:21:30 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* copy_file() above has already called pg_flush_data() on the files
|
|
|
|
* it created. Now we need to fsync those files, because a checkpoint
|
|
|
|
* won't do it for us while we're in recovery. We do this in a
|
|
|
|
* separate pass to allow the kernel to perform all the flushes
|
|
|
|
* (especially the metadata ones) at once.
|
|
|
|
*/
|
|
|
|
dbspace_dir = AllocateDir(dbspacedirname);
|
|
|
|
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
|
|
|
|
{
|
|
|
|
ForkNumber forkNum;
|
2022-07-28 16:08:10 +02:00
|
|
|
int relnumchars;
|
2022-09-28 15:45:27 +02:00
|
|
|
char relnumbuf[OIDCHARS + 1];
|
2014-11-14 18:21:30 +01:00
|
|
|
char mainpath[MAXPGPATH];
|
|
|
|
|
|
|
|
/* Skip anything that doesn't look like a relation data file. */
|
2022-07-28 16:08:10 +02:00
|
|
|
if (!parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
|
2014-11-14 18:21:30 +01:00
|
|
|
&forkNum))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Also skip it unless this is the init fork. */
|
|
|
|
if (forkNum != INIT_FORKNUM)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Construct main fork pathname. */
|
2022-07-28 16:08:10 +02:00
|
|
|
memcpy(relnumbuf, de->d_name, relnumchars);
|
|
|
|
relnumbuf[relnumchars] = '\0';
|
2014-11-14 18:21:30 +01:00
|
|
|
snprintf(mainpath, sizeof(mainpath), "%s/%s%s",
|
2022-07-28 16:08:10 +02:00
|
|
|
dbspacedirname, relnumbuf, de->d_name + relnumchars + 1 +
|
2014-11-14 18:21:30 +01:00
|
|
|
strlen(forkNames[INIT_FORKNUM]));
|
|
|
|
|
|
|
|
fsync_fname(mainpath, false);
|
|
|
|
}
|
|
|
|
|
|
|
|
FreeDir(dbspace_dir);
|
|
|
|
|
Treat directory open failures as hard errors in ResetUnloggedRelations().
Previously, this code just reported such problems at LOG level and kept
going. The problem with this approach is that transient failures (e.g.,
ENFILE) could prevent us from resetting unlogged relations to empty,
yet allow recovery to appear to complete successfully. That seems like
a data corruption hazard large enough to treat such problems as reasons
to fail startup.
For the same reason, treat unlink failures for unlogged files as hard
errors not just LOG messages. It's a little odd that we did it like that
when file-level errors in other steps (copy_file, fsync_fname) are ERRORs.
The sole case that I left alone is that ENOENT failure on a tablespace
(not database) directory is not an error, though it will now be logged
rather than just silently ignored. This is to cover the scenario where
a previous DROP TABLESPACE removed the tablespace directory but failed
before removing the pg_tblspc symlink. I'm not sure that that's very
likely in practice, but that seems like the only real excuse for the
old behavior here, so let's allow for it. (As coded, this will also
allow ENOENT on $PGDATA/base/. But since we'll fail soon enough if
that's gone, I don't think we need to complicate this code by
distinguishing that from a true tablespace case.)
Discussion: https://postgr.es/m/21040.1512418508@sss.pgh.pa.us
2017-12-05 02:52:48 +01:00
|
|
|
/*
|
|
|
|
* Lastly, fsync the database directory itself, ensuring the
|
|
|
|
* filesystem remembers the file creations and deletions we've done.
|
|
|
|
* We don't bother with this during a call that does only
|
|
|
|
* UNLOGGED_RELATION_CLEANUP, because if recovery crashes before we
|
|
|
|
* get to doing UNLOGGED_RELATION_INIT, we'll redo the cleanup step
|
|
|
|
* too at the next startup attempt.
|
|
|
|
*/
|
2016-03-10 03:53:53 +01:00
|
|
|
fsync_fname(dbspacedirname, true);
|
2010-12-29 12:48:53 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Basic parsing of putative relation filenames.
|
|
|
|
*
|
2012-04-24 04:43:09 +02:00
|
|
|
* This function returns true if the file appears to be in the correct format
|
2010-12-29 12:48:53 +01:00
|
|
|
* for a non-temporary relation and false otherwise.
|
|
|
|
*
|
|
|
|
* NB: If this function returns true, the caller is entitled to assume that
|
2022-09-28 15:45:27 +02:00
|
|
|
* *relnumchars has been set to a value no more than OIDCHARS, and thus
|
|
|
|
* that a buffer of OIDCHARS+1 characters is sufficient to hold the
|
2022-07-28 16:08:10 +02:00
|
|
|
* RelFileNumber portion of the filename. This is critical to protect against
|
|
|
|
* a possible buffer overrun.
|
2010-12-29 12:48:53 +01:00
|
|
|
*/
|
2018-03-23 17:14:12 +01:00
|
|
|
bool
|
2022-07-28 16:08:10 +02:00
|
|
|
parse_filename_for_nontemp_relation(const char *name, int *relnumchars,
|
2010-12-29 12:48:53 +01:00
|
|
|
ForkNumber *fork)
|
|
|
|
{
|
|
|
|
int pos;
|
|
|
|
|
|
|
|
/* Look for a non-empty string of digits (that isn't too long). */
|
|
|
|
for (pos = 0; isdigit((unsigned char) name[pos]); ++pos)
|
|
|
|
;
|
2022-09-28 15:45:27 +02:00
|
|
|
if (pos == 0 || pos > OIDCHARS)
|
2010-12-29 12:48:53 +01:00
|
|
|
return false;
|
2022-07-28 16:08:10 +02:00
|
|
|
*relnumchars = pos;
|
2010-12-29 12:48:53 +01:00
|
|
|
|
|
|
|
/* Check for a fork name. */
|
|
|
|
if (name[pos] != '_')
|
|
|
|
*fork = MAIN_FORKNUM;
|
|
|
|
else
|
|
|
|
{
|
|
|
|
int forkchar;
|
|
|
|
|
|
|
|
forkchar = forkname_chars(&name[pos + 1], fork);
|
|
|
|
if (forkchar <= 0)
|
|
|
|
return false;
|
|
|
|
pos += forkchar + 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Check for a segment number. */
|
|
|
|
if (name[pos] == '.')
|
|
|
|
{
|
|
|
|
int segchar;
|
2011-04-10 17:42:00 +02:00
|
|
|
|
2010-12-29 12:48:53 +01:00
|
|
|
for (segchar = 1; isdigit((unsigned char) name[pos + segchar]); ++segchar)
|
|
|
|
;
|
|
|
|
if (segchar <= 1)
|
|
|
|
return false;
|
|
|
|
pos += segchar;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Now we should be at the end. */
|
|
|
|
if (name[pos] != '\0')
|
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|