2016-09-29 18:00:00 +02:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* File-processing utility routines.
|
|
|
|
*
|
|
|
|
* Assorted utility functions to work on files.
|
|
|
|
*
|
|
|
|
*
|
2018-01-03 05:30:12 +01:00
|
|
|
* Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
|
2016-09-29 18:00:00 +02:00
|
|
|
* Portions Copyright (c) 1994, Regents of the University of California
|
|
|
|
*
|
|
|
|
* src/common/file_utils.c
|
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
#include "postgres_fe.h"
|
|
|
|
|
|
|
|
#include <dirent.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <unistd.h>
|
|
|
|
|
|
|
|
#include "common/file_utils.h"
|
|
|
|
|
|
|
|
|
|
|
|
/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */
|
|
|
|
#if defined(HAVE_SYNC_FILE_RANGE)
|
|
|
|
#define PG_FLUSH_DATA_WORKS 1
|
|
|
|
#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
|
|
|
|
#define PG_FLUSH_DATA_WORKS 1
|
|
|
|
#endif
|
|
|
|
|
2016-10-20 17:24:37 +02:00
|
|
|
/*
|
|
|
|
* pg_xlog has been renamed to pg_wal in version 10.
|
|
|
|
*/
|
2017-05-17 22:31:56 +02:00
|
|
|
#define MINIMUM_VERSION_FOR_PG_WAL 100000
|
2016-10-20 17:24:37 +02:00
|
|
|
|
2016-09-29 18:00:00 +02:00
|
|
|
#ifdef PG_FLUSH_DATA_WORKS
|
2016-09-29 18:00:00 +02:00
|
|
|
static int pre_sync_fname(const char *fname, bool isdir,
|
2017-05-17 22:31:56 +02:00
|
|
|
const char *progname);
|
2016-09-29 18:00:00 +02:00
|
|
|
#endif
|
|
|
|
static void walkdir(const char *path,
|
2017-05-17 22:31:56 +02:00
|
|
|
int (*action) (const char *fname, bool isdir, const char *progname),
|
|
|
|
bool process_symlinks, const char *progname);
|
2016-09-29 18:00:00 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Issue fsync recursively on PGDATA and all its contents.
|
|
|
|
*
|
2016-10-20 17:24:37 +02:00
|
|
|
* We fsync regular files and directories wherever they are, but we follow
|
|
|
|
* symlinks only for pg_wal (or pg_xlog) and immediately under pg_tblspc.
|
|
|
|
* Other symlinks are presumed to point at files we're not responsible for
|
|
|
|
* fsyncing, and might not have privileges to write at all.
|
|
|
|
*
|
|
|
|
* serverVersion indicates the version of the server to be fsync'd.
|
2016-09-29 18:00:00 +02:00
|
|
|
*
|
|
|
|
* Errors are reported but not considered fatal.
|
|
|
|
*/
|
|
|
|
void
|
2016-10-20 17:24:37 +02:00
|
|
|
fsync_pgdata(const char *pg_data,
|
|
|
|
const char *progname,
|
|
|
|
int serverVersion)
|
2016-09-29 18:00:00 +02:00
|
|
|
{
|
|
|
|
bool xlog_is_symlink;
|
2016-10-20 17:24:37 +02:00
|
|
|
char pg_wal[MAXPGPATH];
|
2016-09-29 18:00:00 +02:00
|
|
|
char pg_tblspc[MAXPGPATH];
|
|
|
|
|
2016-10-20 17:24:37 +02:00
|
|
|
/* handle renaming of pg_xlog to pg_wal in post-10 clusters */
|
|
|
|
snprintf(pg_wal, MAXPGPATH, "%s/%s", pg_data,
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
serverVersion < MINIMUM_VERSION_FOR_PG_WAL ? "pg_xlog" : "pg_wal");
|
2016-09-29 18:00:00 +02:00
|
|
|
snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data);
|
|
|
|
|
|
|
|
/*
|
2016-10-20 17:24:37 +02:00
|
|
|
* If pg_wal is a symlink, we'll need to recurse into it separately,
|
2016-09-29 18:00:00 +02:00
|
|
|
* because the first walkdir below will ignore it.
|
|
|
|
*/
|
|
|
|
xlog_is_symlink = false;
|
|
|
|
|
|
|
|
#ifndef WIN32
|
|
|
|
{
|
|
|
|
struct stat st;
|
|
|
|
|
2016-10-20 17:24:37 +02:00
|
|
|
if (lstat(pg_wal, &st) < 0)
|
2016-09-29 18:00:00 +02:00
|
|
|
fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"),
|
2016-10-20 17:24:37 +02:00
|
|
|
progname, pg_wal, strerror(errno));
|
2016-09-29 18:00:00 +02:00
|
|
|
else if (S_ISLNK(st.st_mode))
|
|
|
|
xlog_is_symlink = true;
|
|
|
|
}
|
|
|
|
#else
|
2016-10-20 17:24:37 +02:00
|
|
|
if (pgwin32_is_junction(pg_wal))
|
2016-09-29 18:00:00 +02:00
|
|
|
xlog_is_symlink = true;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If possible, hint to the kernel that we're soon going to fsync the data
|
|
|
|
* directory and its contents.
|
|
|
|
*/
|
|
|
|
#ifdef PG_FLUSH_DATA_WORKS
|
|
|
|
walkdir(pg_data, pre_sync_fname, false, progname);
|
|
|
|
if (xlog_is_symlink)
|
2016-10-20 17:24:37 +02:00
|
|
|
walkdir(pg_wal, pre_sync_fname, false, progname);
|
2016-09-29 18:00:00 +02:00
|
|
|
walkdir(pg_tblspc, pre_sync_fname, true, progname);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Now we do the fsync()s in the same order.
|
|
|
|
*
|
|
|
|
* The main call ignores symlinks, so in addition to specially processing
|
2016-10-20 17:24:37 +02:00
|
|
|
* pg_wal if it's a symlink, pg_tblspc has to be visited separately with
|
2016-09-29 18:00:00 +02:00
|
|
|
* process_symlinks = true. Note that if there are any plain directories
|
|
|
|
* in pg_tblspc, they'll get fsync'd twice. That's not an expected case
|
|
|
|
* so we don't worry about optimizing it.
|
|
|
|
*/
|
|
|
|
walkdir(pg_data, fsync_fname, false, progname);
|
|
|
|
if (xlog_is_symlink)
|
2016-10-20 17:24:37 +02:00
|
|
|
walkdir(pg_wal, fsync_fname, false, progname);
|
2016-09-29 18:00:00 +02:00
|
|
|
walkdir(pg_tblspc, fsync_fname, true, progname);
|
|
|
|
}
|
|
|
|
|
2017-03-22 15:00:30 +01:00
|
|
|
/*
|
|
|
|
* Issue fsync recursively on the given directory and all its contents.
|
|
|
|
*
|
|
|
|
* This is a convenient wrapper on top of walkdir().
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
fsync_dir_recurse(const char *dir, const char *progname)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If possible, hint to the kernel that we're soon going to fsync the data
|
|
|
|
* directory and its contents.
|
|
|
|
*/
|
|
|
|
#ifdef PG_FLUSH_DATA_WORKS
|
|
|
|
walkdir(dir, pre_sync_fname, false, progname);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
walkdir(dir, fsync_fname, false, progname);
|
|
|
|
}
|
|
|
|
|
2016-09-29 18:00:00 +02:00
|
|
|
/*
|
|
|
|
* walkdir: recursively walk a directory, applying the action to each
|
|
|
|
* regular file and directory (including the named directory itself).
|
|
|
|
*
|
|
|
|
* If process_symlinks is true, the action and recursion are also applied
|
|
|
|
* to regular files and directories that are pointed to by symlinks in the
|
|
|
|
* given directory; otherwise symlinks are ignored. Symlinks are always
|
|
|
|
* ignored in subdirectories, ie we intentionally don't pass down the
|
|
|
|
* process_symlinks flag to recursive calls.
|
|
|
|
*
|
|
|
|
* Errors are reported but not considered fatal.
|
|
|
|
*
|
|
|
|
* See also walkdir in fd.c, which is a backend version of this logic.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
walkdir(const char *path,
|
2016-09-29 18:00:00 +02:00
|
|
|
int (*action) (const char *fname, bool isdir, const char *progname),
|
2016-09-29 18:00:00 +02:00
|
|
|
bool process_symlinks, const char *progname)
|
|
|
|
{
|
|
|
|
DIR *dir;
|
|
|
|
struct dirent *de;
|
|
|
|
|
|
|
|
dir = opendir(path);
|
|
|
|
if (dir == NULL)
|
|
|
|
{
|
|
|
|
fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"),
|
|
|
|
progname, path, strerror(errno));
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
while (errno = 0, (de = readdir(dir)) != NULL)
|
|
|
|
{
|
2017-04-11 20:13:31 +02:00
|
|
|
char subpath[MAXPGPATH * 2];
|
2016-09-29 18:00:00 +02:00
|
|
|
struct stat fst;
|
|
|
|
int sret;
|
|
|
|
|
|
|
|
if (strcmp(de->d_name, ".") == 0 ||
|
|
|
|
strcmp(de->d_name, "..") == 0)
|
|
|
|
continue;
|
|
|
|
|
2017-04-11 20:13:31 +02:00
|
|
|
snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name);
|
2016-09-29 18:00:00 +02:00
|
|
|
|
|
|
|
if (process_symlinks)
|
|
|
|
sret = stat(subpath, &fst);
|
|
|
|
else
|
|
|
|
sret = lstat(subpath, &fst);
|
|
|
|
|
|
|
|
if (sret < 0)
|
|
|
|
{
|
|
|
|
fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"),
|
|
|
|
progname, subpath, strerror(errno));
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (S_ISREG(fst.st_mode))
|
|
|
|
(*action) (subpath, false, progname);
|
|
|
|
else if (S_ISDIR(fst.st_mode))
|
|
|
|
walkdir(subpath, action, false, progname);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (errno)
|
|
|
|
fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"),
|
|
|
|
progname, path, strerror(errno));
|
|
|
|
|
|
|
|
(void) closedir(dir);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It's important to fsync the destination directory itself as individual
|
|
|
|
* file fsyncs don't guarantee that the directory entry for the file is
|
|
|
|
* synced. Recent versions of ext4 have made the window much wider but
|
|
|
|
* it's been an issue for ext3 and other filesystems in the past.
|
|
|
|
*/
|
|
|
|
(*action) (path, true, progname);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Hint to the OS that it should get ready to fsync() this file.
|
|
|
|
*
|
|
|
|
* Ignores errors trying to open unreadable files, and reports other errors
|
|
|
|
* non-fatally.
|
|
|
|
*/
|
|
|
|
#ifdef PG_FLUSH_DATA_WORKS
|
|
|
|
|
2016-09-29 18:00:00 +02:00
|
|
|
static int
|
2016-09-29 18:00:00 +02:00
|
|
|
pre_sync_fname(const char *fname, bool isdir, const char *progname)
|
|
|
|
{
|
|
|
|
int fd;
|
|
|
|
|
2018-09-14 03:04:14 +02:00
|
|
|
fd = open(fname, O_RDONLY | PG_BINARY, 0);
|
2016-09-29 18:00:00 +02:00
|
|
|
|
|
|
|
if (fd < 0)
|
|
|
|
{
|
|
|
|
if (errno == EACCES || (isdir && errno == EISDIR))
|
2016-09-29 18:00:00 +02:00
|
|
|
return 0;
|
2016-09-29 18:00:00 +02:00
|
|
|
fprintf(stderr, _("%s: could not open file \"%s\": %s\n"),
|
|
|
|
progname, fname, strerror(errno));
|
2016-09-29 18:00:00 +02:00
|
|
|
return -1;
|
2016-09-29 18:00:00 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We do what pg_flush_data() would do in the backend: prefer to use
|
|
|
|
* sync_file_range, but fall back to posix_fadvise. We ignore errors
|
|
|
|
* because this is only a hint.
|
|
|
|
*/
|
|
|
|
#if defined(HAVE_SYNC_FILE_RANGE)
|
|
|
|
(void) sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE);
|
|
|
|
#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
|
|
|
|
(void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
|
|
|
|
#else
|
|
|
|
#error PG_FLUSH_DATA_WORKS should not have been defined
|
|
|
|
#endif
|
|
|
|
|
|
|
|
(void) close(fd);
|
2016-09-29 18:00:00 +02:00
|
|
|
return 0;
|
2016-09-29 18:00:00 +02:00
|
|
|
}
|
|
|
|
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
#endif /* PG_FLUSH_DATA_WORKS */
|
2016-09-29 18:00:00 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* fsync_fname -- Try to fsync a file or directory
|
|
|
|
*
|
|
|
|
* Ignores errors trying to open unreadable files, or trying to fsync
|
|
|
|
* directories on systems where that isn't allowed/required. Reports
|
|
|
|
* other errors non-fatally.
|
|
|
|
*/
|
2016-09-29 18:00:00 +02:00
|
|
|
int
|
2016-09-29 18:00:00 +02:00
|
|
|
fsync_fname(const char *fname, bool isdir, const char *progname)
|
|
|
|
{
|
|
|
|
int fd;
|
|
|
|
int flags;
|
|
|
|
int returncode;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Some OSs require directories to be opened read-only whereas other
|
|
|
|
* systems don't allow us to fsync files opened read-only; so we need both
|
|
|
|
* cases here. Using O_RDWR will cause us to fail to fsync files that are
|
|
|
|
* not writable by our userid, but we assume that's OK.
|
|
|
|
*/
|
|
|
|
flags = PG_BINARY;
|
|
|
|
if (!isdir)
|
|
|
|
flags |= O_RDWR;
|
|
|
|
else
|
|
|
|
flags |= O_RDONLY;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Open the file, silently ignoring errors about unreadable files (or
|
|
|
|
* unsupported operations, e.g. opening a directory under Windows), and
|
|
|
|
* logging others.
|
|
|
|
*/
|
2018-09-14 03:04:14 +02:00
|
|
|
fd = open(fname, flags, 0);
|
2016-09-29 18:00:00 +02:00
|
|
|
if (fd < 0)
|
|
|
|
{
|
|
|
|
if (errno == EACCES || (isdir && errno == EISDIR))
|
2016-09-29 18:00:00 +02:00
|
|
|
return 0;
|
2016-09-29 18:00:00 +02:00
|
|
|
fprintf(stderr, _("%s: could not open file \"%s\": %s\n"),
|
|
|
|
progname, fname, strerror(errno));
|
2016-09-29 18:00:00 +02:00
|
|
|
return -1;
|
2016-09-29 18:00:00 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
returncode = fsync(fd);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Some OSes don't allow us to fsync directories at all, so we can ignore
|
|
|
|
* those errors. Anything else needs to be reported.
|
|
|
|
*/
|
|
|
|
if (returncode != 0 && !(isdir && errno == EBADF))
|
2016-09-29 18:00:00 +02:00
|
|
|
{
|
2016-09-29 18:00:00 +02:00
|
|
|
fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
|
|
|
|
progname, fname, strerror(errno));
|
2016-10-02 18:33:46 +02:00
|
|
|
(void) close(fd);
|
2016-09-29 18:00:00 +02:00
|
|
|
return -1;
|
|
|
|
}
|
2016-09-29 18:00:00 +02:00
|
|
|
|
|
|
|
(void) close(fd);
|
2016-09-29 18:00:00 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fsync_parent_path -- fsync the parent path of a file or directory
|
|
|
|
*
|
|
|
|
* This is aimed at making file operations persistent on disk in case of
|
|
|
|
* an OS crash or power failure.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
fsync_parent_path(const char *fname, const char *progname)
|
|
|
|
{
|
|
|
|
char parentpath[MAXPGPATH];
|
|
|
|
|
|
|
|
strlcpy(parentpath, fname, MAXPGPATH);
|
|
|
|
get_parent_directory(parentpath);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* get_parent_directory() returns an empty string if the input argument is
|
|
|
|
* just a file name (see comments in path.c), so handle that as being the
|
|
|
|
* current directory.
|
|
|
|
*/
|
|
|
|
if (strlen(parentpath) == 0)
|
|
|
|
strlcpy(parentpath, ".", MAXPGPATH);
|
|
|
|
|
|
|
|
if (fsync_fname(parentpath, true, progname) != 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* durable_rename -- rename(2) wrapper, issuing fsyncs required for durability
|
|
|
|
*
|
|
|
|
* Wrapper around rename, similar to the backend version.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
durable_rename(const char *oldfile, const char *newfile, const char *progname)
|
|
|
|
{
|
2017-05-17 22:31:56 +02:00
|
|
|
int fd;
|
2016-09-29 18:00:00 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* First fsync the old and target path (if it exists), to ensure that they
|
|
|
|
* are properly persistent on disk. Syncing the target file is not
|
|
|
|
* strictly necessary, but it makes it easier to reason about crashes;
|
|
|
|
* because it's then guaranteed that either source or target file exists
|
|
|
|
* after a crash.
|
|
|
|
*/
|
|
|
|
if (fsync_fname(oldfile, false, progname) != 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
fd = open(newfile, PG_BINARY | O_RDWR, 0);
|
|
|
|
if (fd < 0)
|
|
|
|
{
|
|
|
|
if (errno != ENOENT)
|
|
|
|
{
|
|
|
|
fprintf(stderr, _("%s: could not open file \"%s\": %s\n"),
|
|
|
|
progname, newfile, strerror(errno));
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (fsync(fd) != 0)
|
|
|
|
{
|
|
|
|
fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
|
|
|
|
progname, newfile, strerror(errno));
|
|
|
|
close(fd);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Time to do the real deal... */
|
|
|
|
if (rename(oldfile, newfile) != 0)
|
|
|
|
{
|
|
|
|
fprintf(stderr, _("%s: could not rename file \"%s\" to \"%s\": %s\n"),
|
|
|
|
progname, oldfile, newfile, strerror(errno));
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* To guarantee renaming the file is persistent, fsync the file with its
|
|
|
|
* new name, and its containing directory.
|
|
|
|
*/
|
|
|
|
if (fsync_fname(newfile, false, progname) != 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
if (fsync_parent_path(newfile, progname) != 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
return 0;
|
2016-09-29 18:00:00 +02:00
|
|
|
}
|