/*------------------------------------------------------------------------- * * fd.c * Virtual file descriptor code. * * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * IDENTIFICATION * $PostgreSQL: pgsql/src/backend/storage/file/fd.c,v 1.106 2004/01/26 22:35:32 tgl Exp $ * * NOTES: * * This code manages a cache of 'virtual' file descriptors (VFDs). * The server opens many file descriptors for a variety of reasons, * including base tables, scratch files (e.g., sort and hash spool * files), and random calls to C library routines like system(3); it * is quite easy to exceed system limits on the number of open files a * single process can have. (This is around 256 on many modern * operating systems, but can be as low as 32 on others.) * * VFDs are managed as an LRU pool, with actual OS file descriptors * being opened and closed as needed. Obviously, if a routine is * opened using these interfaces, all subsequent operations must also * be through these interfaces (the File type is not a real file * descriptor). * * For this scheme to work, most (if not all) routines throughout the * server should use these interfaces instead of calling the C library * routines (e.g., open(2) and fopen(3)) themselves. Otherwise, we * may find ourselves short of real file descriptors anyway. * * This file used to contain a bunch of stuff to support RAID levels 0 * (jbod), 1 (duplex) and 5 (xor parity). That stuff is all gone * because the parallel query processing code that called it is all * gone. If you really need it you could get it from the original * POSTGRES source. *------------------------------------------------------------------------- */ #include "postgres.h" #include #include #include #include #include #include #include #include "miscadmin.h" #include "storage/fd.h" #include "storage/ipc.h" /* * Problem: Postgres does a system(ld...) to do dynamic loading. * This will open several extra files in addition to those used by * Postgres. We need to guarantee that there are file descriptors free * for ld to use. * * The current solution is to limit the number of file descriptors * that this code will allocate at one time: it leaves RESERVE_FOR_LD free. * * (Even though most dynamic loaders now use dlopen(3) or the * equivalent, the OS must still open several files to perform the * dynamic loading. And stdin/stdout/stderr count too. Keep this here.) */ #ifndef RESERVE_FOR_LD #define RESERVE_FOR_LD 10 #endif /* * We need to ensure that we have at least some file descriptors * available to postgreSQL after we've reserved the ones for LD, * so we set that value here. * * I think 10 is an appropriate value so that's what it'll be * for now. */ #ifndef FD_MINFREE #define FD_MINFREE 10 #endif /* * A number of platforms return values for sysconf(_SC_OPEN_MAX) that are * far beyond what they can really support. This GUC parameter limits what * we will believe. */ int max_files_per_process = 1000; /* Debugging.... */ #ifdef FDDEBUG #define DO_DB(A) A #else #define DO_DB(A) /* A */ #endif #define VFD_CLOSED (-1) #define FileIsValid(file) \ ((file) > 0 && (file) < (int) SizeVfdCache && VfdCache[file].fileName != NULL) #define FileIsNotOpen(file) (VfdCache[file].fd == VFD_CLOSED) #define FileUnknownPos (-1L) /* these are the assigned bits in fdstate below: */ #define FD_TEMPORARY (1 << 0) /* T = delete when closed */ #define FD_XACT_TEMPORARY (1 << 1) /* T = delete at eoXact */ typedef struct vfd { signed short fd; /* current FD, or VFD_CLOSED if none */ unsigned short fdstate; /* bitflags for VFD's state */ File nextFree; /* link to next free VFD, if in freelist */ File lruMoreRecently; /* doubly linked recency-of-use list */ File lruLessRecently; long seekPos; /* current logical file position */ char *fileName; /* name of file, or NULL for unused VFD */ /* NB: fileName is malloc'd, and must be free'd when closing the VFD */ int fileFlags; /* open(2) flags for (re)opening the file */ int fileMode; /* mode to pass to open(2) */ } Vfd; /* * Virtual File Descriptor array pointer and size. This grows as * needed. 'File' values are indexes into this array. * Note that VfdCache[0] is not a usable VFD, just a list header. */ static Vfd *VfdCache; static Size SizeVfdCache = 0; /* * Number of file descriptors known to be in use by VFD entries. */ static int nfile = 0; /* * List of stdio FILEs opened with AllocateFile. * * Since we don't want to encourage heavy use of AllocateFile, it seems * OK to put a pretty small maximum limit on the number of simultaneously * allocated files. */ #define MAX_ALLOCATED_FILES 32 static int numAllocatedFiles = 0; static FILE *allocatedFiles[MAX_ALLOCATED_FILES]; /* * Number of temporary files opened during the current session; * this is used in generation of tempfile names. */ static long tempFileCounter = 0; /*-------------------- * * Private Routines * * Delete - delete a file from the Lru ring * LruDelete - remove a file from the Lru ring and close its FD * Insert - put a file at the front of the Lru ring * LruInsert - put a file at the front of the Lru ring and open it * ReleaseLruFile - Release an fd by closing the last entry in the Lru ring * AllocateVfd - grab a free (or new) file record (from VfdArray) * FreeVfd - free a file record * * The Least Recently Used ring is a doubly linked list that begins and * ends on element zero. Element zero is special -- it doesn't represent * a file and its "fd" field always == VFD_CLOSED. Element zero is just an * anchor that shows us the beginning/end of the ring. * Only VFD elements that are currently really open (have an FD assigned) are * in the Lru ring. Elements that are "virtually" open can be recognized * by having a non-null fileName field. * * example: * * /--less----\ /---------\ * v \ v \ * #0 --more---> LeastRecentlyUsed --more-\ \ * ^\ | | * \\less--> MostRecentlyUsedFile <---/ | * \more---/ \--less--/ * *-------------------- */ static void Delete(File file); static void LruDelete(File file); static void Insert(File file); static int LruInsert(File file); static bool ReleaseLruFile(void); static File AllocateVfd(void); static void FreeVfd(File file); static int FileAccess(File file); static File fileNameOpenFile(FileName fileName, int fileFlags, int fileMode); static char *filepath(const char *filename); static long pg_nofile(void); static void AtProcExit_Files(int code, Datum arg); static void CleanupTempFiles(bool isProcExit); /* * pg_fsync --- same as fsync except does nothing if enableFsync is off */ int pg_fsync(int fd) { if (enableFsync) return fsync(fd); else return 0; } /* * pg_fdatasync --- same as fdatasync except does nothing if enableFsync is off * * Not all platforms have fdatasync; treat as fsync if not available. */ int pg_fdatasync(int fd) { if (enableFsync) { #ifdef HAVE_FDATASYNC return fdatasync(fd); #else return fsync(fd); #endif } else return 0; } /* * BasicOpenFile --- same as open(2) except can free other FDs if needed * * This is exported for use by places that really want a plain kernel FD, * but need to be proof against running out of FDs. Once an FD has been * successfully returned, it is the caller's responsibility to ensure that * it will not be leaked on ereport()! Most users should *not* call this * routine directly, but instead use the VFD abstraction level, which * provides protection against descriptor leaks as well as management of * files that need to be open for more than a short period of time. * * Ideally this should be the *only* direct call of open() in the backend. * In practice, the postmaster calls open() directly, and there are some * direct open() calls done early in backend startup. Those are OK since * this module wouldn't have any open files to close at that point anyway. */ int BasicOpenFile(FileName fileName, int fileFlags, int fileMode) { int fd; tryAgain: fd = open(fileName, fileFlags, fileMode); if (fd >= 0) return fd; /* success! */ if (errno == EMFILE || errno == ENFILE) { int save_errno = errno; ereport(LOG, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("out of file descriptors: %m; release and retry"))); errno = 0; if (ReleaseLruFile()) goto tryAgain; errno = save_errno; } return -1; /* failure */ } /* * pg_nofile: determine number of filedescriptors that fd.c is allowed to use */ static long pg_nofile(void) { static long no_files = 0; /* need do this calculation only once */ if (no_files == 0) { /* * Ask the system what its files-per-process limit is. */ #ifdef HAVE_SYSCONF no_files = sysconf(_SC_OPEN_MAX); if (no_files <= 0) { #ifdef NOFILE no_files = (long) NOFILE; #else no_files = (long) max_files_per_process; #endif elog(LOG, "sysconf(_SC_OPEN_MAX) failed; using %ld", no_files); } #else /* !HAVE_SYSCONF */ #ifdef NOFILE no_files = (long) NOFILE; #else no_files = (long) max_files_per_process; #endif #endif /* HAVE_SYSCONF */ /* * Some platforms return hopelessly optimistic values. Apply a * configurable upper limit. */ if (no_files > (long) max_files_per_process) no_files = (long) max_files_per_process; /* * Make sure we have enough to get by after reserving some for LD. */ if ((no_files - RESERVE_FOR_LD) < FD_MINFREE) ereport(FATAL, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("insufficient file descriptors available to start server process"), errdetail("System allows %ld, we need at least %d.", no_files, RESERVE_FOR_LD + FD_MINFREE))); no_files -= RESERVE_FOR_LD; } return no_files; } #if defined(FDDEBUG) static void _dump_lru(void) { int mru = VfdCache[0].lruLessRecently; Vfd *vfdP = &VfdCache[mru]; char buf[2048]; snprintf(buf, sizeof(buf), "LRU: MOST %d ", mru); while (mru != 0) { mru = vfdP->lruLessRecently; vfdP = &VfdCache[mru]; snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), "%d ", mru); } snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), "LEAST"); elog(LOG, buf); } #endif /* FDDEBUG */ static void Delete(File file) { Vfd *vfdP; Assert(file != 0); DO_DB(elog(LOG, "Delete %d (%s)", file, VfdCache[file].fileName)); DO_DB(_dump_lru()); vfdP = &VfdCache[file]; VfdCache[vfdP->lruLessRecently].lruMoreRecently = vfdP->lruMoreRecently; VfdCache[vfdP->lruMoreRecently].lruLessRecently = vfdP->lruLessRecently; DO_DB(_dump_lru()); } static void LruDelete(File file) { Vfd *vfdP; Assert(file != 0); DO_DB(elog(LOG, "LruDelete %d (%s)", file, VfdCache[file].fileName)); vfdP = &VfdCache[file]; /* delete the vfd record from the LRU ring */ Delete(file); /* save the seek position */ vfdP->seekPos = (long) lseek(vfdP->fd, 0L, SEEK_CUR); Assert(vfdP->seekPos != -1L); /* close the file */ if (close(vfdP->fd)) elog(ERROR, "failed to close \"%s\": %m", vfdP->fileName); --nfile; vfdP->fd = VFD_CLOSED; } static void Insert(File file) { Vfd *vfdP; Assert(file != 0); DO_DB(elog(LOG, "Insert %d (%s)", file, VfdCache[file].fileName)); DO_DB(_dump_lru()); vfdP = &VfdCache[file]; vfdP->lruMoreRecently = 0; vfdP->lruLessRecently = VfdCache[0].lruLessRecently; VfdCache[0].lruLessRecently = file; VfdCache[vfdP->lruLessRecently].lruMoreRecently = file; DO_DB(_dump_lru()); } static int LruInsert(File file) { Vfd *vfdP; Assert(file != 0); DO_DB(elog(LOG, "LruInsert %d (%s)", file, VfdCache[file].fileName)); vfdP = &VfdCache[file]; if (FileIsNotOpen(file)) { while (nfile + numAllocatedFiles >= pg_nofile()) { if (!ReleaseLruFile()) break; } /* * The open could still fail for lack of file descriptors, eg due * to overall system file table being full. So, be prepared to * release another FD if necessary... */ vfdP->fd = BasicOpenFile(vfdP->fileName, vfdP->fileFlags, vfdP->fileMode); if (vfdP->fd < 0) { DO_DB(elog(LOG, "RE_OPEN FAILED: %d", errno)); return vfdP->fd; } else { DO_DB(elog(LOG, "RE_OPEN SUCCESS")); ++nfile; } /* seek to the right position */ if (vfdP->seekPos != 0L) { long returnValue; returnValue = (long) lseek(vfdP->fd, vfdP->seekPos, SEEK_SET); Assert(returnValue != -1L); } } /* * put it at the head of the Lru ring */ Insert(file); return 0; } static bool ReleaseLruFile(void) { DO_DB(elog(LOG, "ReleaseLruFile. Opened %d", nfile)); if (nfile > 0) { /* * There are opened files and so there should be at least one used * vfd in the ring. */ Assert(VfdCache[0].lruMoreRecently != 0); LruDelete(VfdCache[0].lruMoreRecently); return true; /* freed a file */ } return false; /* no files available to free */ } static File AllocateVfd(void) { Index i; File file; DO_DB(elog(LOG, "AllocateVfd. Size %d", SizeVfdCache)); if (SizeVfdCache == 0) { /* initialize header entry first time through */ VfdCache = (Vfd *) malloc(sizeof(Vfd)); if (VfdCache == NULL) ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); MemSet((char *) &(VfdCache[0]), 0, sizeof(Vfd)); VfdCache->fd = VFD_CLOSED; SizeVfdCache = 1; /* * register proc-exit call to ensure temp files are dropped at * exit */ on_proc_exit(AtProcExit_Files, 0); } if (VfdCache[0].nextFree == 0) { /* * The free list is empty so it is time to increase the size of * the array. We choose to double it each time this happens. * However, there's not much point in starting *real* small. */ Size newCacheSize = SizeVfdCache * 2; Vfd *newVfdCache; if (newCacheSize < 32) newCacheSize = 32; /* * Be careful not to clobber VfdCache ptr if realloc fails. */ newVfdCache = (Vfd *) realloc(VfdCache, sizeof(Vfd) * newCacheSize); if (newVfdCache == NULL) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); VfdCache = newVfdCache; /* * Initialize the new entries and link them into the free list. */ for (i = SizeVfdCache; i < newCacheSize; i++) { MemSet((char *) &(VfdCache[i]), 0, sizeof(Vfd)); VfdCache[i].nextFree = i + 1; VfdCache[i].fd = VFD_CLOSED; } VfdCache[newCacheSize - 1].nextFree = 0; VfdCache[0].nextFree = SizeVfdCache; /* * Record the new size */ SizeVfdCache = newCacheSize; } file = VfdCache[0].nextFree; VfdCache[0].nextFree = VfdCache[file].nextFree; return file; } static void FreeVfd(File file) { Vfd *vfdP = &VfdCache[file]; DO_DB(elog(LOG, "FreeVfd: %d (%s)", file, vfdP->fileName ? vfdP->fileName : "")); if (vfdP->fileName != NULL) { free(vfdP->fileName); vfdP->fileName = NULL; } vfdP->fdstate = 0x0; vfdP->nextFree = VfdCache[0].nextFree; VfdCache[0].nextFree = file; } /* filepath() * Convert given pathname to absolute. * * Result is a palloc'd string. * * (Generally, this isn't actually necessary, considering that we * should be cd'd into the database directory. Presently it is only * necessary to do it in "bootstrap" mode. Maybe we should change * bootstrap mode to do the cd, and save a few cycles/bytes here.) */ static char * filepath(const char *filename) { char *buf; /* Not an absolute path name? Then fill in with database path... */ if (!is_absolute_path(filename)) { buf = (char *) palloc(strlen(DatabasePath) + strlen(filename) + 2); sprintf(buf, "%s/%s", DatabasePath, filename); } else buf = pstrdup(filename); #ifdef FILEDEBUG printf("filepath: path is %s\n", buf); #endif return buf; } static int FileAccess(File file) { int returnValue; DO_DB(elog(LOG, "FileAccess %d (%s)", file, VfdCache[file].fileName)); /* * Is the file open? If not, open it and put it at the head of the * LRU ring (possibly closing the least recently used file to get an * FD). */ if (FileIsNotOpen(file)) { returnValue = LruInsert(file); if (returnValue != 0) return returnValue; } else if (VfdCache[0].lruLessRecently != file) { /* * We now know that the file is open and that it is not the last * one accessed, so we need to move it to the head of the Lru * ring. */ Delete(file); Insert(file); } return 0; } /* * Called when we get a shared invalidation message on some relation. */ #ifdef NOT_USED void FileInvalidate(File file) { Assert(FileIsValid(file)); if (!FileIsNotOpen(file)) LruDelete(file); } #endif static File fileNameOpenFile(FileName fileName, int fileFlags, int fileMode) { char *fnamecopy; File file; Vfd *vfdP; DO_DB(elog(LOG, "fileNameOpenFile: %s %x %o", fileName, fileFlags, fileMode)); /* * We need a malloc'd copy of the file name; fail cleanly if no room. */ fnamecopy = strdup(fileName); if (fnamecopy == NULL) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); file = AllocateVfd(); vfdP = &VfdCache[file]; while (nfile + numAllocatedFiles >= pg_nofile()) { if (!ReleaseLruFile()) break; } vfdP->fd = BasicOpenFile(fileName, fileFlags, fileMode); if (vfdP->fd < 0) { FreeVfd(file); free(fnamecopy); return -1; } ++nfile; DO_DB(elog(LOG, "fileNameOpenFile: success %d", vfdP->fd)); Insert(file); vfdP->fileName = fnamecopy; /* Saved flags are adjusted to be OK for re-opening file */ vfdP->fileFlags = fileFlags & ~(O_CREAT | O_TRUNC | O_EXCL); vfdP->fileMode = fileMode; vfdP->seekPos = 0; vfdP->fdstate = 0x0; return file; } /* * open a file in the database directory ($PGDATA/base/...) */ File FileNameOpenFile(FileName fileName, int fileFlags, int fileMode) { File fd; char *fname; fname = filepath(fileName); fd = fileNameOpenFile(fname, fileFlags, fileMode); pfree(fname); return fd; } /* * open a file in an arbitrary directory */ File PathNameOpenFile(FileName fileName, int fileFlags, int fileMode) { return fileNameOpenFile(fileName, fileFlags, fileMode); } /* * Open a temporary file that will disappear when we close it. * * This routine takes care of generating an appropriate tempfile name. * There's no need to pass in fileFlags or fileMode either, since only * one setting makes any sense for a temp file. * * interXact: if true, don't close the file at end-of-transaction. In * most cases, you don't want temporary files to outlive the transaction * that created them, so this should be false -- but if you need * "somewhat" temporary storage, this might be useful. In either case, * the file is removed when the File is explicitly closed. */ File OpenTemporaryFile(bool interXact) { char tempfilepath[MAXPGPATH]; File file; /* * Generate a tempfile name that should be unique within the current * database instance. */ snprintf(tempfilepath, sizeof(tempfilepath), "%s/%s%d.%ld", PG_TEMP_FILES_DIR, PG_TEMP_FILE_PREFIX, MyProcPid, tempFileCounter++); /* * Open the file. Note: we don't use O_EXCL, in case there is an * orphaned temp file that can be reused. */ file = FileNameOpenFile(tempfilepath, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY, 0600); if (file <= 0) { char *dirpath; /* * We might need to create the pg_tempfiles subdirectory, if no * one has yet done so. * * Don't check for error from mkdir; it could fail if someone else * just did the same thing. If it doesn't work then we'll bomb * out on the second create attempt, instead. */ dirpath = filepath(PG_TEMP_FILES_DIR); mkdir(dirpath, S_IRWXU); pfree(dirpath); file = FileNameOpenFile(tempfilepath, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY, 0600); if (file <= 0) elog(ERROR, "could not create temporary file \"%s\": %m", tempfilepath); } /* Mark it for deletion at close */ VfdCache[file].fdstate |= FD_TEMPORARY; /* Mark it for deletion at EOXact */ if (!interXact) VfdCache[file].fdstate |= FD_XACT_TEMPORARY; return file; } /* * close a file when done with it */ void FileClose(File file) { Vfd *vfdP; Assert(FileIsValid(file)); DO_DB(elog(LOG, "FileClose: %d (%s)", file, VfdCache[file].fileName)); vfdP = &VfdCache[file]; if (!FileIsNotOpen(file)) { /* remove the file from the lru ring */ Delete(file); /* close the file */ if (close(vfdP->fd)) elog(ERROR, "failed to close \"%s\": %m", vfdP->fileName); --nfile; vfdP->fd = VFD_CLOSED; } /* * Delete the file if it was temporary */ if (vfdP->fdstate & FD_TEMPORARY) { /* reset flag so that die() interrupt won't cause problems */ vfdP->fdstate &= ~FD_TEMPORARY; if (unlink(vfdP->fileName)) elog(LOG, "failed to unlink \"%s\": %m", vfdP->fileName); } /* * Return the Vfd slot to the free list */ FreeVfd(file); } /* * close a file and forcibly delete the underlying Unix file */ void FileUnlink(File file) { Assert(FileIsValid(file)); DO_DB(elog(LOG, "FileUnlink: %d (%s)", file, VfdCache[file].fileName)); /* force FileClose to delete it */ VfdCache[file].fdstate |= FD_TEMPORARY; FileClose(file); } int FileRead(File file, char *buffer, int amount) { int returnCode; Assert(FileIsValid(file)); DO_DB(elog(LOG, "FileRead: %d (%s) %ld %d %p", file, VfdCache[file].fileName, VfdCache[file].seekPos, amount, buffer)); FileAccess(file); returnCode = read(VfdCache[file].fd, buffer, amount); if (returnCode > 0) VfdCache[file].seekPos += returnCode; else VfdCache[file].seekPos = FileUnknownPos; return returnCode; } int FileWrite(File file, char *buffer, int amount) { int returnCode; Assert(FileIsValid(file)); DO_DB(elog(LOG, "FileWrite: %d (%s) %ld %d %p", file, VfdCache[file].fileName, VfdCache[file].seekPos, amount, buffer)); FileAccess(file); errno = 0; returnCode = write(VfdCache[file].fd, buffer, amount); /* if write didn't set errno, assume problem is no disk space */ if (returnCode != amount && errno == 0) errno = ENOSPC; if (returnCode > 0) VfdCache[file].seekPos += returnCode; else VfdCache[file].seekPos = FileUnknownPos; return returnCode; } long FileSeek(File file, long offset, int whence) { Assert(FileIsValid(file)); DO_DB(elog(LOG, "FileSeek: %d (%s) %ld %ld %d", file, VfdCache[file].fileName, VfdCache[file].seekPos, offset, whence)); if (FileIsNotOpen(file)) { switch (whence) { case SEEK_SET: if (offset < 0) elog(ERROR, "invalid seek offset: %ld", offset); VfdCache[file].seekPos = offset; break; case SEEK_CUR: VfdCache[file].seekPos += offset; break; case SEEK_END: FileAccess(file); VfdCache[file].seekPos = lseek(VfdCache[file].fd, offset, whence); break; default: elog(ERROR, "invalid whence: %d", whence); break; } } else { switch (whence) { case SEEK_SET: if (offset < 0) elog(ERROR, "invalid seek offset: %ld", offset); if (VfdCache[file].seekPos != offset) VfdCache[file].seekPos = lseek(VfdCache[file].fd, offset, whence); break; case SEEK_CUR: if (offset != 0 || VfdCache[file].seekPos == FileUnknownPos) VfdCache[file].seekPos = lseek(VfdCache[file].fd, offset, whence); break; case SEEK_END: VfdCache[file].seekPos = lseek(VfdCache[file].fd, offset, whence); break; default: elog(ERROR, "invalid whence: %d", whence); break; } } return VfdCache[file].seekPos; } /* * XXX not actually used but here for completeness */ #ifdef NOT_USED long FileTell(File file) { Assert(FileIsValid(file)); DO_DB(elog(LOG, "FileTell %d (%s)", file, VfdCache[file].fileName)); return VfdCache[file].seekPos; } #endif int FileTruncate(File file, long offset) { int returnCode; Assert(FileIsValid(file)); DO_DB(elog(LOG, "FileTruncate %d (%s)", file, VfdCache[file].fileName)); FileAccess(file); returnCode = ftruncate(VfdCache[file].fd, (size_t) offset); return returnCode; } /* * Routines that want to use stdio (ie, FILE*) should use AllocateFile * rather than plain fopen(). This lets fd.c deal with freeing FDs if * necessary to open the file. When done, call FreeFile rather than fclose. * * Note that files that will be open for any significant length of time * should NOT be handled this way, since they cannot share kernel file * descriptors with other files; there is grave risk of running out of FDs * if anyone locks down too many FDs. Most callers of this routine are * simply reading a config file that they will read and close immediately. * * fd.c will automatically close all files opened with AllocateFile at * transaction commit or abort; this prevents FD leakage if a routine * that calls AllocateFile is terminated prematurely by ereport(ERROR). * * Ideally this should be the *only* direct call of fopen() in the backend. */ FILE * AllocateFile(char *name, char *mode) { FILE *file; DO_DB(elog(LOG, "AllocateFile: Allocated %d", numAllocatedFiles)); if (numAllocatedFiles >= MAX_ALLOCATED_FILES) elog(ERROR, "too many private FDs demanded"); TryAgain: if ((file = fopen(name, mode)) != NULL) { allocatedFiles[numAllocatedFiles] = file; numAllocatedFiles++; return file; } if (errno == EMFILE || errno == ENFILE) { int save_errno = errno; ereport(LOG, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("out of file descriptors: %m; release and retry"))); errno = 0; if (ReleaseLruFile()) goto TryAgain; errno = save_errno; } return NULL; } /* * Close a file returned by AllocateFile. * * Note we do not check fclose's return value --- it is up to the caller * to handle close errors. */ int FreeFile(FILE *file) { int i; DO_DB(elog(LOG, "FreeFile: Allocated %d", numAllocatedFiles)); /* Remove file from list of allocated files, if it's present */ for (i = numAllocatedFiles; --i >= 0;) { if (allocatedFiles[i] == file) { numAllocatedFiles--; allocatedFiles[i] = allocatedFiles[numAllocatedFiles]; break; } } if (i < 0) elog(WARNING, "file passed to FreeFile was not obtained from AllocateFile"); return fclose(file); } /* * closeAllVfds * * Force all VFDs into the physically-closed state, so that the fewest * possible number of kernel file descriptors are in use. There is no * change in the logical state of the VFDs. */ void closeAllVfds(void) { Index i; if (SizeVfdCache > 0) { Assert(FileIsNotOpen(0)); /* Make sure ring not corrupted */ for (i = 1; i < SizeVfdCache; i++) { if (!FileIsNotOpen(i)) LruDelete(i); } } } /* * AtEOXact_Files * * This routine is called during transaction commit or abort (it doesn't * particularly care which). All still-open per-transaction temporary file * VFDs are closed, which also causes the underlying files to be * deleted. Furthermore, all "allocated" stdio files are closed. */ void AtEOXact_Files(void) { CleanupTempFiles(false); } /* * AtProcExit_Files * * on_proc_exit hook to clean up temp files during backend shutdown. * Here, we want to clean up *all* temp files including interXact ones. */ static void AtProcExit_Files(int code, Datum arg) { CleanupTempFiles(true); } /* * Close temporary files and delete their underlying files. * * isProcExit: if true, this is being called as the backend process is * exiting. If that's the case, we should remove all temporary files; if * that's not the case, we are being called for transaction commit/abort * and should only remove transaction-local temp files. In either case, * also clean up "allocated" stdio files. */ static void CleanupTempFiles(bool isProcExit) { Index i; if (SizeVfdCache > 0) { Assert(FileIsNotOpen(0)); /* Make sure ring not corrupted */ for (i = 1; i < SizeVfdCache; i++) { unsigned short fdstate = VfdCache[i].fdstate; if ((fdstate & FD_TEMPORARY) && VfdCache[i].fileName != NULL) { /* * If we're in the process of exiting a backend process, * close all temporary files. Otherwise, only close * temporary files local to the current transaction. */ if (isProcExit || (fdstate & FD_XACT_TEMPORARY)) FileClose(i); } } } while (numAllocatedFiles > 0) FreeFile(allocatedFiles[0]); } /* * Remove temporary files left over from a prior postmaster session * * This should be called during postmaster startup. It will forcibly * remove any leftover files created by OpenTemporaryFile. * * NOTE: we could, but don't, call this during a post-backend-crash restart * cycle. The argument for not doing it is that someone might want to examine * the temp files for debugging purposes. This does however mean that * OpenTemporaryFile had better allow for collision with an existing temp * file name. */ void RemovePgTempFiles(void) { char db_path[MAXPGPATH]; char temp_path[MAXPGPATH]; char rm_path[MAXPGPATH]; DIR *db_dir; DIR *temp_dir; struct dirent *db_de; struct dirent *temp_de; /* * Cycle through pg_tempfiles for all databases and remove old temp * files. */ snprintf(db_path, sizeof(db_path), "%s/base", DataDir); if ((db_dir = opendir(db_path)) != NULL) { while ((db_de = readdir(db_dir)) != NULL) { if (strcmp(db_de->d_name, ".") == 0 #ifndef EXEC_BACKEND /* no PG_TEMP_FILES_DIR in DataDir in non EXEC_BACKEND case */ || strcmp(db_de->d_name, "..") == 0 #endif ) continue; snprintf(temp_path, sizeof(temp_path), "%s/%s/%s", db_path, db_de->d_name, PG_TEMP_FILES_DIR); if ((temp_dir = opendir(temp_path)) != NULL) { while ((temp_de = readdir(temp_dir)) != NULL) { if (strcmp(temp_de->d_name, ".") == 0 || strcmp(temp_de->d_name, "..") == 0) continue; snprintf(rm_path, sizeof(temp_path), "%s/%s/%s/%s", db_path, db_de->d_name, PG_TEMP_FILES_DIR, temp_de->d_name); if (strncmp(temp_de->d_name, PG_TEMP_FILE_PREFIX, strlen(PG_TEMP_FILE_PREFIX)) == 0) unlink(rm_path); else elog(LOG, "unexpected file found in temporary-files directory: \"%s\"", rm_path); } closedir(temp_dir); } } closedir(db_dir); } }