diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index 5775d8d1ee..6315dacf40 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -1,5 +1,5 @@ @@ -474,6 +474,23 @@ HINT: Stop the postmaster and use a standalone backend to VACUUM in "mydb". + + Check files after crash + + + stale file + + + + PostgreSQL recovers automatically after crash + using the write-ahead log (see ) and no manual + operations are normally needed. However, if there was a transaction running + when the crash occured that created or dropped a relation, the + transaction might have left a stale file in the data directory. If this + happens, you will get a notice in the log file stating which files can be + deleted. + + Log File Maintenance diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 48e8b425de..096b822599 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.189 2005/04/28 21:47:10 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.190 2005/05/02 18:26:52 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -43,6 +43,7 @@ #include "utils/builtins.h" #include "utils/guc.h" #include "utils/relcache.h" +#include "utils/flatfiles.h" /* @@ -4525,6 +4526,8 @@ StartupXLOG(void) CreateCheckPoint(true, true); + CheckStaleRelFiles(); + /* * Close down recovery environment */ @@ -4536,6 +4539,12 @@ StartupXLOG(void) */ remove_backup_label(); } + else + { + XLogInitRelationCache(); + CheckStaleRelFiles(); + XLogCloseRelationCache(); + } /* * Preallocate additional log files, if wanted. diff --git a/src/backend/catalog/catalog.c b/src/backend/catalog/catalog.c index 9e08e2120b..044b4a1bbd 100644 --- a/src/backend/catalog/catalog.c +++ b/src/backend/catalog/catalog.c @@ -9,7 +9,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/catalog/catalog.c,v 1.59 2005/04/14 20:03:23 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/catalog/catalog.c,v 1.60 2005/05/02 18:26:53 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -106,6 +106,39 @@ GetDatabasePath(Oid dbNode, Oid spcNode) return path; } +/* + * GetTablespacePath - construct path to a tablespace symbolic link + * + * Result is a palloc'd string. + * + * XXX this must agree with relpath and GetDatabasePath! + */ +char * +GetTablespacePath(Oid spcNode) +{ + int pathlen; + char *path; + + Assert(spcNode != GLOBALTABLESPACE_OID); + + if (spcNode == DEFAULTTABLESPACE_OID) + { + /* The default tablespace is {datadir}/base */ + pathlen = strlen(DataDir) + 5 + 1; + path = (char *) palloc(pathlen); + snprintf(path, pathlen, "%s/base", + DataDir); + } + else + { + /* All other tablespaces have symlinks in pg_tblspc */ + pathlen = strlen(DataDir) + 11 + OIDCHARS + 1; + path = (char *) palloc(pathlen); + snprintf(path, pathlen, "%s/pg_tblspc/%u", + DataDir, spcNode); + } + return path; +} /* * IsSystemRelation diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index 9a18fd6e83..be0cc55594 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -37,7 +37,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/tablespace.c,v 1.17 2005/04/14 20:03:24 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/commands/tablespace.c,v 1.18 2005/05/02 18:26:53 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -341,8 +341,7 @@ CreateTableSpace(CreateTableSpaceStmt *stmt) /* * All seems well, create the symlink */ - linkloc = (char *) palloc(strlen(DataDir) + 11 + 10 + 1); - sprintf(linkloc, "%s/pg_tblspc/%u", DataDir, tablespaceoid); + linkloc = GetTablespacePath(tablespaceoid); if (symlink(location, linkloc) < 0) ereport(ERROR, @@ -495,8 +494,7 @@ remove_tablespace_directories(Oid tablespaceoid, bool redo) char *subfile; struct stat st; - location = (char *) palloc(strlen(DataDir) + 11 + 10 + 1); - sprintf(location, "%s/pg_tblspc/%u", DataDir, tablespaceoid); + location = GetTablespacePath(tablespaceoid); /* * Check if the tablespace still contains any files. We try to rmdir @@ -1036,8 +1034,7 @@ tblspc_redo(XLogRecPtr lsn, XLogRecord *record) set_short_version(location); /* Create the symlink if not already present */ - linkloc = (char *) palloc(strlen(DataDir) + 11 + 10 + 1); - sprintf(linkloc, "%s/pg_tblspc/%u", DataDir, xlrec->ts_id); + linkloc = GetTablespacePath(xlrec->ts_id); if (symlink(location, linkloc) < 0) { diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c index 29ddf03535..68586bba7e 100644 --- a/src/backend/utils/adt/misc.c +++ b/src/backend/utils/adt/misc.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/misc.c,v 1.40 2004/12/31 22:01:22 pgsql Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/misc.c,v 1.41 2005/05/02 18:26:53 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -26,6 +26,7 @@ #include "funcapi.h" #include "catalog/pg_type.h" #include "catalog/pg_tablespace.h" +#include "catalog/catalog.h" #define atooid(x) ((Oid) strtoul((x), NULL, 10)) @@ -144,11 +145,6 @@ pg_tablespace_databases(PG_FUNCTION_ARGS) fctx = palloc(sizeof(ts_db_fctx)); - /* - * size = path length + tablespace dirname length + 2 dir sep - * chars + oid + terminator - */ - fctx->location = (char *) palloc(strlen(DataDir) + 11 + 10 + 1); if (tablespaceOid == GLOBALTABLESPACE_OID) { fctx->dirdesc = NULL; @@ -157,12 +153,7 @@ pg_tablespace_databases(PG_FUNCTION_ARGS) } else { - if (tablespaceOid == DEFAULTTABLESPACE_OID) - sprintf(fctx->location, "%s/base", DataDir); - else - sprintf(fctx->location, "%s/pg_tblspc/%u", DataDir, - tablespaceOid); - + fctx->location = GetTablespacePath(tablespaceOid); fctx->dirdesc = AllocateDir(fctx->location); if (!fctx->dirdesc) diff --git a/src/backend/utils/init/Makefile b/src/backend/utils/init/Makefile index ffc5ff9e06..3e5f885390 100644 --- a/src/backend/utils/init/Makefile +++ b/src/backend/utils/init/Makefile @@ -4,7 +4,7 @@ # Makefile for utils/init # # IDENTIFICATION -# $PostgreSQL: pgsql/src/backend/utils/init/Makefile,v 1.18 2005/02/20 02:22:00 tgl Exp $ +# $PostgreSQL: pgsql/src/backend/utils/init/Makefile,v 1.19 2005/05/02 18:26:53 momjian Exp $ # #------------------------------------------------------------------------- @@ -12,7 +12,7 @@ subdir = src/backend/utils/init top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = flatfiles.o globals.o miscinit.o postinit.o +OBJS = flatfiles.o globals.o miscinit.o postinit.o checkfiles.o all: SUBSYS.o diff --git a/src/backend/utils/init/checkfiles.c b/src/backend/utils/init/checkfiles.c new file mode 100644 index 0000000000..1c0a53c1d4 --- /dev/null +++ b/src/backend/utils/init/checkfiles.c @@ -0,0 +1,225 @@ +/*------------------------------------------------------------------------- + * + * checkfiles.c + * support to clean up stale relation files on crash recovery + * + * If a backend crashes while in a transaction that has created or + * deleted a relfilenode, a stale file can be left over in the data + * directory. This file contains routines to clean up those stale + * files on recovery. + * + * This adds a 17% increase in startup cost for 100 empty databases. bjm + * One optimization would be to create a 'dirty' file on a postmaster recovery + * and remove the dirty flag only when a clean startup detects no unreferenced + * files, and use the 'dirty' flag to determine if we should run this on + * a clean startup. + * + * $PostgreSQL: pgsql/src/backend/utils/init/checkfiles.c,v 1.1 2005/05/02 18:26:53 momjian Exp $ + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "storage/fd.h" + +#include "utils/flatfiles.h" +#include "miscadmin.h" +#include "catalog/pg_tablespace.h" +#include "catalog/catalog.h" +#include "access/skey.h" +#include "utils/fmgroids.h" +#include "access/relscan.h" +#include "access/heapam.h" +#include "utils/resowner.h" + +static void CheckStaleRelFilesFrom(Oid tablespaceoid, Oid dboid); +static void CheckStaleRelFilesFromTablespace(Oid tablespaceoid); + +/* Like AllocateDir, but ereports on failure */ +static DIR * +AllocateDirChecked(char *path) +{ + DIR *dirdesc = AllocateDir(path); + + if (dirdesc == NULL) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open directory \"%s\": %m", + path))); + return dirdesc; +} + +/* + * Scan through all tablespaces for relations left over + * by aborted transactions. + * + * For example, if a transaction issues + * BEGIN; CREATE TABLE foobar (); + * and then the backend crashes, the file is left in the + * tablespace until CheckStaleRelFiles deletes it. + */ +void +CheckStaleRelFiles(void) +{ + DIR *dirdesc; + struct dirent *de; + char *path; + int pathlen; + + pathlen = strlen(DataDir) + 11 + 1; + path = (char *) palloc(pathlen); + snprintf(path, pathlen, "%s/pg_tblspc/", DataDir); + dirdesc = AllocateDirChecked(path); + while ((de = readdir(dirdesc)) != NULL) + { + char *invalid; + Oid tablespaceoid; + + /* Check that the directory name looks like valid tablespace link. */ + tablespaceoid = (Oid) strtol(de->d_name, &invalid, 10); + if (invalid[0] == '\0') + CheckStaleRelFilesFromTablespace(tablespaceoid); + } + FreeDir(dirdesc); + pfree(path); + + CheckStaleRelFilesFromTablespace(DEFAULTTABLESPACE_OID); +} + +/* Scan a specific tablespace for stale relations */ +static void +CheckStaleRelFilesFromTablespace(Oid tablespaceoid) +{ + DIR *dirdesc; + struct dirent *de; + char *path; + + path = GetTablespacePath(tablespaceoid); + + dirdesc = AllocateDirChecked(path); + while ((de = readdir(dirdesc)) != NULL) + { + char *invalid; + Oid dboid; + + dboid = (Oid) strtol(de->d_name, &invalid, 10); + if (invalid[0] == '\0') + CheckStaleRelFilesFrom(tablespaceoid, dboid); + } + FreeDir(dirdesc); + pfree(path); +} + +/* Scan a specific database in a specific tablespace for stale relations. + * + * First, pg_class for the database is opened, and the relfilenodes of all + * relations mentioned there are stored in a hash table. + * + * Then the directory is scanned. Every file in the directory that's not + * found in pg_class (the hash table) is logged. + */ +static void +CheckStaleRelFilesFrom(Oid tablespaceoid, Oid dboid) +{ + DIR *dirdesc; + struct dirent *de; + HASHCTL hashctl; + HTAB *relfilenodeHash; + MemoryContext mcxt; + RelFileNode rnode; + char *path; + + /* + * We create a private memory context so that we can easily deallocate the + * hash table and its contents + */ + mcxt = AllocSetContextCreate(TopMemoryContext, "CheckStaleRelFiles", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + + hashctl.hash = tag_hash; + + /* + * The entry contents is not used for anything, we just check if an oid is + * in the hash table or not. + */ + hashctl.keysize = sizeof(Oid); + hashctl.entrysize = 1; + hashctl.hcxt = mcxt; + relfilenodeHash = hash_create("relfilenodeHash", 100, &hashctl, + HASH_FUNCTION + | HASH_ELEM | HASH_CONTEXT); + + /* Read all relfilenodes from pg_class into the hash table */ + { + ResourceOwner owner, + oldowner; + Relation rel; + HeapScanDesc scan; + HeapTuple tuple; + + /* Need a resowner to keep the heapam and buffer code happy */ + owner = ResourceOwnerCreate(NULL, "CheckStaleRelFiles"); + oldowner = CurrentResourceOwner; + CurrentResourceOwner = owner; + + rnode.spcNode = tablespaceoid; + rnode.dbNode = dboid; + rnode.relNode = RelationRelationId; + rel = XLogOpenRelation(true, 0, rnode); + + scan = heap_beginscan(rel, SnapshotNow, 0, NULL); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_class classform = (Form_pg_class) GETSTRUCT(tuple); + + hash_search(relfilenodeHash, &classform->relfilenode, + HASH_ENTER, NULL); + } + heap_endscan(scan); + + XLogCloseRelation(rnode); + CurrentResourceOwner = oldowner; + ResourceOwnerDelete(owner); + } + + /* Scan the directory */ + path = GetDatabasePath(dboid, tablespaceoid); + + dirdesc = AllocateDirChecked(path); + while ((de = readdir(dirdesc)) != NULL) + { + char *invalid; + Oid relfilenode; + + relfilenode = strtol(de->d_name, &invalid, 10); + if (invalid[0] == '\0') + { + /* + * Filename was a valid number, check if pg_class knows about it + */ + if (hash_search(relfilenodeHash, &relfilenode, + HASH_FIND, NULL) == NULL) + { + char *filepath; + + rnode.spcNode = tablespaceoid; + rnode.dbNode = dboid; + rnode.relNode = relfilenode; + + filepath = relpath(rnode); + + ereport(LOG, + (errcode_for_file_access(), + errmsg("The table or index file \"%s\" is stale and can be safely removed", + filepath))); + pfree(filepath); + } + } + } + FreeDir(dirdesc); + pfree(path); + hash_destroy(relfilenodeHash); + MemoryContextDelete(mcxt); +} diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index e18c806788..836a04583d 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/catalog.h,v 1.30 2004/12/31 22:03:24 pgsql Exp $ + * $PostgreSQL: pgsql/src/include/catalog/catalog.h,v 1.31 2005/05/02 18:26:54 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -19,6 +19,7 @@ extern char *relpath(RelFileNode rnode); extern char *GetDatabasePath(Oid dbNode, Oid spcNode); +extern char *GetTablespacePath(Oid spcNode); extern bool IsSystemRelation(Relation relation); extern bool IsToastRelation(Relation relation); diff --git a/src/include/utils/flatfiles.h b/src/include/utils/flatfiles.h index 02e5175925..21eca19477 100644 --- a/src/include/utils/flatfiles.h +++ b/src/include/utils/flatfiles.h @@ -4,7 +4,7 @@ * Routines for maintaining "flat file" images of the shared catalogs. * * - * $PostgreSQL: pgsql/src/include/utils/flatfiles.h,v 1.1 2005/02/20 02:22:07 tgl Exp $ + * $PostgreSQL: pgsql/src/include/utils/flatfiles.h,v 1.2 2005/05/02 18:26:54 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -30,4 +30,7 @@ extern void AtEOSubXact_UpdateFlatFiles(bool isCommit, extern Datum flatfile_update_trigger(PG_FUNCTION_ARGS); +/* from checkfiles.c */ +extern void CheckStaleRelFiles(void); + #endif /* FLATFILES_H */