From 92c2ecc130239b38580b5c11c532d2e027fc2d06 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Sun, 30 Jul 2006 02:07:18 +0000 Subject: [PATCH] Modify snapshot definition so that lazy vacuums are ignored by other vacuums. This allows a OLTP-like system with big tables to continue regular vacuuming on small-but-frequently-updated tables while the big tables are being vacuumed. Original patch from Hannu Krossing, rewritten by Tom Lane and updated by me. --- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xact.c | 5 ++- src/backend/access/transam/xlog.c | 4 +-- src/backend/catalog/index.c | 5 +-- src/backend/commands/vacuum.c | 50 ++++++++++++++++++++++++--- src/backend/storage/ipc/procarray.c | 43 ++++++++++++++++------- src/backend/storage/lmgr/proc.c | 4 ++- src/include/storage/proc.h | 9 +++-- src/include/storage/procarray.h | 4 +-- 9 files changed, 99 insertions(+), 28 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 481d4af86b..9b9fed3ed0 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -7,7 +7,7 @@ * Portions Copyright (c) 1994, Regents of the University of California * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/access/transam/twophase.c,v 1.21 2006/07/14 14:52:17 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/access/transam/twophase.c,v 1.22 2006/07/30 02:07:18 alvherre Exp $ * * NOTES * Each global transaction is associated with a global transaction @@ -279,6 +279,7 @@ MarkAsPreparing(TransactionId xid, const char *gid, gxact->proc.pid = 0; gxact->proc.databaseId = databaseid; gxact->proc.roleId = owner; + gxact->proc.inVacuum = false; gxact->proc.lwWaiting = false; gxact->proc.lwExclusive = false; gxact->proc.lwWaitLink = NULL; diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index e9473acaa6..81c2583571 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -10,7 +10,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/access/transam/xact.c,v 1.224 2006/07/24 16:32:44 petere Exp $ + * $PostgreSQL: pgsql/src/backend/access/transam/xact.c,v 1.225 2006/07/30 02:07:18 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -1529,6 +1529,7 @@ CommitTransaction(void) LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); MyProc->xid = InvalidTransactionId; MyProc->xmin = InvalidTransactionId; + MyProc->inVacuum = false; /* must be cleared with xid/xmin */ /* Clear the subtransaction-XID cache too while holding the lock */ MyProc->subxids.nxids = 0; @@ -1764,6 +1765,7 @@ PrepareTransaction(void) LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); MyProc->xid = InvalidTransactionId; MyProc->xmin = InvalidTransactionId; + MyProc->inVacuum = false; /* must be cleared with xid/xmin */ /* Clear the subtransaction-XID cache too while holding the lock */ MyProc->subxids.nxids = 0; @@ -1927,6 +1929,7 @@ AbortTransaction(void) LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); MyProc->xid = InvalidTransactionId; MyProc->xmin = InvalidTransactionId; + MyProc->inVacuum = false; /* must be cleared with xid/xmin */ /* Clear the subtransaction-XID cache too while holding the lock */ MyProc->subxids.nxids = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e9265dde48..6b5c5ac0da 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.244 2006/07/14 14:52:17 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.245 2006/07/30 02:07:18 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -5413,7 +5413,7 @@ CreateCheckPoint(bool shutdown, bool force) * StartupSUBTRANS hasn't been called yet. */ if (!InRecovery) - TruncateSUBTRANS(GetOldestXmin(true)); + TruncateSUBTRANS(GetOldestXmin(true, false)); if (!shutdown) ereport(DEBUG2, diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 1e498ccf75..829fd5e4fb 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.269 2006/07/13 16:49:13 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.270 2006/07/30 02:07:18 alvherre Exp $ * * * INTERFACE ROUTINES @@ -1367,7 +1367,8 @@ IndexBuildHeapScan(Relation heapRelation, else { snapshot = SnapshotAny; - OldestXmin = GetOldestXmin(heapRelation->rd_rel->relisshared); + /* okay to ignore lazy VACUUMs here */ + OldestXmin = GetOldestXmin(heapRelation->rd_rel->relisshared, true); } scan = heap_beginscan(heapRelation, /* relation */ diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index b1f187dc47..eb0fce72ed 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -13,7 +13,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.335 2006/07/14 14:52:18 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.336 2006/07/30 02:07:18 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -37,6 +37,7 @@ #include "postmaster/autovacuum.h" #include "storage/freespace.h" #include "storage/pmsignal.h" +#include "storage/proc.h" #include "storage/procarray.h" #include "utils/acl.h" #include "utils/builtins.h" @@ -589,7 +590,16 @@ vacuum_set_xid_limits(VacuumStmt *vacstmt, bool sharedRel, { TransactionId limit; - *oldestXmin = GetOldestXmin(sharedRel); + /* + * We can always ignore processes running lazy vacuum. This is because we + * use these values only for deciding which tuples we must keep in the + * tables. Since lazy vacuum doesn't write its xid to the table, it's + * safe to ignore it. In theory it could be problematic to ignore lazy + * vacuums on a full vacuum, but keep in mind that only one vacuum process + * can be working on a particular table at any time, and that each vacuum + * is always an independent transaction. + */ + *oldestXmin = GetOldestXmin(sharedRel, true); Assert(TransactionIdIsNormal(*oldestXmin)); @@ -645,6 +655,11 @@ vacuum_set_xid_limits(VacuumStmt *vacstmt, bool sharedRel, * pg_class would've been obsoleted. Of course, this only works for * fixed-size never-null columns, but these are. * + * Another reason for doing it this way is that when we are in a lazy + * VACUUM and have inVacuum set, we mustn't do any updates --- somebody + * vacuuming pg_class might think they could delete a tuple marked with + * xmin = our xid. + * * This routine is shared by full VACUUM, lazy VACUUM, and stand-alone * ANALYZE. */ @@ -996,8 +1011,35 @@ vacuum_rel(Oid relid, VacuumStmt *vacstmt, char expected_relkind) /* Begin a transaction for vacuuming this relation */ StartTransactionCommand(); - /* functions in indexes may want a snapshot set */ - ActiveSnapshot = CopySnapshot(GetTransactionSnapshot()); + + if (vacstmt->full) + { + /* functions in indexes may want a snapshot set */ + ActiveSnapshot = CopySnapshot(GetTransactionSnapshot()); + } + else + { + /* + * During a lazy VACUUM we do not run any user-supplied functions, + * and so it should be safe to not create a transaction snapshot. + * + * We can furthermore set the inVacuum flag, which lets other + * concurrent VACUUMs know that they can ignore this one while + * determining their OldestXmin. (The reason we don't set inVacuum + * during a full VACUUM is exactly that we may have to run user- + * defined functions for functional indexes, and we want to make + * sure that if they use the snapshot set above, any tuples it + * requires can't get removed from other tables. An index function + * that depends on the contents of other tables is arguably broken, + * but we won't break it here by violating transaction semantics.) + * + * Note: the inVacuum flag remains set until CommitTransaction or + * AbortTransaction. We don't want to clear it until we reset + * MyProc->xid/xmin, else OldestXmin might appear to go backwards, + * which is probably Not Good. + */ + MyProc->inVacuum = true; + } /* * Tell the cache replacement strategy that vacuum is causing all diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index c8a173fbee..e9d0da12ff 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -23,7 +23,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/storage/ipc/procarray.c,v 1.14 2006/07/14 14:52:22 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/storage/ipc/procarray.c,v 1.15 2006/07/30 02:07:18 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -388,20 +388,24 @@ TransactionIdIsActive(TransactionId xid) * If allDbs is TRUE then all backends are considered; if allDbs is FALSE * then only backends running in my own database are considered. * + * If ignoreVacuum is TRUE then backends with inVacuum set are ignored. + * * This is used by VACUUM to decide which deleted tuples must be preserved * in a table. allDbs = TRUE is needed for shared relations, but allDbs = * FALSE is sufficient for non-shared relations, since only backends in my - * own database could ever see the tuples in them. + * own database could ever see the tuples in them. Also, we can ignore + * concurrently running lazy VACUUMs because (a) they must be working on other + * tables, and (b) they don't need to do snapshot-based lookups. * * This is also used to determine where to truncate pg_subtrans. allDbs - * must be TRUE for that case. + * must be TRUE for that case, and ignoreVacuum FALSE. * * Note: we include the currently running xids in the set of considered xids. * This ensures that if a just-started xact has not yet set its snapshot, * when it does set the snapshot it cannot set xmin less than what we compute. */ TransactionId -GetOldestXmin(bool allDbs) +GetOldestXmin(bool allDbs, bool ignoreVacuum) { ProcArrayStruct *arrayP = procArray; TransactionId result; @@ -425,6 +429,9 @@ GetOldestXmin(bool allDbs) { PGPROC *proc = arrayP->procs[index]; + if (ignoreVacuum && proc->inVacuum) + continue; + if (allDbs || proc->databaseId == MyDatabaseId) { /* Fetch xid just once - see GetNewTransactionId */ @@ -432,8 +439,18 @@ GetOldestXmin(bool allDbs) if (TransactionIdIsNormal(xid)) { + /* First consider the transaction own's Xid */ if (TransactionIdPrecedes(xid, result)) result = xid; + + /* + * Also consider the transaction's Xmin, if set. + * + * Note that this Xmin may seem to be guaranteed to be always + * lower than the transaction's Xid, but this is not so because + * there is a time window on which the Xid is already assigned + * but the Xmin has not being calculated yet. + */ xid = proc->xmin; if (TransactionIdIsNormal(xid)) if (TransactionIdPrecedes(xid, result)) @@ -471,8 +488,8 @@ GetOldestXmin(bool allDbs) * RecentXmin: the xmin computed for the most recent snapshot. XIDs * older than this are known not running any more. * RecentGlobalXmin: the global xmin (oldest TransactionXmin across all - * running transactions). This is the same computation done by - * GetOldestXmin(TRUE). + * running transactions, except those running LAZY VACUUM). This is + * the same computation done by GetOldestXmin(true, false). *---------- */ Snapshot @@ -561,15 +578,17 @@ GetSnapshotData(Snapshot snapshot, bool serializable) /* * Ignore my own proc (dealt with my xid above), procs not running a - * transaction, and xacts started since we read the next transaction - * ID. There's no need to store XIDs above what we got from - * ReadNewTransactionId, since we'll treat them as running anyway. We - * also assume that such xacts can't compute an xmin older than ours, - * so they needn't be considered in computing globalxmin. + * transaction, xacts started since we read the next transaction + * ID, and xacts executing LAZY VACUUM. There's no need to store XIDs + * above what we got from ReadNewTransactionId, since we'll treat them + * as running anyway. We also assume that such xacts can't compute an + * xmin older than ours, so they needn't be considered in computing + * globalxmin. */ if (proc == MyProc || !TransactionIdIsNormal(xid) || - TransactionIdFollowsOrEquals(xid, xmax)) + TransactionIdFollowsOrEquals(xid, xmax) || + proc->inVacuum) continue; if (TransactionIdPrecedes(xid, xmin)) diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 04cd338aac..07fc3e3d25 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/storage/lmgr/proc.c,v 1.178 2006/07/23 23:08:46 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/storage/lmgr/proc.c,v 1.179 2006/07/30 02:07:18 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -257,6 +257,7 @@ InitProcess(void) /* databaseId and roleId will be filled in later */ MyProc->databaseId = InvalidOid; MyProc->roleId = InvalidOid; + MyProc->inVacuum = false; MyProc->lwWaiting = false; MyProc->lwExclusive = false; MyProc->lwWaitLink = NULL; @@ -388,6 +389,7 @@ InitDummyProcess(void) MyProc->xmin = InvalidTransactionId; MyProc->databaseId = InvalidOid; MyProc->roleId = InvalidOid; + MyProc->inVacuum = false; MyProc->lwWaiting = false; MyProc->lwExclusive = false; MyProc->lwWaitLink = NULL; diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 83816dd05d..0345b42f97 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/storage/proc.h,v 1.89 2006/07/13 16:49:20 momjian Exp $ + * $PostgreSQL: pgsql/src/include/storage/proc.h,v 1.90 2006/07/30 02:07:18 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -66,13 +66,16 @@ struct PGPROC * this proc */ TransactionId xmin; /* minimal running XID as it was when we were - * starting our xact: vacuum must not remove - * tuples deleted by xid >= xmin ! */ + * starting our xact, excluding LAZY VACUUM: + * vacuum must not remove tuples deleted by + * xid >= xmin ! */ int pid; /* This backend's process id, or 0 */ Oid databaseId; /* OID of database this backend is using */ Oid roleId; /* OID of role using this backend */ + bool inVacuum; /* true if current xact is a LAZY VACUUM */ + /* Info about LWLock the process is currently waiting for, if any. */ bool lwWaiting; /* true if waiting for an LW lock */ bool lwExclusive; /* true if waiting for exclusive access */ diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index e1710701a1..b9da4db918 100644 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/storage/procarray.h,v 1.9 2006/06/19 01:51:22 tgl Exp $ + * $PostgreSQL: pgsql/src/include/storage/procarray.h,v 1.10 2006/07/30 02:07:18 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -24,7 +24,7 @@ extern void ProcArrayRemove(PGPROC *proc); extern bool TransactionIdIsInProgress(TransactionId xid); extern bool TransactionIdIsActive(TransactionId xid); -extern TransactionId GetOldestXmin(bool allDbs); +extern TransactionId GetOldestXmin(bool allDbs, bool ignoreVacuum); extern PGPROC *BackendPidGetProc(int pid); extern int BackendXidGetPid(TransactionId xid);