From 3ccde312ec8ee47f5f797b070d34a675799448ae Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Wed, 13 Aug 2008 00:07:50 +0000 Subject: [PATCH] Have autovacuum consider processing TOAST tables separately from their main tables. This requires vacuum() to accept processing a toast table standalone, so there's a user-visible change in that it's now possible (for a superuser) to execute "VACUUM pg_toast.pg_toast_XXX". --- src/backend/commands/vacuum.c | 37 ++-- src/backend/postmaster/autovacuum.c | 276 +++++++++++++++------------- src/backend/tcop/utility.c | 4 +- src/include/commands/vacuum.h | 4 +- 4 files changed, 176 insertions(+), 145 deletions(-) diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 176bd00693..7eb5777d3a 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -13,7 +13,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.375 2008/06/05 15:47:32 alvherre Exp $ + * $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.376 2008/08/13 00:07:50 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -213,8 +213,8 @@ static BufferAccessStrategy vac_strategy; static List *get_rel_oids(Oid relid, const RangeVar *vacrel, const char *stmttype); static void vac_truncate_clog(TransactionId frozenXID); -static void vacuum_rel(Oid relid, VacuumStmt *vacstmt, char expected_relkind, - bool for_wraparound); +static void vacuum_rel(Oid relid, VacuumStmt *vacstmt, bool do_toast, + bool for_wraparound); static void full_vacuum_rel(Relation onerel, VacuumStmt *vacstmt); static void scan_heap(VRelStats *vacrelstats, Relation onerel, VacPageList vacuum_pages, VacPageList fraged_pages); @@ -268,6 +268,9 @@ static Size PageGetFreeSpaceWithFillFactor(Relation relation, Page page); * OID to be processed, and vacstmt->relation is ignored. (The non-invalid * case is currently only used by autovacuum.) * + * do_toast is passed as FALSE by autovacuum, because it processes TOAST + * tables separately. + * * for_wraparound is used by autovacuum to let us know when it's forcing * a vacuum for wraparound, which should not be auto-cancelled. * @@ -281,7 +284,7 @@ static Size PageGetFreeSpaceWithFillFactor(Relation relation, Page page); * at transaction commit. */ void -vacuum(VacuumStmt *vacstmt, Oid relid, +vacuum(VacuumStmt *vacstmt, Oid relid, bool do_toast, BufferAccessStrategy bstrategy, bool for_wraparound, bool isTopLevel) { const char *stmttype = vacstmt->vacuum ? "VACUUM" : "ANALYZE"; @@ -433,7 +436,7 @@ vacuum(VacuumStmt *vacstmt, Oid relid, Oid relid = lfirst_oid(cur); if (vacstmt->vacuum) - vacuum_rel(relid, vacstmt, RELKIND_RELATION, for_wraparound); + vacuum_rel(relid, vacstmt, do_toast, for_wraparound); if (vacstmt->analyze) { @@ -975,8 +978,7 @@ vac_truncate_clog(TransactionId frozenXID) * At entry and exit, we are not inside a transaction. */ static void -vacuum_rel(Oid relid, VacuumStmt *vacstmt, char expected_relkind, - bool for_wraparound) +vacuum_rel(Oid relid, VacuumStmt *vacstmt, bool do_toast, bool for_wraparound) { LOCKMODE lmode; Relation onerel; @@ -1013,8 +1015,8 @@ vacuum_rel(Oid relid, VacuumStmt *vacstmt, char expected_relkind, * by autovacuum; it's used to avoid cancelling a vacuum that was * invoked in an emergency. * - * Note: this flag remains set until CommitTransaction or - * AbortTransaction. We don't want to clear it until we reset + * Note: these flags remain set until CommitTransaction or + * AbortTransaction. We don't want to clear them until we reset * MyProc->xid/xmin, else OldestXmin might appear to go backwards, * which is probably Not Good. */ @@ -1087,10 +1089,11 @@ vacuum_rel(Oid relid, VacuumStmt *vacstmt, char expected_relkind, } /* - * Check that it's a plain table; we used to do this in get_rel_oids() but - * seems safer to check after we've locked the relation. + * Check that it's a vacuumable table; we used to do this in get_rel_oids() + * but seems safer to check after we've locked the relation. */ - if (onerel->rd_rel->relkind != expected_relkind) + if (onerel->rd_rel->relkind != RELKIND_RELATION && + onerel->rd_rel->relkind != RELKIND_TOASTVALUE) { ereport(WARNING, (errmsg("skipping \"%s\" --- cannot vacuum indexes, views, or special system tables", @@ -1132,9 +1135,13 @@ vacuum_rel(Oid relid, VacuumStmt *vacstmt, char expected_relkind, LockRelationIdForSession(&onerelid, lmode); /* - * Remember the relation's TOAST relation for later + * Remember the relation's TOAST relation for later, if the caller asked + * us to process it. */ - toast_relid = onerel->rd_rel->reltoastrelid; + if (do_toast) + toast_relid = onerel->rd_rel->reltoastrelid; + else + toast_relid = InvalidOid; /* * Switch to the table owner's userid, so that any index functions are @@ -1173,7 +1180,7 @@ vacuum_rel(Oid relid, VacuumStmt *vacstmt, char expected_relkind, * totally unimportant for toast relations. */ if (toast_relid != InvalidOid) - vacuum_rel(toast_relid, vacstmt, RELKIND_TOASTVALUE, for_wraparound); + vacuum_rel(toast_relid, vacstmt, false, for_wraparound); /* * Now release the session-level lock on the master table. diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 7d37de2256..9f218d6bdc 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -55,7 +55,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/postmaster/autovacuum.c,v 1.83 2008/07/23 20:20:10 alvherre Exp $ + * $PostgreSQL: pgsql/src/backend/postmaster/autovacuum.c,v 1.84 2008/08/13 00:07:50 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -93,6 +93,7 @@ #include "storage/procarray.h" #include "storage/sinvaladt.h" #include "tcop/tcopprot.h" +#include "utils/dynahash.h" #include "utils/flatfiles.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" @@ -161,8 +162,8 @@ typedef struct avw_dbase /* struct to keep track of tables to vacuum and/or analyze, in 1st pass */ typedef struct av_relation { + Oid ar_toastrelid; /* hash key - must be first */ Oid ar_relid; - Oid ar_toastrelid; } av_relation; /* struct to keep track of tables to vacuum and/or analyze, after rechecking */ @@ -279,7 +280,7 @@ static void autovac_balance_cost(void); static void do_autovacuum(void); static void FreeWorkerInfo(int code, Datum arg); -static autovac_table *table_recheck_autovac(Oid relid); +static autovac_table *table_recheck_autovac(Oid relid, HTAB *table_toast_map); static void relation_needs_vacanalyze(Oid relid, Form_pg_autovacuum avForm, Form_pg_class classForm, PgStat_StatTabEntry *tabentry, bool *dovacuum, @@ -287,7 +288,8 @@ static void relation_needs_vacanalyze(Oid relid, Form_pg_autovacuum avForm, static void autovacuum_do_vac_analyze(autovac_table *tab, BufferAccessStrategy bstrategy); -static HeapTuple get_pg_autovacuum_tuple_relid(Relation avRel, Oid relid); +static HeapTuple get_pg_autovacuum_tuple_relid(Relation avRel, Oid relid, + HTAB *table_toast_map); static PgStat_StatTabEntry *get_pgstat_tabentry_relid(Oid relid, bool isshared, PgStat_StatDBEntry *shared, PgStat_StatDBEntry *dbentry); @@ -1821,12 +1823,13 @@ do_autovacuum(void) HeapScanDesc relScan; Form_pg_database dbForm; List *table_oids = NIL; - List *toast_oids = NIL; - List *table_toast_list = NIL; + HASHCTL ctl; + HTAB *table_toast_map; ListCell *volatile cell; PgStat_StatDBEntry *shared; PgStat_StatDBEntry *dbentry; BufferAccessStrategy bstrategy; + ScanKeyData key; /* * StartTransactionCommand and CommitTransactionCommand will automatically @@ -1884,25 +1887,42 @@ do_autovacuum(void) classRel = heap_open(RelationRelationId, AccessShareLock); avRel = heap_open(AutovacuumRelationId, AccessShareLock); - /* - * Scan pg_class and determine which tables to vacuum. - * - * The stats subsystem collects stats for toast tables independently of - * the stats for their parent tables. We need to check those stats since - * in cases with short, wide tables there might be proportionally much - * more activity in the toast table than in its parent. - * - * Since we can only issue VACUUM against the parent table, we need to - * transpose a decision to vacuum a toast table into a decision to vacuum - * its parent. There's no point in considering ANALYZE on a toast table, - * either. To support this, we keep a list of OIDs of toast tables that - * need vacuuming alongside the list of regular tables. Regular tables - * will be entered into the table list even if they appear not to need - * vacuuming; we go back and re-mark them after finding all the vacuumable - * toast tables. - */ - relScan = heap_beginscan(classRel, SnapshotNow, 0, NULL); + /* create hash table for toast <-> main relid mapping */ + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(Oid) * 2; + ctl.hash = oid_hash; + table_toast_map = hash_create("TOAST to main relid map", + 100, + &ctl, + HASH_ELEM | HASH_FUNCTION); + + /* + * Scan pg_class to determine which tables to vacuum. + * + * We do this in two passes: on the first one we collect the list of + * plain relations, and on the second one we collect TOAST tables. + * The reason for doing the second pass is that during it we want to use + * the main relation's pg_autovacuum entry if the TOAST table does not have + * any, and we cannot obtain it unless we know beforehand what's the main + * table OID. + * + * We need to check TOAST tables separately because in cases with short, + * wide tables there might be proportionally much more activity in the + * TOAST table than in its parent. + */ + ScanKeyInit(&key, + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_RELATION)); + + relScan = heap_beginscan(classRel, SnapshotNow, 1, &key); + + /* + * On the first pass, we collect main tables to vacuum, and also the + * main table relid to TOAST relid mapping. + */ while ((tuple = heap_getnext(relScan, ForwardScanDirection)) != NULL) { Form_pg_class classForm = (Form_pg_class) GETSTRUCT(tuple); @@ -1915,15 +1935,10 @@ do_autovacuum(void) bool wraparound; int backendID; - /* Consider only regular and toast tables. */ - if (classForm->relkind != RELKIND_RELATION && - classForm->relkind != RELKIND_TOASTVALUE) - continue; - relid = HeapTupleGetOid(tuple); /* Fetch the pg_autovacuum tuple for the relation, if any */ - avTup = get_pg_autovacuum_tuple_relid(avRel, relid); + avTup = get_pg_autovacuum_tuple_relid(avRel, relid, NULL); if (HeapTupleIsValid(avTup)) avForm = (Form_pg_autovacuum) GETSTRUCT(avTup); @@ -1952,7 +1967,7 @@ do_autovacuum(void) * vacuum for wraparound, forcibly drop it. Otherwise just * log a complaint. */ - if (wraparound && classForm->relkind == RELKIND_RELATION) + if (wraparound) { ObjectAddress object; @@ -1976,65 +1991,88 @@ do_autovacuum(void) } } } - else if (classForm->relkind == RELKIND_RELATION) + else { /* Plain relations that need work are added to table_oids */ if (dovacuum || doanalyze) table_oids = lappend_oid(table_oids, relid); - else if (OidIsValid(classForm->reltoastrelid)) + + /* + * Remember the association for the second pass. Note: we must do + * this even if the table is going to be vacuumed, because we + * don't automatically vacuum toast tables along the parent table. + */ + if (OidIsValid(classForm->reltoastrelid)) { - /* - * If it doesn't appear to need vacuuming, but it has a toast - * table, remember the association to revisit below. - */ - av_relation *rel = palloc(sizeof(av_relation)); + av_relation *hentry; + bool found; - rel->ar_relid = relid; - rel->ar_toastrelid = classForm->reltoastrelid; + hentry = hash_search(table_toast_map, + &classForm->reltoastrelid, + HASH_ENTER, &found); - table_toast_list = lappend(table_toast_list, rel); + if (!found) + { + /* hash_search already filled in the key */ + hentry->ar_relid = relid; + } } } - else - { - /* TOAST relations that need vacuum are added to toast_oids */ - if (dovacuum) - toast_oids = lappend_oid(toast_oids, relid); - } if (HeapTupleIsValid(avTup)) heap_freetuple(avTup); } heap_endscan(relScan); - heap_close(avRel, AccessShareLock); - heap_close(classRel, AccessShareLock); - /* - * Add to the list of tables to vacuum, the OIDs of the tables that - * correspond to the saved OIDs of toast tables needing vacuum. - */ - foreach(cell, toast_oids) + /* second pass: check TOAST tables */ + ScanKeyInit(&key, + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_TOASTVALUE)); + + relScan = heap_beginscan(classRel, SnapshotNow, 1, &key); + while ((tuple = heap_getnext(relScan, ForwardScanDirection)) != NULL) { - Oid toastoid = lfirst_oid(cell); - ListCell *cell2; + Form_pg_class classForm = (Form_pg_class) GETSTRUCT(tuple); + Form_pg_autovacuum avForm = NULL; + PgStat_StatTabEntry *tabentry; + HeapTuple avTup; + Oid relid; + bool dovacuum; + bool doanalyze; + bool wraparound; - foreach(cell2, table_toast_list) - { - av_relation *ar = lfirst(cell2); + /* + * Skip temp tables (i.e. those in temp namespaces). We cannot safely + * process other backends' temp tables. + */ + if (isAnyTempNamespace(classForm->relnamespace)) + continue; - if (ar->ar_toastrelid == toastoid) - { - table_oids = lappend_oid(table_oids, ar->ar_relid); - break; - } - } + relid = HeapTupleGetOid(tuple); + + /* Fetch the pg_autovacuum tuple for this rel */ + avTup = get_pg_autovacuum_tuple_relid(avRel, relid, table_toast_map); + + if (HeapTupleIsValid(avTup)) + avForm = (Form_pg_autovacuum) GETSTRUCT(avTup); + + /* Fetch the pgstat entry for this table */ + tabentry = get_pgstat_tabentry_relid(relid, classForm->relisshared, + shared, dbentry); + + relation_needs_vacanalyze(relid, avForm, classForm, tabentry, + &dovacuum, &doanalyze, &wraparound); + + /* ignore analyze for toast tables */ + if (dovacuum) + table_oids = lappend_oid(table_oids, relid); } - list_free_deep(table_toast_list); - table_toast_list = NIL; - list_free(toast_oids); - toast_oids = NIL; + heap_endscan(relScan); + heap_close(avRel, AccessShareLock); + heap_close(classRel, AccessShareLock); /* * Create a buffer access strategy object for VACUUM to use. We want to @@ -2118,7 +2156,7 @@ do_autovacuum(void) * vacuumed in the last 500ms (PGSTAT_STAT_INTERVAL). This is a bug. */ MemoryContextSwitchTo(AutovacMemCxt); - tab = table_recheck_autovac(relid); + tab = table_recheck_autovac(relid, table_toast_map); if (tab == NULL) { /* someone else vacuumed the table */ @@ -2231,6 +2269,11 @@ deleted: LWLockRelease(AutovacuumLock); } + /* + * We leak table_toast_map here (among other things), but since we're going + * away soon, it's not a problem. + */ + /* * Update pg_database.datfrozenxid, and truncate pg_clog if possible. We * only need to do this once, not after each table. @@ -2244,9 +2287,14 @@ deleted: /* * Returns a copy of the pg_autovacuum tuple for the given relid, or NULL if * there isn't any. avRel is pg_autovacuum, already open and suitably locked. + * + * If table_toast_map is not null, use it to find an alternative OID with which + * to search a pg_autovacuum entry, if the passed relid does not yield one + * directly. */ static HeapTuple -get_pg_autovacuum_tuple_relid(Relation avRel, Oid relid) +get_pg_autovacuum_tuple_relid(Relation avRel, Oid relid, + HTAB *table_toast_map) { ScanKeyData entry[1]; SysScanDesc avScan; @@ -2267,6 +2315,18 @@ get_pg_autovacuum_tuple_relid(Relation avRel, Oid relid) systable_endscan(avScan); + if (!HeapTupleIsValid(avTup) && table_toast_map != NULL) + { + av_relation *hentry; + bool found; + + hentry = hash_search(table_toast_map, &relid, HASH_FIND, &found); + if (found) + /* avoid second recursion */ + avTup = get_pg_autovacuum_tuple_relid(avRel, hentry->ar_relid, + NULL); + } + return avTup; } @@ -2297,14 +2357,13 @@ get_pgstat_tabentry_relid(Oid relid, bool isshared, PgStat_StatDBEntry *shared, /* * table_recheck_autovac * - * Recheck whether a plain table still needs vacuum or analyze; be it because - * it does directly, or because its TOAST table does. Return value is a valid - * autovac_table pointer if it does, NULL otherwise. + * Recheck whether a table still needs vacuum or analyze. Return value is a + * valid autovac_table pointer if it does, NULL otherwise. * * Note that the returned autovac_table does not have the name fields set. */ static autovac_table * -table_recheck_autovac(Oid relid) +table_recheck_autovac(Oid relid, HTAB *table_toast_map) { Form_pg_autovacuum avForm = NULL; Form_pg_class classForm; @@ -2315,11 +2374,9 @@ table_recheck_autovac(Oid relid) bool doanalyze; autovac_table *tab = NULL; PgStat_StatTabEntry *tabentry; - bool doit = false; PgStat_StatDBEntry *shared; PgStat_StatDBEntry *dbentry; - bool wraparound, - toast_wraparound = false; + bool wraparound; /* use fresh stats */ autovac_refresh_stats(); @@ -2335,9 +2392,15 @@ table_recheck_autovac(Oid relid) return NULL; classForm = (Form_pg_class) GETSTRUCT(classTup); - /* fetch the pg_autovacuum entry, if any */ + /* + * Fetch the pg_autovacuum entry, if any. For a toast table, also try the + * main rel's pg_autovacuum entry if there isn't one for the TOAST table + * itself. + */ avRel = heap_open(AutovacuumRelationId, AccessShareLock); - avTup = get_pg_autovacuum_tuple_relid(avRel, relid); + avTup = get_pg_autovacuum_tuple_relid(avRel, relid, + classForm->relkind == RELKIND_TOASTVALUE ? table_toast_map : NULL); + if (HeapTupleIsValid(avTup)) avForm = (Form_pg_autovacuum) GETSTRUCT(avTup); @@ -2348,51 +2411,12 @@ table_recheck_autovac(Oid relid) relation_needs_vacanalyze(relid, avForm, classForm, tabentry, &dovacuum, &doanalyze, &wraparound); - /* OK, it needs vacuum by itself */ - if (dovacuum) - doit = true; - /* it doesn't need vacuum, but what about it's TOAST table? */ - else if (OidIsValid(classForm->reltoastrelid)) - { - Oid toastrelid = classForm->reltoastrelid; - HeapTuple toastClassTup; + /* ignore ANALYZE for toast tables */ + if (classForm->relkind == RELKIND_TOASTVALUE) + doanalyze = false; - toastClassTup = SearchSysCacheCopy(RELOID, - ObjectIdGetDatum(toastrelid), - 0, 0, 0); - if (HeapTupleIsValid(toastClassTup)) - { - bool toast_dovacuum; - bool toast_doanalyze; - bool toast_wraparound; - Form_pg_class toastClassForm; - PgStat_StatTabEntry *toasttabentry; - - toastClassForm = (Form_pg_class) GETSTRUCT(toastClassTup); - toasttabentry = get_pgstat_tabentry_relid(toastrelid, - toastClassForm->relisshared, - shared, dbentry); - - /* note we use the pg_autovacuum entry for the main table */ - relation_needs_vacanalyze(toastrelid, avForm, - toastClassForm, toasttabentry, - &toast_dovacuum, &toast_doanalyze, - &toast_wraparound); - /* we only consider VACUUM for toast tables */ - if (toast_dovacuum) - { - dovacuum = true; - doit = true; - } - - heap_freetuple(toastClassTup); - } - } - - if (doanalyze) - doit = true; - - if (doit) + /* OK, it needs something done */ + if (doanalyze || dovacuum) { int freeze_min_age; int vac_cost_limit; @@ -2439,7 +2463,7 @@ table_recheck_autovac(Oid relid) tab->at_freeze_min_age = freeze_min_age; tab->at_vacuum_cost_limit = vac_cost_limit; tab->at_vacuum_cost_delay = vac_cost_delay; - tab->at_wraparound = wraparound || toast_wraparound; + tab->at_wraparound = wraparound; tab->at_relname = NULL; tab->at_nspname = NULL; tab->at_datname = NULL; @@ -2633,7 +2657,7 @@ autovacuum_do_vac_analyze(autovac_table *tab, /* Let pgstat know what we're doing */ autovac_report_activity(tab); - vacuum(&vacstmt, tab->at_relid, bstrategy, tab->at_wraparound, true); + vacuum(&vacstmt, tab->at_relid, false, bstrategy, tab->at_wraparound, true); } /* diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 3b8d5249f5..1e419a7265 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -10,7 +10,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/tcop/utility.c,v 1.295 2008/07/18 20:26:06 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/tcop/utility.c,v 1.296 2008/08/13 00:07:50 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -836,7 +836,7 @@ ProcessUtility(Node *parsetree, break; case T_VacuumStmt: - vacuum((VacuumStmt *) parsetree, InvalidOid, NULL, false, + vacuum((VacuumStmt *) parsetree, InvalidOid, true, NULL, false, isTopLevel); break; diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index a5a3d1a60e..d45c3d0c96 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2008, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/commands/vacuum.h,v 1.79 2008/07/01 10:33:09 heikki Exp $ + * $PostgreSQL: pgsql/src/include/commands/vacuum.h,v 1.80 2008/08/13 00:07:50 alvherre Exp $ * *------------------------------------------------------------------------- */ @@ -125,7 +125,7 @@ extern int vacuum_freeze_min_age; /* in commands/vacuum.c */ -extern void vacuum(VacuumStmt *vacstmt, Oid relid, +extern void vacuum(VacuumStmt *vacstmt, Oid relid, bool do_toast, BufferAccessStrategy bstrategy, bool for_wraparound, bool isTopLevel); extern void vac_open_indexes(Relation relation, LOCKMODE lockmode, int *nindexes, Relation **Irel);