From 29094193f526bf90671d71b59a2e007aad1fcae5 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 14 Jul 2005 05:13:45 +0000 Subject: [PATCH] Integrate autovacuum functionality into the backend. There's still a few loose ends to be dealt with, but it seems to work. Alvaro Herrera, based on the contrib code by Matthew O'Connor. --- doc/src/sgml/catalogs.sgml | 105 ++- doc/src/sgml/runtime.sgml | 125 ++- src/backend/catalog/Makefile | 4 +- src/backend/commands/analyze.c | 55 +- src/backend/commands/vacuum.c | 37 +- src/backend/commands/vacuumlazy.c | 7 +- src/backend/postmaster/Makefile | 5 +- src/backend/postmaster/autovacuum.c | 750 ++++++++++++++++++ src/backend/postmaster/pgstat.c | 281 ++++++- src/backend/postmaster/postmaster.c | 98 ++- src/backend/tcop/postgres.c | 4 +- src/backend/tcop/utility.c | 4 +- src/backend/utils/init/miscinit.c | 5 +- src/backend/utils/init/postinit.c | 16 +- src/backend/utils/misc/guc.c | 56 +- src/backend/utils/misc/postgresql.conf.sample | 12 + src/include/catalog/catversion.h | 4 +- src/include/catalog/indexing.h | 4 +- src/include/catalog/pg_autovacuum.h | 60 ++ src/include/commands/vacuum.h | 4 +- src/include/pgstat.h | 85 +- src/include/postmaster/autovacuum.h | 38 + src/include/tcop/tcopprot.h | 3 +- src/include/utils/guc_tables.h | 3 +- src/test/regress/expected/sanity_check.out | 3 +- 25 files changed, 1678 insertions(+), 90 deletions(-) create mode 100644 src/backend/postmaster/autovacuum.c create mode 100644 src/include/catalog/pg_autovacuum.h create mode 100644 src/include/postmaster/autovacuum.h diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 560a6b3f35..fa0abca950 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1,6 +1,6 @@ @@ -88,6 +88,11 @@ authorization identifier membership relationships + + pg_autovacuum + per-relation autovacuum configuration parameters + + pg_cast casts (data type conversions) @@ -1102,6 +1107,104 @@ + + <structname>pg_autovacuum</structname> + + + pg_autovacuum + + + + The catalog pg_autovacuum stores optional + per-relation configuration parameters for autovacuum. + If there is an entry here for a particular relation, the given + parameters will be used for autovacuuming that table. If no entry + is present, the system-wide defaults will be used. + + + + <structname>pg_autovacuum</> Columns + + + + + Name + Type + References + Description + + + + + + vacrelid + oid + pg_class.oid + The table this entry is for + + + + enabled + bool + + If false, this table is never autovacuumed + + + + vac_base_thresh + integer + + Minimum number of modified tuples before vacuum + + + + vac_scale_factor + float4 + + Multiplier for reltuples to add to + vac_base_thresh + + + + anl_base_thresh + integer + + Minimum number of modified tuples before analyze + + + + anl_scale_factor + float4 + + Multiplier for reltuples to add to + anl_base_thresh + + + +
+ + + The autovacuum daemon will initiate a VACUUM operation + on a particular table when the number of updated or deleted tuples + exceeds vac_base_thresh plus + vac_scale_factor times the number of + live tuples currently estimated to be in the relation. + Similarly, it will initiate an ANALYZE operation + when the number of inserted, updated or deleted tuples + exceeds anl_base_thresh plus + anl_scale_factor times the number of + live tuples currently estimated to be in the relation. + + + + Any of the numerical fields can contain -1 (or indeed + any negative value) to indicate that the system-wide default should + be used for this particular value. + + + + + <structname>pg_cast</structname> diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 1aed577b08..4cae3fa894 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1,5 +1,5 @@ @@ -3173,7 +3173,7 @@ archive_command = 'copy "%p" /mnt/server/archivedir/"%f"' # Windows If on, collected statistics are zeroed out whenever the server is restarted. If off, statistics are accumulated across server restarts. The default is on. This option can only - be set at server start. + be set at server start. @@ -3182,6 +3182,127 @@ archive_command = 'copy "%p" /mnt/server/archivedir/"%f"' # Windows + + Automatic Vacuuming + + + Beginning in PostgreSQL 8.1, there is an optional server + process called the autovacuum daemon, whose purpose is + to automate the issuance of periodic VACUUM and + ANALYZE commands. When enabled, the autovacuum daemon + runs periodically and checks for tables that have had a large number + of updated or deleted tuples. This check uses the row-level statistics + collection facility; therefore, the autovacuum daemon cannot be used + unless and + are set TRUE. Also, it's + important to allow a slot for the autovacuum process when choosing + the value of . + + + + + + autovacuum (boolean) + + autovacuum configuration parameter + + + + Controls whether the server should start the + autovacuum subprocess. This is off by default. + This option can only be set at server start or in the + postgresql.conf file. + + + + + + autovacuum_naptime (integer) + + autovacuum_naptime configuration parameter + + + + Specifies the delay between activity rounds for the autovacuum + subprocess. In each round the subprocess examines one database + and issues VACUUM and ANALYZE commands + as needed for tables in that database. The delay is measured + in seconds, and the default is 60. + This option can only be set at server start or in the + postgresql.conf file. + + + + + + autovacuum_vacuum_threshold (integer) + + autovacuum_vacuum_threshold configuration parameter + + + + Specifies the minimum number of updated or deleted tuples needed + to trigger a VACUUM in any one table. + The default is 1000. + This option can only be set at server start or in the + postgresql.conf file. + + + + + + autovacuum_analyze_threshold (integer) + + autovacuum_analyze_threshold configuration parameter + + + + Specifies the minimum number of inserted, updated or deleted tuples + needed to trigger an ANALYZE in any one table. + The default is 500. + This option can only be set at server start or in the + postgresql.conf file. + + + + + + autovacuum_vacuum_scale_factor (floating point) + + autovacuum_vacuum_scale_factor configuration parameter + + + + Specifies a fraction of the table size to add to + autovacuum_vacuum_threshold + when deciding whether to trigger a VACUUM. + The default is 0.4. + This option can only be set at server start or in the + postgresql.conf file. + + + + + + autovacuum_analyze_scale_factor (floating point) + + autovacuum_analyze_scale_factor configuration parameter + + + + Specifies a fraction of the table size to add to + autovacuum_analyze_threshold + when deciding whether to trigger an ANALYZE. + The default is 0.2. + This option can only be set at server start or in the + postgresql.conf file. + + + + + + + Client Connection Defaults diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index fdcb2ff5a5..c2dd227d1d 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -2,7 +2,7 @@ # # Makefile for backend/catalog # -# $PostgreSQL: pgsql/src/backend/catalog/Makefile,v 1.55 2005/07/07 20:39:57 tgl Exp $ +# $PostgreSQL: pgsql/src/backend/catalog/Makefile,v 1.56 2005/07/14 05:13:39 tgl Exp $ # #------------------------------------------------------------------------- @@ -27,7 +27,7 @@ SUBSYS.o: $(OBJS) # indexing.h had better be last. POSTGRES_BKI_SRCS := $(addprefix $(top_srcdir)/src/include/catalog/,\ - pg_proc.h pg_type.h pg_attribute.h pg_class.h \ + pg_proc.h pg_type.h pg_attribute.h pg_class.h pg_autovacuum.h \ pg_attrdef.h pg_constraint.h pg_inherits.h pg_index.h \ pg_operator.h pg_opclass.h pg_am.h pg_amop.h pg_amproc.h \ pg_language.h pg_largeobject.h pg_aggregate.h pg_statistic.h \ diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 9642fa9cf5..03783f121e 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/analyze.c,v 1.86 2005/05/06 17:24:53 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/commands/analyze.c,v 1.87 2005/07/14 05:13:39 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -29,6 +29,7 @@ #include "parser/parse_expr.h" #include "parser/parse_oper.h" #include "parser/parse_relation.h" +#include "pgstat.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/datum.h" @@ -77,7 +78,7 @@ static void compute_index_stats(Relation onerel, double totalrows, MemoryContext col_context); static VacAttrStats *examine_attribute(Relation onerel, int attnum); static int acquire_sample_rows(Relation onerel, HeapTuple *rows, - int targrows, double *totalrows); + int targrows, double *totalrows, double *totaldeadrows); static double random_fract(void); static double init_selection_state(int n); static double get_next_S(double t, int n, double *stateptr); @@ -108,7 +109,8 @@ analyze_rel(Oid relid, VacuumStmt *vacstmt) AnlIndexData *indexdata; int targrows, numrows; - double totalrows; + double totalrows, + totaldeadrows; HeapTuple *rows; if (vacstmt->verbose) @@ -309,6 +311,14 @@ analyze_rel(Oid relid, VacuumStmt *vacstmt) */ if (attr_cnt <= 0 && !analyzableindex) { + /* + * We report that the table is empty; this is just so that the + * autovacuum code doesn't go nuts trying to get stats about + * a zero-column table. + */ + if (!vacstmt->vacuum) + pgstat_report_analyze(RelationGetRelid(onerel), 0, 0); + vac_close_indexes(nindexes, Irel, AccessShareLock); relation_close(onerel, AccessShareLock); return; @@ -340,7 +350,8 @@ analyze_rel(Oid relid, VacuumStmt *vacstmt) * Acquire the sample rows */ rows = (HeapTuple *) palloc(targrows * sizeof(HeapTuple)); - numrows = acquire_sample_rows(onerel, rows, targrows, &totalrows); + numrows = acquire_sample_rows(onerel, rows, targrows, + &totalrows, &totaldeadrows); /* * Compute the statistics. Temporary results during the calculations @@ -423,6 +434,10 @@ analyze_rel(Oid relid, VacuumStmt *vacstmt) totalindexrows, false); } + + /* report results to the stats collector, too */ + pgstat_report_analyze(RelationGetRelid(onerel), totalrows, + totaldeadrows); } /* Done with indexes */ @@ -752,23 +767,25 @@ BlockSampler_Next(BlockSampler bs) * the number of different blocks represented by the sample tends to be * too small. We can live with that for now. Improvements are welcome. * - * We also estimate the total number of rows in the table, and return that - * into *totalrows. An important property of this sampling method is that - * because we do look at a statistically unbiased set of blocks, we should - * get an unbiased estimate of the average number of live rows per block. - * The previous sampling method put too much credence in the row density near - * the start of the table. + * We also estimate the total numbers of live and dead rows in the table, + * and return them into *totalrows and *totaldeadrows, respectively. + * + * An important property of this sampling method is that because we do + * look at a statistically unbiased set of blocks, we should get + * unbiased estimates of the average numbers of live and dead rows per + * block. The previous sampling method put too much credence in the row + * density near the start of the table. * * The returned list of tuples is in order by physical position in the table. * (We will rely on this later to derive correlation estimates.) */ static int acquire_sample_rows(Relation onerel, HeapTuple *rows, int targrows, - double *totalrows) + double *totalrows, double *totaldeadrows) { int numrows = 0; /* # rows collected */ double liverows = 0; /* # rows seen */ - double deadrows = 0; + double deadrows = 0; /* # dead rows seen */ double rowstoskip = -1; /* -1 means not set yet */ BlockNumber totalblocks; BlockSamplerData bs; @@ -864,11 +881,7 @@ acquire_sample_rows(Relation onerel, HeapTuple *rows, int targrows, } else { - /* - * Count dead rows, but not empty slots. This information - * is currently not used, but it seems likely we'll want - * it someday. - */ + /* Count dead rows, but not empty slots */ if (targtuple.t_data != NULL) deadrows += 1; } @@ -890,12 +903,18 @@ acquire_sample_rows(Relation onerel, HeapTuple *rows, int targrows, qsort((void *) rows, numrows, sizeof(HeapTuple), compare_rows); /* - * Estimate total number of live rows in relation. + * Estimate total numbers of rows in relation. */ if (bs.m > 0) + { *totalrows = floor((liverows * totalblocks) / bs.m + 0.5); + *totaldeadrows = floor((deadrows * totalblocks) / bs.m + 0.5); + } else + { *totalrows = 0.0; + *totaldeadrows = 0.0; + } /* * Emit some interesting relation info diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index f81cdc0daf..23b0911e8c 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -13,7 +13,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.310 2005/06/14 22:15:32 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.311 2005/07/14 05:13:39 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -166,7 +166,8 @@ static TransactionId FreezeLimit; /* non-export function prototypes */ -static List *get_rel_oids(const RangeVar *vacrel, const char *stmttype); +static List *get_rel_oids(List *relids, const RangeVar *vacrel, + const char *stmttype); static void vac_update_dbstats(Oid dbid, TransactionId vacuumXID, TransactionId frozenXID); @@ -221,9 +222,18 @@ static bool enough_space(VacPage vacpage, Size len); /* * Primary entry point for VACUUM and ANALYZE commands. + * + * relids is normally NIL; if it is not, then it provides the list of + * relation OIDs to be processed, and vacstmt->relation is ignored. + * (The non-NIL case is currently only used by autovacuum.) + * + * It is the caller's responsibility that both vacstmt and relids + * (if given) be allocated in a memory context that won't disappear + * at transaction commit. In fact this context must be QueryContext + * to avoid complaints from PreventTransactionChain. */ void -vacuum(VacuumStmt *vacstmt) +vacuum(VacuumStmt *vacstmt, List *relids) { const char *stmttype = vacstmt->vacuum ? "VACUUM" : "ANALYZE"; TransactionId initialOldestXmin = InvalidTransactionId; @@ -302,11 +312,14 @@ vacuum(VacuumStmt *vacstmt) ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); - /* Assume we are processing everything unless one table is mentioned */ - all_rels = (vacstmt->relation == NULL); + /* Remember whether we are processing everything in the DB */ + all_rels = (relids == NIL && vacstmt->relation == NULL); - /* Build list of relations to process (note this lives in vac_context) */ - relations = get_rel_oids(vacstmt->relation, stmttype); + /* + * Build list of relations to process, unless caller gave us one. + * (If we build one, we put it in vac_context for safekeeping.) + */ + relations = get_rel_oids(relids, vacstmt->relation, stmttype); if (vacstmt->vacuum && all_rels) { @@ -512,11 +525,15 @@ vacuum(VacuumStmt *vacstmt) * per-relation transactions. */ static List * -get_rel_oids(const RangeVar *vacrel, const char *stmttype) +get_rel_oids(List *relids, const RangeVar *vacrel, const char *stmttype) { List *oid_list = NIL; MemoryContext oldcontext; + /* List supplied by VACUUM's caller? */ + if (relids) + return relids; + if (vacrel) { /* Process a specific relation */ @@ -1146,6 +1163,10 @@ full_vacuum_rel(Relation onerel, VacuumStmt *vacstmt) /* update statistics in pg_class */ vac_update_relstats(RelationGetRelid(onerel), vacrelstats->rel_pages, vacrelstats->rel_tuples, vacrelstats->hasindex); + + /* report results to the stats collector, too */ + pgstat_report_vacuum(RelationGetRelid(onerel), vacstmt->analyze, + vacrelstats->rel_tuples); } diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c index 52207000ee..64cab8bbca 100644 --- a/src/backend/commands/vacuumlazy.c +++ b/src/backend/commands/vacuumlazy.c @@ -31,7 +31,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/vacuumlazy.c,v 1.54 2005/05/19 21:35:46 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/commands/vacuumlazy.c,v 1.55 2005/07/14 05:13:40 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -44,6 +44,7 @@ #include "access/xlog.h" #include "commands/vacuum.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/freespace.h" #include "storage/smgr.h" #include "utils/lsyscache.h" @@ -179,6 +180,10 @@ lazy_vacuum_rel(Relation onerel, VacuumStmt *vacstmt) vacrelstats->rel_pages, vacrelstats->rel_tuples, hasindex); + + /* report results to the stats collector, too */ + pgstat_report_vacuum(RelationGetRelid(onerel), vacstmt->analyze, + vacrelstats->rel_tuples); } diff --git a/src/backend/postmaster/Makefile b/src/backend/postmaster/Makefile index 054bb09060..9eed90863c 100644 --- a/src/backend/postmaster/Makefile +++ b/src/backend/postmaster/Makefile @@ -4,7 +4,7 @@ # Makefile for src/backend/postmaster # # IDENTIFICATION -# $PostgreSQL: pgsql/src/backend/postmaster/Makefile,v 1.20 2005/03/10 07:14:03 neilc Exp $ +# $PostgreSQL: pgsql/src/backend/postmaster/Makefile,v 1.21 2005/07/14 05:13:40 tgl Exp $ # #------------------------------------------------------------------------- @@ -12,7 +12,8 @@ subdir = src/backend/postmaster top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -OBJS = bgwriter.o fork_process.o pgarch.o pgstat.o postmaster.o syslogger.o +OBJS = bgwriter.o autovacuum.o pgarch.o pgstat.o postmaster.o syslogger.o \ + fork_process.o all: SUBSYS.o diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c new file mode 100644 index 0000000000..de6df77031 --- /dev/null +++ b/src/backend/postmaster/autovacuum.c @@ -0,0 +1,750 @@ +/*------------------------------------------------------------------------- + * + * autovacuum.c + * + * PostgreSQL Integrated Autovacuum Daemon + * + * + * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * $PostgreSQL: pgsql/src/backend/postmaster/autovacuum.c,v 1.1 2005/07/14 05:13:40 tgl Exp $ + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include +#include + +#include "access/genam.h" +#include "access/heapam.h" +#include "catalog/indexing.h" +#include "catalog/pg_autovacuum.h" +#include "catalog/pg_database.h" +#include "commands/vacuum.h" +#include "libpq/hba.h" +#include "libpq/pqsignal.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "postmaster/autovacuum.h" +#include "postmaster/fork_process.h" +#include "postmaster/postmaster.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/proc.h" +#include "storage/sinval.h" +#include "tcop/tcopprot.h" +#include "utils/flatfiles.h" +#include "utils/fmgroids.h" +#include "utils/memutils.h" +#include "utils/ps_status.h" +#include "utils/relcache.h" + + +/* + * GUC parameters + */ +bool autovacuum_start_daemon = false; +int autovacuum_naptime; +int autovacuum_vac_thresh; +double autovacuum_vac_scale; +int autovacuum_anl_thresh; +double autovacuum_anl_scale; + +/* Flag to tell if we are in the autovacuum daemon process */ +static bool am_autovacuum = false; + +/* Last time autovac daemon started/stopped (only valid in postmaster) */ +static time_t last_autovac_start_time = 0; +static time_t last_autovac_stop_time = 0; + +/* struct to keep list of candidate databases for vacuum */ +typedef struct autovac_dbase +{ + Oid oid; + char *name; + PgStat_StatDBEntry *entry; +} autovac_dbase; + + +#ifdef EXEC_BACKEND +static pid_t autovac_forkexec(void); +#endif +NON_EXEC_STATIC void AutoVacMain(int argc, char *argv[]); +static void autovac_check_wraparound(void); +static void do_autovacuum(PgStat_StatDBEntry *dbentry); +static List *autovac_get_database_list(void); +static void test_rel_for_autovac(Oid relid, PgStat_StatTabEntry *tabentry, + Form_pg_class classForm, Form_pg_autovacuum avForm, + List **vacuum_tables, List **analyze_tables); +static void autovacuum_do_vac_analyze(List *relids, bool dovacuum); + + +/* + * Main entry point for autovacuum controller process. + * + * This code is heavily based on pgarch.c, q.v. + */ +int +autovac_start(void) +{ + time_t curtime; + pid_t AutoVacPID; + + /* Do nothing if no autovacuum process needed */ + if (!AutoVacuumingActive()) + return 0; + + /* + * Do nothing if too soon since last autovacuum exit. This limits + * how often the daemon runs. Since the time per iteration can be + * quite variable, it seems more useful to measure/control the time + * since last subprocess exit than since last subprocess launch. + * + * However, we *also* check the time since last subprocess launch; + * this prevents thrashing under fork-failure conditions. + * + * Note that since we will be re-called from the postmaster main loop, + * we will get another chance later if we do nothing now. + * + * XXX todo: implement sleep scale factor that existed in contrib code. + */ + curtime = time(NULL); + if ((unsigned int) (curtime - last_autovac_stop_time) < + (unsigned int) autovacuum_naptime) + return 0; + + if ((unsigned int) (curtime - last_autovac_start_time) < + (unsigned int) autovacuum_naptime) + return 0; + + last_autovac_start_time = curtime; + +#ifdef EXEC_BACKEND + switch((AutoVacPID = autovac_forkexec())) +#else + switch((AutoVacPID = fork_process())) +#endif + { + case -1: + ereport(LOG, + (errmsg("could not fork autovacuum process: %m"))); + return 0; + +#ifndef EXEC_BACKEND + case 0: + /* in postmaster child ... */ + /* Close the postmaster's sockets */ + ClosePostmasterPorts(false); + + AutoVacMain(0, NULL); + break; +#endif + default: + return (int) AutoVacPID; + } + + /* shouldn't get here */ + return 0; +} + +/* + * autovac_stopped --- called by postmaster when subprocess exit is detected + */ +void +autovac_stopped(void) +{ + last_autovac_stop_time = time(NULL); +} + +#ifdef EXEC_BACKEND +/* + * autovac_forkexec() + * + * Format up the arglist for the autovacuum process, then fork and exec. + */ +static pid_t +autovac_forkexec(void) +{ + char *av[10]; + int ac = 0; + + av[ac++] = "postgres"; + av[ac++] = "-forkautovac"; + av[ac++] = NULL; /* filled in by postmaster_forkexec */ + av[ac] = NULL; + + Assert(ac < lengthof(av)); + + return postmaster_forkexec(ac, av); +} +#endif /* EXEC_BACKEND */ + +/* + * AutoVacMain + */ +NON_EXEC_STATIC void +AutoVacMain(int argc, char *argv[]) +{ + ListCell *cell; + List *dblist; + autovac_dbase *db; + sigjmp_buf local_sigjmp_buf; + + /* we are a postmaster subprocess now */ + IsUnderPostmaster = true; + am_autovacuum = true; + + /* reset MyProcPid */ + MyProcPid = getpid(); + + /* Lose the postmaster's on-exit routines */ + on_exit_reset(); + + /* + * Set up signal handlers. We operate on databases much like a + * regular backend, so we use the same signal handling. See + * equivalent code in tcop/postgres.c. + * + * Currently, we don't pay attention to postgresql.conf changes + * that happen during a single daemon iteration, so we can ignore + * SIGHUP. + */ + pqsignal(SIGHUP, SIG_IGN); + /* + * Presently, SIGINT will lead to autovacuum shutdown, because that's + * how we handle ereport(ERROR). It could be improved however. + */ + pqsignal(SIGINT, StatementCancelHandler); + pqsignal(SIGTERM, die); + pqsignal(SIGQUIT, quickdie); + pqsignal(SIGALRM, handle_sig_alarm); + + pqsignal(SIGPIPE, SIG_IGN); + pqsignal(SIGUSR1, CatchupInterruptHandler); + /* We don't listen for async notifies */ + pqsignal(SIGUSR2, SIG_IGN); + pqsignal(SIGCHLD, SIG_DFL); + + /* Identify myself via ps */ + init_ps_display("autovacuum process", "", ""); + set_ps_display(""); + + /* Early initialization */ + BaseInit(); + + /* + * If an exception is encountered, processing resumes here. + * + * See notes in postgres.c about the design of this coding. + */ + if (sigsetjmp(local_sigjmp_buf, 1) != 0) + { + /* Prevents interrupts while cleaning up */ + HOLD_INTERRUPTS(); + + /* Report the error to the server log */ + EmitErrorReport(); + + /* + * We can now go away. Note that because we'll call InitProcess, + * a callback will be registered to do ProcKill, which will clean + * up necessary state. + */ + proc_exit(0); + } + + /* We can now handle ereport(ERROR) */ + PG_exception_stack = &local_sigjmp_buf; + + PG_SETMASK(&UnBlockSig); + + /* Get a list of databases */ + dblist = autovac_get_database_list(); + + /* + * Choose a database to connect to. We pick the database that was least + * recently auto-vacuumed. + * + * XXX This could be improved if we had more info about whether it needs + * vacuuming before connecting to it. Perhaps look through the pgstats + * data for the database's tables? + * + * XXX it is NOT good that we totally ignore databases that have no + * pgstats entry ... + */ + db = NULL; + + foreach(cell, dblist) + { + autovac_dbase *tmp = lfirst(cell); + + tmp->entry = pgstat_fetch_stat_dbentry(tmp->oid); + if (!tmp->entry) + continue; + + /* + * Don't try to access a database that was dropped. This could only + * happen if we read the pg_database flat file right before it was + * modified, after the database was dropped from the pg_database + * table. + */ + if (tmp->entry->destroy != 0) + continue; + + if (!db || + tmp->entry->last_autovac_time < db->entry->last_autovac_time) + db = tmp; + } + + if (db) + { + /* + * Connect to the selected database + */ + InitPostgres(db->name, NULL); + SetProcessingMode(NormalProcessing); + pgstat_report_autovac(); + set_ps_display(db->name); + ereport(LOG, + (errmsg("autovacuum: processing database \"%s\"", db->name))); + /* + * And do an appropriate amount of work on it + */ + do_autovacuum(db->entry); + } + + /* One iteration done, go away */ + proc_exit(0); +} + +/* + * autovac_get_database_list + * + * Return a list of all databases. Note we cannot use pg_database, + * because we aren't connected yet; we use the flat database file. + */ +static List * +autovac_get_database_list(void) +{ + char *filename; + List *dblist = NIL; + char thisname[NAMEDATALEN]; + FILE *db_file; + Oid db_id; + Oid db_tablespace; + + filename = database_getflatfilename(); + db_file = AllocateFile(filename, "r"); + if (db_file == NULL) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", filename))); + + while (read_pg_database_line(db_file, thisname, &db_id, &db_tablespace)) + { + autovac_dbase *db; + + db = (autovac_dbase *) palloc(sizeof(autovac_dbase)); + + db->oid = db_id; + db->name = pstrdup(thisname); + /* this gets set later */ + db->entry = NULL; + + dblist = lappend(dblist, db); + } + + FreeFile(db_file); + pfree(filename); + + return dblist; +} + +/* + * Process a database. + * + * Note that test_rel_for_autovac generates two separate lists, one for + * vacuum and other for analyze. This is to facilitate processing all + * analyzes first, and then all vacuums. + * + * Note that CHECK_FOR_INTERRUPTS is supposed to be used in certain spots in + * order not to ignore shutdown commands for too long. + */ +static void +do_autovacuum(PgStat_StatDBEntry *dbentry) +{ + Relation classRel, + avRel; + HeapTuple tuple; + HeapScanDesc relScan; + List *vacuum_tables = NIL, + *analyze_tables = NIL; + MemoryContext AutovacMemCxt; + + /* Memory context where cross-transaction state is stored */ + AutovacMemCxt = AllocSetContextCreate(TopMemoryContext, + "Autovacuum context", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + + /* Start a transaction so our commands have one to play into. */ + StartTransactionCommand(); + + /* + * StartTransactionCommand and CommitTransactionCommand will + * automatically switch to other contexts. We need this one + * to keep the list of relations to vacuum/analyze across + * transactions. + */ + MemoryContextSwitchTo(AutovacMemCxt); + + /* + * If this database is old enough to need a whole-database VACUUM, + * don't bother checking each table. If that happens, this function + * will issue the VACUUM command and won't return. + */ + autovac_check_wraparound(); + + CHECK_FOR_INTERRUPTS(); + + classRel = heap_open(RelationRelationId, AccessShareLock); + avRel = heap_open(AutovacuumRelationId, AccessShareLock); + + relScan = heap_beginscan(classRel, SnapshotNow, 0, NULL); + + /* Scan pg_class looking for tables to vacuum */ + while ((tuple = heap_getnext(relScan, ForwardScanDirection)) != NULL) + { + Form_pg_class classForm = (Form_pg_class) GETSTRUCT(tuple); + Form_pg_autovacuum avForm = NULL; + PgStat_StatTabEntry *tabentry; + SysScanDesc avScan; + HeapTuple avTup; + ScanKeyData entry[1]; + Oid relid; + + /* Skip non-table entries. */ + /* XXX possibly allow RELKIND_TOASTVALUE entries here too? */ + if (classForm->relkind != RELKIND_RELATION) + continue; + + relid = HeapTupleGetOid(tuple); + + /* See if we have a pg_autovacuum entry for this relation. */ + ScanKeyInit(&entry[0], + Anum_pg_autovacuum_vacrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + + avScan = systable_beginscan(avRel, AutovacuumRelidIndexId, true, + SnapshotNow, 1, entry); + + avTup = systable_getnext(avScan); + + if (HeapTupleIsValid(avTup)) + avForm = (Form_pg_autovacuum) GETSTRUCT(avTup); + + tabentry = hash_search(dbentry->tables, &relid, + HASH_FIND, NULL); + + test_rel_for_autovac(relid, tabentry, classForm, avForm, + &vacuum_tables, &analyze_tables); + + systable_endscan(avScan); + } + + heap_endscan(relScan); + heap_close(avRel, AccessShareLock); + heap_close(classRel, AccessShareLock); + + CHECK_FOR_INTERRUPTS(); + + /* + * Perform operations on collected tables. + */ + + if (analyze_tables) + autovacuum_do_vac_analyze(analyze_tables, false); + + CHECK_FOR_INTERRUPTS(); + + /* get back to proper context */ + MemoryContextSwitchTo(AutovacMemCxt); + + if (vacuum_tables) + autovacuum_do_vac_analyze(vacuum_tables, true); + + /* Finally close out the last transaction. */ + CommitTransactionCommand(); +} + +/* + * test_rel_for_autovac + * + * Check whether a table needs to be vacuumed or analyzed. Add it to the + * respective list if so. + * + * A table needs to be vacuumed if the number of dead tuples exceeds a + * threshold. This threshold is calculated as + * + * threshold = vac_base_thresh + vac_scale_factor * reltuples + * + * For analyze, the analysis done is that the number of tuples inserted, + * deleted and updated since the last analyze exceeds a threshold calculated + * in the same fashion as above. Note that the collector actually stores + * the number of tuples (both live and dead) that there were as of the last + * analyze. This is asymmetric to the VACUUM case. + * + * A table whose pg_autovacuum.enabled value is false, is automatically + * skipped. Thus autovacuum can be disabled for specific tables. + * + * A table whose vac_base_thresh value is <0 takes the base value from the + * autovacuum_vacuum_threshold GUC variable. Similarly, a vac_scale_factor + * value <0 is substituted with the value of + * autovacuum_vacuum_scale_factor GUC variable. Ditto for analyze. + */ +static void +test_rel_for_autovac(Oid relid, PgStat_StatTabEntry *tabentry, + Form_pg_class classForm, + Form_pg_autovacuum avForm, + List **vacuum_tables, List **analyze_tables) +{ + Relation rel; + float4 reltuples; /* pg_class.reltuples */ + /* constants from pg_autovacuum or GUC variables */ + int vac_base_thresh, + anl_base_thresh; + float4 vac_scale_factor, + anl_scale_factor; + /* thresholds calculated from above constants */ + float4 vacthresh, + anlthresh; + /* number of vacuum (resp. analyze) tuples at this time */ + float4 vactuples, + anltuples; + + /* User disabled it in pg_autovacuum? */ + if (avForm && !avForm->enabled) + return; + + rel = RelationIdGetRelation(relid); + /* The table was recently dropped? */ + if (rel == NULL) + return; + + /* Not found in stat hash? */ + if (tabentry == NULL) + { + /* + * Analyze this table. It will emit a stat message for the + * collector that will initialize the entry for the next time + * around, so we won't have to guess again. + */ + elog(DEBUG2, "table %s not known to stat system, will ANALYZE", + RelationGetRelationName(rel)); + *analyze_tables = lappend_oid(*analyze_tables, relid); + RelationClose(rel); + return; + } + + reltuples = rel->rd_rel->reltuples; + vactuples = tabentry->n_dead_tuples; + anltuples = tabentry->n_live_tuples + tabentry->n_dead_tuples - + tabentry->last_anl_tuples; + + /* + * If there is a tuple in pg_autovacuum, use it; else, use the GUC + * defaults. Note that the fields may contain "-1" (or indeed any + * negative value), which means use the GUC defaults for each setting. + */ + if (avForm != NULL) + { + vac_scale_factor = (avForm->vac_scale_factor < 0) ? + autovacuum_vac_scale : avForm->vac_scale_factor; + vac_base_thresh = (avForm->vac_base_thresh < 0) ? + autovacuum_vac_thresh : avForm->vac_base_thresh; + + anl_scale_factor = (avForm->anl_scale_factor < 0) ? + autovacuum_anl_scale : avForm->anl_scale_factor; + anl_base_thresh = (avForm->anl_base_thresh < 0) ? + autovacuum_anl_thresh : avForm->anl_base_thresh; + } + else + { + vac_scale_factor = autovacuum_vac_scale; + vac_base_thresh = autovacuum_vac_thresh; + + anl_scale_factor = autovacuum_anl_scale; + anl_base_thresh = autovacuum_anl_thresh; + } + + vacthresh = (float4) vac_base_thresh + vac_scale_factor * reltuples; + anlthresh = (float4) anl_base_thresh + anl_scale_factor * reltuples; + + /* + * Note that we don't need to take special consideration for stat + * reset, because if that happens, the last vacuum and analyze counts + * will be reset too. + */ + + elog(DEBUG2, "%s: vac: %.0f (threshold %.0f), anl: %.0f (threshold %.0f)", + RelationGetRelationName(rel), + vactuples, vacthresh, anltuples, anlthresh); + + /* Determine if this table needs vacuum or analyze. */ + if (vactuples > vacthresh) + { + elog(DEBUG2, "will VACUUM ANALYZE %s", + RelationGetRelationName(rel)); + *vacuum_tables = lappend_oid(*vacuum_tables, relid); + } + else if (anltuples > anlthresh) + { + elog(DEBUG2, "will ANALYZE %s", + RelationGetRelationName(rel)); + *analyze_tables = lappend_oid(*analyze_tables, relid); + } + + RelationClose(rel); +} + +/* + * autovacuum_do_vac_analyze + * Vacuum or analyze a list of tables; or all tables if relids = NIL + * + * We must be in AutovacMemCxt when this routine is called. + */ +static void +autovacuum_do_vac_analyze(List *relids, bool dovacuum) +{ + VacuumStmt *vacstmt = makeNode(VacuumStmt); + + /* + * Point QueryContext to the autovac memory context to fake out the + * PreventTransactionChain check inside vacuum(). Note that this + * is also why we palloc vacstmt instead of just using a local variable. + */ + QueryContext = CurrentMemoryContext; + + /* Set up command parameters */ + vacstmt->vacuum = dovacuum; + vacstmt->full = false; + vacstmt->analyze = true; + vacstmt->freeze = false; + vacstmt->verbose = false; + vacstmt->relation = NULL; /* all tables, or not used if relids != NIL */ + vacstmt->va_cols = NIL; + + vacuum(vacstmt, relids); +} + +/* + * autovac_check_wraparound + * Check database Xid wraparound + * + * Check pg_database to see if the last database-wide VACUUM was too long ago, + * and issue one now if so. If this comes to pass, we do not return, as there + * is no point in checking individual tables -- they will all get vacuumed + * anyway. + */ +static void +autovac_check_wraparound(void) +{ + Relation relation; + ScanKeyData entry[1]; + HeapScanDesc scan; + HeapTuple tuple; + Form_pg_database dbform; + int32 age; + bool whole_db; + + relation = heap_open(DatabaseRelationId, AccessShareLock); + + /* Must use a heap scan, since there's no syscache for pg_database */ + ScanKeyInit(&entry[0], + ObjectIdAttributeNumber, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(MyDatabaseId)); + + scan = heap_beginscan(relation, SnapshotNow, 1, entry); + + tuple = heap_getnext(scan, ForwardScanDirection); + + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "could not find tuple for database %u", MyDatabaseId); + + dbform = (Form_pg_database) GETSTRUCT(tuple); + + /* + * We decide to vacuum at the same point where vacuum.c's + * vac_truncate_clog() would decide to start giving warnings. + */ + age = (int32) (GetTopTransactionId() - dbform->datfrozenxid); + whole_db = (age > (int32) ((MaxTransactionId >> 3) * 3)); + + heap_endscan(scan); + heap_close(relation, AccessShareLock); + + if (whole_db) + { + elog(LOG, "autovacuum: VACUUM ANALYZE whole database"); + autovacuum_do_vac_analyze(NIL, true); + proc_exit(0); + } +} + +/* + * AutoVacuumingActive + * Check GUC vars and report whether the autovacuum process should be + * running. + */ +bool +AutoVacuumingActive(void) +{ + if (!autovacuum_start_daemon || !pgstat_collect_startcollector || + !pgstat_collect_tuplelevel) + return false; + return true; +} + +/* + * autovac_init + * This is called at postmaster initialization. + * + * Annoy the user if he got it wrong. + */ +void +autovac_init(void) +{ + if (!autovacuum_start_daemon) + return; + + if (!pgstat_collect_startcollector || !pgstat_collect_tuplelevel) + { + ereport(WARNING, + (errmsg("autovacuum not started because of misconfiguration"), + errhint("Enable options \"stats_start_collector\" and \"stats_row_level\"."))); + /* + * Set the GUC var so we don't fork autovacuum uselessly, and also to + * help debugging. + */ + autovacuum_start_daemon = false; + } +} + +/* + * IsAutoVacuumProcess + * Return whether this process is an autovacuum process. + */ +bool +IsAutoVacuumProcess(void) +{ + return am_autovacuum; +} diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 325bb8b451..d6096fd2b8 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -13,7 +13,7 @@ * * Copyright (c) 2001-2005, PostgreSQL Global Development Group * - * $PostgreSQL: pgsql/src/backend/postmaster/pgstat.c,v 1.99 2005/07/04 04:51:47 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/postmaster/pgstat.c,v 1.100 2005/07/14 05:13:40 tgl Exp $ * ---------- */ #include "postgres.h" @@ -38,6 +38,7 @@ #include "libpq/pqsignal.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "postmaster/autovacuum.h" #include "postmaster/fork_process.h" #include "postmaster/postmaster.h" #include "storage/backendid.h" @@ -167,7 +168,7 @@ static void pgstat_read_statsfile(HTAB **dbhash, Oid onlydb, int *numbackends); static void backend_read_statsfile(void); -static void pgstat_setheader(PgStat_MsgHdr *hdr, int mtype); +static void pgstat_setheader(PgStat_MsgHdr *hdr, StatMsgType mtype); static void pgstat_send(void *msg, int len); static void pgstat_recv_bestart(PgStat_MsgBestart *msg, int len); @@ -177,6 +178,9 @@ static void pgstat_recv_tabstat(PgStat_MsgTabstat *msg, int len); static void pgstat_recv_tabpurge(PgStat_MsgTabpurge *msg, int len); static void pgstat_recv_dropdb(PgStat_MsgDropdb *msg, int len); static void pgstat_recv_resetcounter(PgStat_MsgResetcounter *msg, int len); +static void pgstat_recv_autovac(PgStat_MsgAutovacStart *msg, int len); +static void pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len); +static void pgstat_recv_analyze(PgStat_MsgAnalyze *msg, int len); /* ------------------------------------------------------------ @@ -618,6 +622,27 @@ pgstat_beterm(int pid) } +/* ---------- + * pgstat_report_autovac() - + * + * Called from autovacuum.c to report startup of an autovacuum process. + * ---------- + */ +void +pgstat_report_autovac(void) +{ + PgStat_MsgAutovacStart msg; + + if (pgStatSock < 0) + return; + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_AUTOVAC_START); + msg.m_databaseid = MyDatabaseId; + msg.m_start_time = GetCurrentTimestamp(); + + pgstat_send(&msg, sizeof(msg)); +} + /* ------------------------------------------------------------ * Public functions used by backends follow *------------------------------------------------------------ @@ -652,6 +677,51 @@ pgstat_bestart(void) on_proc_exit(pgstat_beshutdown_hook, 0); } +/* --------- + * pgstat_report_vacuum() - + * + * Tell the collector about the table we just vacuumed. + * --------- + */ +void +pgstat_report_vacuum(Oid tableoid, bool analyze, PgStat_Counter tuples) +{ + PgStat_MsgVacuum msg; + + if (pgStatSock < 0) + return; + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_VACUUM); + msg.m_databaseid = MyDatabaseId; + msg.m_tableoid = tableoid; + msg.m_analyze = analyze; + msg.m_tuples = tuples; + pgstat_send(&msg, sizeof(msg)); +} + +/* -------- + * pgstat_report_analyze() - + * + * Tell the collector about the table we just analyzed. + * -------- + */ +void +pgstat_report_analyze(Oid tableoid, PgStat_Counter livetuples, + PgStat_Counter deadtuples) +{ + PgStat_MsgAnalyze msg; + + if (pgStatSock < 0) + return; + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_ANALYZE); + msg.m_databaseid = MyDatabaseId; + msg.m_tableoid = tableoid; + msg.m_live_tuples = livetuples; + msg.m_dead_tuples = deadtuples; + pgstat_send(&msg, sizeof(msg)); +} + /* * Flush any remaining statistics counts out to the collector at process * exit. Without this, operations triggered during backend exit (such as @@ -1279,7 +1349,7 @@ pgstat_fetch_stat_numbackends(void) * ---------- */ static void -pgstat_setheader(PgStat_MsgHdr *hdr, int mtype) +pgstat_setheader(PgStat_MsgHdr *hdr, StatMsgType mtype) { hdr->m_type = mtype; hdr->m_backendid = MyBackendId; @@ -1653,6 +1723,18 @@ PgstatCollectorMain(int argc, char *argv[]) nread); break; + case PGSTAT_MTYPE_AUTOVAC_START: + pgstat_recv_autovac((PgStat_MsgAutovacStart *) &msg, nread); + break; + + case PGSTAT_MTYPE_VACUUM: + pgstat_recv_vacuum((PgStat_MsgVacuum *) &msg, nread); + break; + + case PGSTAT_MTYPE_ANALYZE: + pgstat_recv_analyze((PgStat_MsgAnalyze *) &msg, nread); + break; + default: break; } @@ -2049,6 +2131,7 @@ pgstat_get_db_entry(Oid databaseid) result->n_blocks_fetched = 0; result->n_blocks_hit = 0; result->destroy = 0; + result->last_autovac_time = 0; memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = sizeof(Oid); @@ -2136,6 +2219,7 @@ pgstat_write_statsfile(void) PgStat_StatBeDead *deadbe; FILE *fpout; int i; + int32 format_id; /* * Open the statistics temp file to write out the current values. @@ -2150,6 +2234,12 @@ pgstat_write_statsfile(void) return; } + /* + * Write the file header --- currently just a format ID. + */ + format_id = PGSTAT_FILE_FORMAT_ID; + fwrite(&format_id, sizeof(format_id), 1, fpout); + /* * Walk through the database table. */ @@ -2182,7 +2272,7 @@ pgstat_write_statsfile(void) } /* - * Write out the DB line including the number of life backends. + * Write out the DB line including the number of live backends. */ fputc('D', fpout); fwrite(dbentry, sizeof(PgStat_StatDBEntry), 1, fpout); @@ -2216,7 +2306,7 @@ pgstat_write_statsfile(void) } /* - * At least we think this is still a life table. Print it's + * At least we think this is still a live table. Print its * access stats. */ fputc('T', fpout); @@ -2312,6 +2402,7 @@ pgstat_read_statsfile(HTAB **dbhash, Oid onlydb, HASHCTL hash_ctl; HTAB *tabhash = NULL; FILE *fpin; + int32 format_id; int maxbackends = 0; int havebackends = 0; bool found; @@ -2319,12 +2410,13 @@ pgstat_read_statsfile(HTAB **dbhash, Oid onlydb, int mcxt_flags; /* - * If running in the collector we use the DynaHashCxt memory context. - * If running in a backend, we use the TopTransactionContext instead, - * so the caller must only know the last XactId when this call - * happened to know if his tables are still valid or already gone! + * If running in the collector or the autovacuum process, we use the + * DynaHashCxt memory context. If running in a backend, we use the + * TopTransactionContext instead, so the caller must only know the last + * XactId when this call happened to know if his tables are still valid or + * already gone! */ - if (pgStatRunningInCollector) + if (pgStatRunningInCollector || IsAutoVacuumProcess()) { use_mcxt = NULL; mcxt_flags = 0; @@ -2363,6 +2455,17 @@ pgstat_read_statsfile(HTAB **dbhash, Oid onlydb, if ((fpin = AllocateFile(PGSTAT_STAT_FILENAME, PG_BINARY_R)) == NULL) return; + /* + * Verify it's of the expected format. + */ + if (fread(&format_id, 1, sizeof(format_id), fpin) != sizeof(format_id) + || format_id != PGSTAT_FILE_FORMAT_ID) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted pgstat.stat file"))); + goto done; + } + /* * We found an existing collector stats file. Read it and put all the * hashtable entries into place. @@ -2552,18 +2655,35 @@ done: * * Because we store the hash tables in TopTransactionContext, the result * is good for the entire current main transaction. + * + * Inside the autovacuum process, the statfile is assumed to be valid + * "forever", that is one iteration, within one database. This means + * we only consider the statistics as they were when the autovacuum + * iteration started. */ static void backend_read_statsfile(void) { - TransactionId topXid = GetTopTransactionId(); - - if (!TransactionIdEquals(pgStatDBHashXact, topXid)) + if (IsAutoVacuumProcess()) { Assert(!pgStatRunningInCollector); - pgstat_read_statsfile(&pgStatDBHash, MyDatabaseId, + /* already read it? */ + if (pgStatDBHash) + return; + pgstat_read_statsfile(&pgStatDBHash, InvalidOid, &pgStatBeTable, &pgStatNumBackends); - pgStatDBHashXact = topXid; + } + else + { + TransactionId topXid = GetTopTransactionId(); + + if (!TransactionIdEquals(pgStatDBHashXact, topXid)) + { + Assert(!pgStatRunningInCollector); + pgstat_read_statsfile(&pgStatDBHash, MyDatabaseId, + &pgStatBeTable, &pgStatNumBackends); + pgStatDBHashXact = topXid; + } } } @@ -2606,6 +2726,129 @@ pgstat_recv_beterm(PgStat_MsgBeterm *msg, int len) pgstat_sub_backend(msg->m_hdr.m_procpid); } +/* ---------- + * pgstat_recv_autovac() - + * + * Process an autovacuum signalling message. + * ---------- + */ +static void +pgstat_recv_autovac(PgStat_MsgAutovacStart *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + + /* + * Lookup the database in the hashtable. + * + * XXX this creates the entry if it doesn't exist. Is this a problem? (We + * could leak an entry if we send an autovac message and the database is + * later destroyed, _and_ the messages are rearranged. Doesn't seem very + * likely though.) Not sure what to do about it. + */ + dbentry = pgstat_get_db_entry(msg->m_databaseid); + + /* + * Store the last autovacuum time in the database entry. + */ + dbentry->last_autovac_time = msg->m_start_time; +} + +/* ---------- + * pgstat_recv_vacuum() - + * + * Process a VACUUM message. + * ---------- + */ +static void +pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatTabEntry *tabentry; + bool found; + + dbentry = pgstat_get_db_entry(msg->m_databaseid); + + tabentry = hash_search(dbentry->tables, &(msg->m_tableoid), + HASH_ENTER, &found); + + /* + * If we are creating the entry, initialize it. + */ + if (!found) + { + tabentry->tableid = msg->m_tableoid; + + tabentry->tuples_returned = 0; + tabentry->tuples_fetched = 0; + tabentry->tuples_inserted = msg->m_tuples; + tabentry->tuples_deleted = 0; + tabentry->tuples_updated = 0; + + tabentry->n_live_tuples = msg->m_tuples; + tabentry->n_dead_tuples = 0; + + if (msg->m_analyze) + tabentry->last_anl_tuples = msg->m_tuples; + else + tabentry->last_anl_tuples = 0; + + tabentry->blocks_fetched = 0; + tabentry->blocks_hit = 0; + } + else + { + tabentry->n_dead_tuples = 0; + tabentry->n_live_tuples = msg->m_tuples; + if (msg->m_analyze) + tabentry->last_anl_tuples = msg->m_tuples; + } +} + +/* ---------- + * pgstat_recv_analyze() - + * + * Process an ANALYZE message. + * ---------- + */ +static void +pgstat_recv_analyze(PgStat_MsgAnalyze *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatTabEntry *tabentry; + bool found; + + dbentry = pgstat_get_db_entry(msg->m_databaseid); + + tabentry = hash_search(dbentry->tables, &(msg->m_tableoid), + HASH_ENTER, &found); + + /* + * If we are creating the entry, initialize it. + */ + if (!found) + { + tabentry->tableid = msg->m_tableoid; + + tabentry->tuples_returned = 0; + tabentry->tuples_fetched = 0; + tabentry->tuples_inserted = 0; + tabentry->tuples_deleted = 0; + tabentry->tuples_updated = 0; + + tabentry->n_live_tuples = msg->m_live_tuples; + tabentry->n_dead_tuples = msg->m_dead_tuples; + tabentry->last_anl_tuples = msg->m_live_tuples + msg->m_dead_tuples; + + tabentry->blocks_fetched = 0; + tabentry->blocks_hit = 0; + } + else + { + tabentry->n_live_tuples = msg->m_live_tuples; + tabentry->n_dead_tuples = msg->m_dead_tuples; + tabentry->last_anl_tuples = msg->m_live_tuples + msg->m_dead_tuples; + } +} /* ---------- * pgstat_recv_activity() - @@ -2690,6 +2933,10 @@ pgstat_recv_tabstat(PgStat_MsgTabstat *msg, int len) tabentry->tuples_deleted = tabmsg[i].t_tuples_deleted; tabentry->blocks_fetched = tabmsg[i].t_blocks_fetched; tabentry->blocks_hit = tabmsg[i].t_blocks_hit; + + tabentry->n_live_tuples = tabmsg[i].t_tuples_inserted; + tabentry->n_dead_tuples = tabmsg[i].t_tuples_updated + + tabmsg[i].t_tuples_deleted; tabentry->destroy = 0; } @@ -2706,6 +2953,10 @@ pgstat_recv_tabstat(PgStat_MsgTabstat *msg, int len) tabentry->tuples_deleted += tabmsg[i].t_tuples_deleted; tabentry->blocks_fetched += tabmsg[i].t_blocks_fetched; tabentry->blocks_hit += tabmsg[i].t_blocks_hit; + + tabentry->n_live_tuples += tabmsg[i].t_tuples_inserted; + tabentry->n_dead_tuples += tabmsg[i].t_tuples_updated + + tabmsg[i].t_tuples_deleted; } /* diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 2de5527648..6964af6789 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -37,7 +37,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.458 2005/07/04 04:51:47 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.459 2005/07/14 05:13:40 tgl Exp $ * * NOTES * @@ -106,6 +106,7 @@ #include "miscadmin.h" #include "nodes/nodes.h" #include "pgstat.h" +#include "postmaster/autovacuum.h" #include "postmaster/fork_process.h" #include "postmaster/pgarch.h" #include "postmaster/postmaster.h" @@ -207,6 +208,7 @@ char *preload_libraries_string = NULL; /* PIDs of special child processes; 0 when not running */ static pid_t StartupPID = 0, BgWriterPID = 0, + AutoVacPID = 0, PgArchPID = 0, PgStatPID = 0, SysLoggerPID = 0; @@ -872,8 +874,8 @@ PostmasterMain(int argc, char *argv[]) * * CAUTION: when changing this list, check for side-effects on the signal * handling setup of child processes. See tcop/postgres.c, - * bootstrap/bootstrap.c, postmaster/bgwriter.c, postmaster/pgarch.c, - * postmaster/pgstat.c, and postmaster/syslogger.c. + * bootstrap/bootstrap.c, postmaster/bgwriter.c, postmaster/autovacuum.c, + * postmaster/pgarch.c, postmaster/pgstat.c, and postmaster/syslogger.c. */ pqinitmask(); PG_SETMASK(&BlockSig); @@ -930,6 +932,11 @@ PostmasterMain(int argc, char *argv[]) */ PgStartTime = GetCurrentTimestamp(); + /* + * Initialize the autovacuum daemon + */ + autovac_init(); + status = ServerLoop(); /* @@ -1251,6 +1258,15 @@ ServerLoop(void) kill(BgWriterPID, SIGUSR2); } + /* + * Start a new autovacuum process, if there isn't one running already. + * (It'll die relatively quickly.) We check that it's not started + * too frequently in autovac_start. + */ + if (AutoVacuumingActive() && AutoVacPID == 0 && + StartupPID == 0 && !FatalError && Shutdown == NoShutdown) + AutoVacPID = autovac_start(); + /* If we have lost the archiver, try to start a new one */ if (XLogArchivingActive() && PgArchPID == 0 && StartupPID == 0 && !FatalError && Shutdown == NoShutdown) @@ -1818,6 +1834,8 @@ SIGHUP_handler(SIGNAL_ARGS) SignalChildren(SIGHUP); if (BgWriterPID != 0) kill(BgWriterPID, SIGHUP); + if (AutoVacPID != 0) + kill(AutoVacPID, SIGHUP); if (PgArchPID != 0) kill(PgArchPID, SIGHUP); if (SysLoggerPID != 0) @@ -1869,6 +1887,16 @@ pmdie(SIGNAL_ARGS) ereport(LOG, (errmsg("received smart shutdown request"))); + /* + * We won't wait out an autovacuum iteration ... + */ + if (AutoVacPID != 0) + { + /* Use statement cancel to shut it down */ + kill(AutoVacPID, SIGINT); + break; /* let reaper() handle this */ + } + if (DLGetHead(BackendList)) break; /* let reaper() handle this */ @@ -1905,13 +1933,15 @@ pmdie(SIGNAL_ARGS) ereport(LOG, (errmsg("received fast shutdown request"))); - if (DLGetHead(BackendList)) + if (DLGetHead(BackendList) || AutoVacPID != 0) { if (!FatalError) { ereport(LOG, (errmsg("aborting any active transactions"))); SignalChildren(SIGTERM); + if (AutoVacPID != 0) + kill(AutoVacPID, SIGTERM); /* reaper() does the rest */ } break; @@ -1953,6 +1983,8 @@ pmdie(SIGNAL_ARGS) kill(StartupPID, SIGQUIT); if (BgWriterPID != 0) kill(BgWriterPID, SIGQUIT); + if (AutoVacPID != 0) + kill(AutoVacPID, SIGQUIT); if (PgArchPID != 0) kill(PgArchPID, SIGQUIT); if (PgStatPID != 0) @@ -2051,7 +2083,7 @@ reaper(SIGNAL_ARGS) /* * Go to shutdown mode if a shutdown request was pending. * Otherwise, try to start the archiver and stats collector - * too. + * too. (We could, but don't, try to start autovacuum here.) */ if (Shutdown > NoShutdown && BgWriterPID != 0) kill(BgWriterPID, SIGUSR2); @@ -2072,8 +2104,8 @@ reaper(SIGNAL_ARGS) if (BgWriterPID != 0 && pid == BgWriterPID) { BgWriterPID = 0; - if (exitstatus == 0 && Shutdown > NoShutdown && - !FatalError && !DLGetHead(BackendList)) + if (exitstatus == 0 && Shutdown > NoShutdown && !FatalError && + !DLGetHead(BackendList) && AutoVacPID == 0) { /* * Normal postmaster exit is here: we've seen normal exit @@ -2098,6 +2130,23 @@ reaper(SIGNAL_ARGS) continue; } + /* + * Was it the autovacuum process? Normal exit can be ignored; + * we'll start a new one at the next iteration of the postmaster's + * main loop, if necessary. + * + * An unexpected exit must crash the system. + */ + if (AutoVacPID != 0 && pid == AutoVacPID) + { + AutoVacPID = 0; + autovac_stopped(); + if (exitstatus != 0) + HandleChildCrash(pid, exitstatus, + _("autovacuum process")); + continue; + } + /* * Was it the archiver? If so, just try to start a new one; no * need to force reset of the rest of the system. (If fail, we'll @@ -2156,7 +2205,8 @@ reaper(SIGNAL_ARGS) * StartupDataBase. (We can ignore the archiver and stats * processes here since they are not connected to shmem.) */ - if (DLGetHead(BackendList) || StartupPID != 0 || BgWriterPID != 0) + if (DLGetHead(BackendList) || StartupPID != 0 || BgWriterPID != 0 || + AutoVacPID != 0) goto reaper_done; ereport(LOG, (errmsg("all server processes terminated; reinitializing"))); @@ -2171,7 +2221,7 @@ reaper(SIGNAL_ARGS) if (Shutdown > NoShutdown) { - if (DLGetHead(BackendList) || StartupPID != 0) + if (DLGetHead(BackendList) || StartupPID != 0 || AutoVacPID != 0) goto reaper_done; /* Start the bgwriter if not running */ if (BgWriterPID == 0) @@ -2239,7 +2289,7 @@ CleanupBackend(int pid, } /* - * HandleChildCrash -- cleanup after failed backend or bgwriter. + * HandleChildCrash -- cleanup after failed backend, bgwriter, or autovacuum. * * The objectives here are to clean up our local state about the child * process, and to signal all other remaining children to quickdie. @@ -2317,6 +2367,18 @@ HandleChildCrash(int pid, int exitstatus, const char *procname) kill(BgWriterPID, (SendStop ? SIGSTOP : SIGQUIT)); } + /* Take care of the autovacuum daemon too */ + if (pid == AutoVacPID) + AutoVacPID = 0; + else if (AutoVacPID != 0 && !FatalError) + { + ereport(DEBUG2, + (errmsg_internal("sending %s to process %d", + (SendStop ? "SIGSTOP" : "SIGQUIT"), + (int) AutoVacPID))); + kill(AutoVacPID, (SendStop ? SIGSTOP : SIGQUIT)); + } + /* Force a power-cycle of the pgarch process too */ /* (Shouldn't be necessary, but just for luck) */ if (PgArchPID != 0 && !FatalError) @@ -3154,6 +3216,7 @@ SubPostmasterMain(int argc, char *argv[]) * can attach at the same address the postmaster used. */ if (strcmp(argv[1], "-forkbackend") == 0 || + strcmp(argv[1], "-forkautovac") == 0 || strcmp(argv[1], "-forkboot") == 0) PGSharedMemoryReAttach(); @@ -3205,6 +3268,17 @@ SubPostmasterMain(int argc, char *argv[]) BootstrapMain(argc - 2, argv + 2); proc_exit(0); } + if (strcmp(argv[1], "-forkautovac") == 0) + { + /* Close the postmaster's sockets */ + ClosePostmasterPorts(false); + + /* Attached process to shared data structures */ + CreateSharedMemoryAndSemaphores(false, 0); + + AutoVacMain(argc - 2, argv + 2); + proc_exit(0); + } if (strcmp(argv[1], "-forkarch") == 0) { /* Close the postmaster's sockets */ @@ -3300,7 +3374,11 @@ sigusr1_handler(SIGNAL_ARGS) * use of this. */ if (Shutdown <= SmartShutdown) + { SignalChildren(SIGUSR1); + if (AutoVacPID != 0) + kill(AutoVacPID, SIGUSR1); + } } if (PgArchPID != 0 && Shutdown == NoShutdown) diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 038a482239..aab81eabe2 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/tcop/postgres.c,v 1.453 2005/07/10 21:13:58 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/tcop/postgres.c,v 1.454 2005/07/14 05:13:41 tgl Exp $ * * NOTES * this is the "main" module of the postgres backend and @@ -2111,7 +2111,7 @@ authdie(SIGNAL_ARGS) * Query-cancel signal from postmaster: abort current transaction * at soonest convenient time */ -static void +void StatementCancelHandler(SIGNAL_ARGS) { int save_errno = errno; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index becd279338..6df4546538 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -10,7 +10,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/tcop/utility.c,v 1.240 2005/06/30 00:00:51 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/tcop/utility.c,v 1.241 2005/07/14 05:13:41 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -843,7 +843,7 @@ ProcessUtility(Node *parsetree, break; case T_VacuumStmt: - vacuum((VacuumStmt *) parsetree); + vacuum((VacuumStmt *) parsetree, NIL); break; case T_ExplainStmt: diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index ffe7db5f7e..389ad06f2f 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/init/miscinit.c,v 1.145 2005/07/04 04:51:50 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/init/miscinit.c,v 1.146 2005/07/14 05:13:41 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -32,6 +32,7 @@ #include "catalog/pg_authid.h" #include "libpq/libpq-be.h" #include "miscadmin.h" +#include "postmaster/autovacuum.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" @@ -394,7 +395,7 @@ void InitializeSessionUserIdStandalone(void) { /* This function should only be called in a single-user backend. */ - AssertState(!IsUnderPostmaster); + AssertState(!IsUnderPostmaster || IsAutoVacuumProcess()); /* call only once */ AssertState(!OidIsValid(AuthenticatedUserId)); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 18e60cab0f..d687c59ec6 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/init/postinit.c,v 1.152 2005/07/04 04:51:50 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/init/postinit.c,v 1.153 2005/07/14 05:13:41 tgl Exp $ * * *------------------------------------------------------------------------- @@ -29,6 +29,7 @@ #include "libpq/hba.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "postmaster/autovacuum.h" #include "postmaster/postmaster.h" #include "storage/backendid.h" #include "storage/fd.h" @@ -268,7 +269,8 @@ BaseInit(void) * InitPostgres * Initialize POSTGRES. * - * In bootstrap mode neither of the parameters are used. + * In bootstrap mode neither of the parameters are used. In autovacuum + * mode, the username parameter is not used. * * The return value indicates whether the userID is a superuser. (That * can only be tested inside a transaction, so we want to do it during @@ -282,6 +284,7 @@ bool InitPostgres(const char *dbname, const char *username) { bool bootstrap = IsBootstrapProcessingMode(); + bool autovacuum = IsAutoVacuumProcess(); bool am_superuser; /* @@ -402,10 +405,11 @@ InitPostgres(const char *dbname, const char *username) RelationCacheInitializePhase2(); /* - * Figure out our postgres user id. In standalone mode we use a fixed - * id, otherwise we figure it out from the authenticated user name. + * Figure out our postgres user id. In standalone mode and in the + * autovacuum process, we use a fixed id, otherwise we figure it out from + * the authenticated user name. */ - if (bootstrap) + if (bootstrap || autovacuum) InitializeSessionUserIdStandalone(); else if (!IsUnderPostmaster) { @@ -441,7 +445,7 @@ InitPostgres(const char *dbname, const char *username) /* * Check if user is a superuser. */ - if (bootstrap) + if (bootstrap || autovacuum) am_superuser = true; else am_superuser = superuser(); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index ca59f6755b..194949fbd7 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -10,7 +10,7 @@ * Written by Peter Eisentraut . * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/misc/guc.c,v 1.273 2005/07/05 23:18:10 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/misc/guc.c,v 1.274 2005/07/14 05:13:42 tgl Exp $ * *-------------------------------------------------------------------- */ @@ -45,6 +45,7 @@ #include "optimizer/prep.h" #include "parser/parse_expr.h" #include "parser/parse_relation.h" +#include "postmaster/autovacuum.h" #include "postmaster/bgwriter.h" #include "postmaster/syslogger.h" #include "postmaster/postmaster.h" @@ -286,6 +287,8 @@ const char *const config_group_names[] = gettext_noop("Statistics / Monitoring"), /* STATS_COLLECTOR */ gettext_noop("Statistics / Query and Index Statistics Collector"), + /* AUTOVACUUM */ + gettext_noop("Auto Vacuum"), /* CLIENT_CONN */ gettext_noop("Client Connection Defaults"), /* CLIENT_CONN_STATEMENT */ @@ -678,6 +681,15 @@ static struct config_bool ConfigureNamesBool[] = false, NULL, NULL }, + { + {"autovacuum", PGC_SIGHUP, AUTOVACUUM, + gettext_noop("Starts the auto vacuum subprocess."), + NULL + }, + &autovacuum_start_daemon, + false, NULL, NULL + }, + { {"trace_notify", PGC_USERSET, DEVELOPER_OPTIONS, gettext_noop("Generates debugging output for LISTEN and NOTIFY."), @@ -1389,6 +1401,31 @@ static struct config_int ConfigureNamesInt[] = BLCKSZ, BLCKSZ, BLCKSZ, NULL, NULL }, + { + {"autovacuum_naptime", PGC_SIGHUP, AUTOVACUUM, + gettext_noop("Time to sleep between autovacuum runs, in seconds."), + NULL + }, + &autovacuum_naptime, + 60, 0, INT_MAX, NULL, NULL + }, + { + {"autovacuum_vacuum_threshold", PGC_SIGHUP, AUTOVACUUM, + gettext_noop("Minimum number of tuple updates or deletes prior to vacuum."), + NULL + }, + &autovacuum_vac_thresh, + 1000, 0, INT_MAX, NULL, NULL + }, + { + {"autovacuum_analyze_threshold", PGC_SIGHUP, AUTOVACUUM, + gettext_noop("Minimum number of tuple inserts, updates or deletes prior to analyze."), + NULL + }, + &autovacuum_anl_thresh, + 500, 0, INT_MAX, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL @@ -1487,6 +1524,23 @@ static struct config_real ConfigureNamesReal[] = 0.5, 0.0, 1.0, assign_random_seed, show_random_seed }, + { + {"autovacuum_vacuum_scale_factor", PGC_SIGHUP, AUTOVACUUM, + gettext_noop("Number of tuple updates or deletes prior to vacuum as a fraction of reltuples."), + NULL + }, + &autovacuum_vac_scale, + 0.4, 0.0, 100.0, NULL, NULL + }, + { + {"autovacuum_analyze_scale_factor", PGC_SIGHUP, AUTOVACUUM, + gettext_noop("Number of tuple inserts, updates or deletes prior to analyze as a fraction of reltuples."), + NULL + }, + &autovacuum_anl_scale, + 0.2, 0.0, 100.0, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0.0, 0.0, 0.0, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 193b0303b3..dc06658e7f 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -284,6 +284,18 @@ #stats_reset_on_server_start = on +#--------------------------------------------------------------------------- +# AUTOVACUUM PARAMETERS +#--------------------------------------------------------------------------- + +#autovacuum = false # enable autovacuum subprocess? +#autovacuum_naptime = 60 # time between autovacuum runs, in seconds +#autovacuum_vacuum_threshold = 1000 # min # of tuple updates before vacuum +#autovacuum_analyze_threshold = 500 # min # of tuple updates before analyze +#autovacuum_vacuum_scale_factor = 0.4 # fraction of rel size before vacuum +#autovacuum_analyze_scale_factor = 0.2 # fraction of rel size before analyze + + #--------------------------------------------------------------------------- # CLIENT CONNECTION DEFAULTS #--------------------------------------------------------------------------- diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 5ce6b4724e..d87e9f9ce6 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -37,7 +37,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.288 2005/07/10 21:13:59 tgl Exp $ + * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.289 2005/07/14 05:13:42 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 200507102 +#define CATALOG_VERSION_NO 200507131 #endif diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index 1ab33708f0..0c4c095e40 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -8,7 +8,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/indexing.h,v 1.89 2005/07/07 20:39:59 tgl Exp $ + * $PostgreSQL: pgsql/src/include/catalog/indexing.h,v 1.90 2005/07/14 05:13:42 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -210,6 +210,8 @@ DECLARE_UNIQUE_INDEX(pg_type_oid_index,2703, on pg_type using btree(oid oid_ops) DECLARE_UNIQUE_INDEX(pg_type_typname_nsp_index,2704, on pg_type using btree(typname name_ops, typnamespace oid_ops)); #define TypeNameNspIndexId 2704 +DECLARE_UNIQUE_INDEX(pg_autovacuum_vacrelid_index,1250, on pg_autovacuum using btree(vacrelid oid_ops)); +#define AutovacuumRelidIndexId 1250 /* last step of initialization script: build the indexes declared above */ BUILD_INDICES diff --git a/src/include/catalog/pg_autovacuum.h b/src/include/catalog/pg_autovacuum.h new file mode 100644 index 0000000000..b0bfc20492 --- /dev/null +++ b/src/include/catalog/pg_autovacuum.h @@ -0,0 +1,60 @@ +/*------------------------------------------------------------------------- + * + * pg_autovacuum.h + * definition of the system "autovacuum" relation (pg_autovacuum) + * + * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * $PostgreSQL: pgsql/src/include/catalog/pg_autovacuum.h,v 1.1 2005/07/14 05:13:42 tgl Exp $ + * + *------------------------------------------------------------------------- + */ +#ifndef PG_AUTOVACUUM_H +#define PG_AUTOVACUUM_H + +/* ---------------- + * postgres.h contains the system type definitions and the + * CATALOG(), BOOTSTRAP and DATA() sugar words so this file + * can be read by both genbki.sh and the C compiler. + * ---------------- + */ + +/* ---------------- + * pg_autovacuum definition. cpp turns this into + * typedef struct FormData_pg_autovacuum + * ---------------- + */ +#define AutovacuumRelationId 1248 +CATALOG(pg_autovacuum,1248) BKI_WITHOUT_OIDS +{ + Oid vacrelid; /* OID of table */ + bool enabled; /* enabled for this table? */ + int4 vac_base_thresh; /* base threshold value */ + float4 vac_scale_factor; /* reltuples scaling factor */ + int4 anl_base_thresh; /* base threshold value */ + float4 anl_scale_factor; /* reltuples scaling factor */ +} FormData_pg_autovacuum; + +/* ---------------- + * Form_pg_autovacuum corresponds to a pointer to a tuple with + * the format of pg_autovacuum relation. + * ---------------- + */ +typedef FormData_pg_autovacuum *Form_pg_autovacuum; + +/* ---------------- + * compiler constants for pg_autovacuum + * ---------------- + */ +#define Natts_pg_autovacuum 6 +#define Anum_pg_autovacuum_vacrelid 1 +#define Anum_pg_autovacuum_enabled 2 +#define Anum_pg_autovacuum_vac_base_thresh 3 +#define Anum_pg_autovacuum_vac_scale_factor 4 +#define Anum_pg_autovacuum_anl_base_thresh 5 +#define Anum_pg_autovacuum_anl_scale_factor 6 + +/* There are no preloaded tuples in pg_autovacuum.h */ + +#endif /* PG_AUTOVACUUM_H */ diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 0434b6f5ec..71ec058022 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/commands/vacuum.h,v 1.59 2004/12/31 22:03:28 pgsql Exp $ + * $PostgreSQL: pgsql/src/include/commands/vacuum.h,v 1.60 2005/07/14 05:13:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -125,7 +125,7 @@ extern DLLIMPORT int default_statistics_target; /* DLLIMPORT for PostGIS */ /* in commands/vacuum.c */ -extern void vacuum(VacuumStmt *vacstmt); +extern void vacuum(VacuumStmt *vacstmt, List *relids); extern void vac_open_indexes(Relation relation, LOCKMODE lockmode, int *nindexes, Relation **Irel); extern void vac_close_indexes(int nindexes, Relation *Irel, LOCKMODE lockmode); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5612d91290..4df12d77d3 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -5,7 +5,7 @@ * * Copyright (c) 2001-2005, PostgreSQL Global Development Group * - * $PostgreSQL: pgsql/src/include/pgstat.h,v 1.32 2005/06/29 22:51:57 tgl Exp $ + * $PostgreSQL: pgsql/src/include/pgstat.h,v 1.33 2005/07/14 05:13:43 tgl Exp $ * ---------- */ #ifndef PGSTAT_H @@ -20,14 +20,20 @@ * The types of backend/postmaster -> collector messages * ---------- */ -#define PGSTAT_MTYPE_DUMMY 0 -#define PGSTAT_MTYPE_BESTART 1 -#define PGSTAT_MTYPE_BETERM 2 -#define PGSTAT_MTYPE_ACTIVITY 3 -#define PGSTAT_MTYPE_TABSTAT 4 -#define PGSTAT_MTYPE_TABPURGE 5 -#define PGSTAT_MTYPE_DROPDB 6 -#define PGSTAT_MTYPE_RESETCOUNTER 7 +typedef enum StatMsgType +{ + PGSTAT_MTYPE_DUMMY, + PGSTAT_MTYPE_BESTART, + PGSTAT_MTYPE_BETERM, + PGSTAT_MTYPE_ACTIVITY, + PGSTAT_MTYPE_TABSTAT, + PGSTAT_MTYPE_TABPURGE, + PGSTAT_MTYPE_DROPDB, + PGSTAT_MTYPE_RESETCOUNTER, + PGSTAT_MTYPE_AUTOVAC_START, + PGSTAT_MTYPE_VACUUM, + PGSTAT_MTYPE_ANALYZE +} StatMsgType; /* ---------- * The data type used for counters. @@ -48,7 +54,7 @@ typedef int64 PgStat_Counter; */ typedef struct PgStat_MsgHdr { - int m_type; + StatMsgType m_type; int m_size; int m_backendid; int m_procpid; @@ -114,6 +120,47 @@ typedef struct PgStat_MsgBeterm PgStat_MsgHdr m_hdr; } PgStat_MsgBeterm; +/* ---------- + * PgStat_MsgAutovacStart Sent by the autovacuum daemon to signal + * that a database is going to be processed + * ---------- + */ +typedef struct PgStat_MsgAutovacStart +{ + PgStat_MsgHdr m_hdr; + Oid m_databaseid; + TimestampTz m_start_time; +} PgStat_MsgAutovacStart; + +/* ---------- + * PgStat_MsgVacuum Sent by the backend or autovacuum daemon + * after VACUUM or VACUUM ANALYZE + * ---------- + */ +typedef struct PgStat_MsgVacuum +{ + PgStat_MsgHdr m_hdr; + Oid m_databaseid; + Oid m_tableoid; + bool m_analyze; + PgStat_Counter m_tuples; +} PgStat_MsgVacuum; + +/* ---------- + * PgStat_MsgAnalyze Sent by the backend or autovacuum daemon + * after ANALYZE + * ---------- + */ +typedef struct PgStat_MsgAnalyze +{ + PgStat_MsgHdr m_hdr; + Oid m_databaseid; + Oid m_tableoid; + PgStat_Counter m_live_tuples; + PgStat_Counter m_dead_tuples; +} PgStat_MsgAnalyze; + + /* ---------- * PgStat_MsgActivity Sent by the backends when they start * to parse a query. @@ -200,14 +247,22 @@ typedef union PgStat_Msg PgStat_MsgTabpurge msg_tabpurge; PgStat_MsgDropdb msg_dropdb; PgStat_MsgResetcounter msg_resetcounter; + PgStat_MsgAutovacStart msg_autovacuum; + PgStat_MsgVacuum msg_vacuum; + PgStat_MsgAnalyze msg_analyze; } PgStat_Msg; /* ------------------------------------------------------------ * Statistic collector data structures follow + * + * PGSTAT_FILE_FORMAT_ID should be changed whenever any of these + * data structures change. * ------------------------------------------------------------ */ +#define PGSTAT_FILE_FORMAT_ID 0x01A5BC93 + /* ---------- * PgStat_StatDBEntry The collectors data per database * ---------- @@ -222,6 +277,7 @@ typedef struct PgStat_StatDBEntry PgStat_Counter n_blocks_fetched; PgStat_Counter n_blocks_hit; int destroy; + TimestampTz last_autovac_time; } PgStat_StatDBEntry; @@ -282,6 +338,10 @@ typedef struct PgStat_StatTabEntry PgStat_Counter tuples_updated; PgStat_Counter tuples_deleted; + PgStat_Counter n_live_tuples; + PgStat_Counter n_dead_tuples; + PgStat_Counter last_anl_tuples; + PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; @@ -323,6 +383,11 @@ extern void pgstat_bestart(void); extern void pgstat_ping(void); extern void pgstat_report_activity(const char *what); extern void pgstat_report_tabstat(void); +extern void pgstat_report_autovac(void); +extern void pgstat_report_vacuum(Oid tableoid, bool analyze, + PgStat_Counter tuples); +extern void pgstat_report_analyze(Oid tableoid, PgStat_Counter livetuples, + PgStat_Counter deadtuples); extern int pgstat_vacuum_tabstat(void); extern void pgstat_reset_counters(void); diff --git a/src/include/postmaster/autovacuum.h b/src/include/postmaster/autovacuum.h new file mode 100644 index 0000000000..85667af1cc --- /dev/null +++ b/src/include/postmaster/autovacuum.h @@ -0,0 +1,38 @@ +/*------------------------------------------------------------------------- + * + * autovacuum.h + * header file for integrated autovacuum daemon + * + * + * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * $PostgreSQL: pgsql/src/include/postmaster/autovacuum.h,v 1.1 2005/07/14 05:13:43 tgl Exp $ + * + *------------------------------------------------------------------------- + */ +#ifndef AUTOVACUUM_H +#define AUTOVACUUM_H + +/* GUC variables */ +extern bool autovacuum_start_daemon; +extern int autovacuum_naptime; +extern int autovacuum_vac_thresh; +extern double autovacuum_vac_scale; +extern int autovacuum_anl_thresh; +extern double autovacuum_anl_scale; + +/* Status inquiry functions */ +extern bool AutoVacuumingActive(void); +extern bool IsAutoVacuumProcess(void); + +/* Functions to start autovacuum process, called from postmaster */ +extern void autovac_init(void); +extern int autovac_start(void); +extern void autovac_stopped(void); + +#ifdef EXEC_BACKEND +extern void AutoVacMain(int argc, char *argv[]); +#endif + +#endif /* AUTOVACUUM_H */ diff --git a/src/include/tcop/tcopprot.h b/src/include/tcop/tcopprot.h index d694146d59..7886d66d7c 100644 --- a/src/include/tcop/tcopprot.h +++ b/src/include/tcop/tcopprot.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/tcop/tcopprot.h,v 1.75 2005/06/03 23:05:30 tgl Exp $ + * $PostgreSQL: pgsql/src/include/tcop/tcopprot.h,v 1.76 2005/07/14 05:13:44 tgl Exp $ * * OLD COMMENTS * This file was created so that other c files could get the two @@ -58,6 +58,7 @@ extern bool assign_max_stack_depth(int newval, bool doit, GucSource source); extern void die(SIGNAL_ARGS); extern void quickdie(SIGNAL_ARGS); extern void authdie(SIGNAL_ARGS); +extern void StatementCancelHandler(SIGNAL_ARGS); extern void prepare_for_client_read(void); extern void client_read_ended(void); extern int PostgresMain(int argc, char *argv[], const char *username); diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h index 39d2af3bab..beddf5dd5f 100644 --- a/src/include/utils/guc_tables.h +++ b/src/include/utils/guc_tables.h @@ -7,7 +7,7 @@ * * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * - * $PostgreSQL: pgsql/src/include/utils/guc_tables.h,v 1.19 2004/12/31 22:03:46 pgsql Exp $ + * $PostgreSQL: pgsql/src/include/utils/guc_tables.h,v 1.20 2005/07/14 05:13:44 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -62,6 +62,7 @@ enum config_group STATS, STATS_MONITORING, STATS_COLLECTOR, + AUTOVACUUM, CLIENT_CONN, CLIENT_CONN_STATEMENT, CLIENT_CONN_LOCALE, diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index 5c0e5ca895..a6210a1705 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -40,6 +40,7 @@ SELECT relname, relhasindex pg_attribute | t pg_auth_members | t pg_authid | t + pg_autovacuum | t pg_cast | t pg_class | t pg_constraint | t @@ -66,7 +67,7 @@ SELECT relname, relhasindex shighway | t tenk1 | t tenk2 | t -(56 rows) +(57 rows) -- -- another sanity check: every system catalog that has OIDs should have