rdelaage
/
postgresql
mirror of https://git.postgresql.org/git/postgresql.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Prevent WAL logging when COPY is done in the same transation that 

created it.

Simon Riggs
This commit is contained in:
Bruce Momjian 2007-01-25 02:17:26 +00:00
parent 693c85d954
commit ef65f6f7a4
10 changed files with 182 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
doc/src/sgml/perform.sgml
View File

@ -1,4 +1,4 @@
<!-- $PostgreSQL: pgsql/doc/src/sgml/perform.sgml,v 1.59 2006/10/07 20:59:03 petere Exp $ -->
<!-- $PostgreSQL: pgsql/doc/src/sgml/perform.sgml,v 1.60 2007/01/25 02:17:25 momjian Exp $ -->
<chapter id="performance-tips">
<title>Performance Tips</title>
@ -800,7 +800,9 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse;
prepared <command>INSERT</command> statement, and then use
<command>EXECUTE</command> as many times as required. This avoids
some of the overhead of repeatedly parsing and planning
<command>INSERT</command>.
<command>INSERT</command>. Different interfaces provide this facility
in different ways; look for Prepared Statements in the interface
documentation.
</para>
<para>
@ -809,6 +811,20 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse;
<command>INSERT</command>, even if <command>PREPARE</> is used and
multiple insertions are batched into a single transaction.
</para>
<para>
<command>COPY</command> is fastest when used within the same
transaction as an earlier <command>CREATE TABLE</command> or
<command>TRUNCATE</command> command. In those cases, no WAL
needs to be written because in case of an error, the files
containing the newly loaded data will be removed automatically.
<command>CREATE TABLE AS SELECT</command> is also optimized
to avoid writing WAL. <command>COPY</command> and
<command>CREATE TABLE AS SELECT</command> will write WAL
when <xref linkend="guc-archive-command"> is set and will not
therefore be optimized in that case.
</para>
</sect2>
<sect2 id="populate-rm-indexes">
@ -877,6 +893,29 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse;
</para>
</sect2>
<sect2 id="populate-pitr">
<title>Turn off <varname>archive_command</varname></title>
<para>
When loading large amounts of data you may want to unset the
<xref linkend="guc-archive-command"> before loading. It may be
faster to take a new base backup once the load has completed
than to allow a large archive to accumulate.
</para>
<para>
This is particularly important advice because certain commands
will perform more slowly when <varname>archive_command</varname>
is set, as a result of their needing to write large amounts of WAL.
This applies to the following commands:
<command>CREATE TABLE AS SELECT</command>,
<command>CREATE INDEX</command> and also <command>COPY</command>, when
it is executed in the same transaction as a prior
<command>CREATE TABLE</command> or <command>TRUNCATE</command> command.
</para>
</sect2>
<sect2 id="populate-analyze">
<title>Run <command>ANALYZE</command> Afterwards</title>
@ -914,8 +953,12 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse;
the first several guidelines are handled automatically. What is left
for you to do is to set appropriate (i.e., larger than normal) values
for <varname>maintenance_work_mem</varname> and
<varname>checkpoint_segments</varname> before loading the dump script,
and then to run <command>ANALYZE</> afterwards.
<varname>checkpoint_segments</varname>, as well as unsetting
<varname>archive_command</varname> before loading the dump script,
and then to run <command>ANALYZE</> afterwards and resetting
<varname>archive_command</varname> if required. All of the
parameters can be reset once the load has completed without needing
to restart the server, as described in <xref linkend="config-setting">.
</para>
<para>

45
src/backend/access/heap/heapam.c
View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.224 2007/01/09 22:00:59 momjian Exp $
* $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.225 2007/01/25 02:17:25 momjian Exp $
*
*
* INTERFACE ROUTINES
@ -28,6 +28,7 @@
* heap_update - replace a tuple in a relation with another tuple
* heap_markpos - mark scan position
* heap_restrpos - restore position to marked location
* heap_sync - sync heap, for when no WAL has been written
*
* NOTES
* This file contains the heap_ routines which implement
@ -50,6 +51,7 @@
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/procarray.h"
#include "storage/smgr.h"
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "utils/relcache.h"
@ -1358,7 +1360,7 @@ heap_get_latest_tid(Relation relation,
* that all new tuples go into new pages not containing any tuples from other
* transactions, that the relation gets fsync'd before commit, and that the
* transaction emits at least one WAL record to ensure RecordTransactionCommit
* will decide to WAL-log the commit.
* will decide to WAL-log the commit. (see heap_sync() comments also)
*
* use_fsm is passed directly to RelationGetBufferForTuple, which see for
* more info.
@ -1418,7 +1420,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
*/
if (HeapTupleHasExternal(tup) ||
(MAXALIGN(tup->t_len) > TOAST_TUPLE_THRESHOLD))
heaptup = toast_insert_or_update(relation, tup, NULL);
heaptup = toast_insert_or_update(relation, tup, NULL, use_wal);
else
heaptup = tup;
@ -1535,6 +1537,18 @@ simple_heap_insert(Relation relation, HeapTuple tup)
return heap_insert(relation, tup, GetCurrentCommandId(), true, true);
}
/*
* fast_heap_insert - insert a tuple with options to improve speed
*
* Currently, this routine allows specifying additional options for speed
* in certain cases, such as WAL-avoiding COPY command
*/
Oid
fast_heap_insert(Relation relation, HeapTuple tup, bool use_wal)
{
return heap_insert(relation, tup, GetCurrentCommandId(), use_wal, use_wal);
}
/*
* heap_delete - delete a tuple
*
@ -2086,11 +2100,11 @@ l2:
*
* Note: below this point, heaptup is the data we actually intend to
* store into the relation; newtup is the caller's original untoasted
* data.
* data. (We always use WAL for toast table updates.)
*/
if (need_toast)
{
heaptup = toast_insert_or_update(relation, newtup, &oldtup);
heaptup = toast_insert_or_update(relation, newtup, &oldtup, true);
newtupsize = MAXALIGN(heaptup->t_len);
}
else
@ -3966,3 +3980,24 @@ heap2_desc(StringInfo buf, uint8 xl_info, char *rec)
else
appendStringInfo(buf, "UNKNOWN");
}
/* ----------------
* heap_sync - sync a heap, for use when no WAL has been written
*
* ----------------
*/
void
heap_sync(Relation rel)
{
if (!rel->rd_istemp)
{
/*
* If we skipped using WAL, and it's not a temp relation,
* we must force the relation down to disk before it's
* safe to commit the transaction. This requires forcing
* out any dirty buffers and then doing a forced fsync.
*/
FlushRelationBuffers(rel);
smgrimmedsync(rel->rd_smgr);
}
}

14
src/backend/access/heap/tuptoaster.c
View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.68 2007/01/05 22:19:22 momjian Exp $
* $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.69 2007/01/25 02:17:26 momjian Exp $
*
*
* INTERFACE ROUTINES
@ -42,7 +42,7 @@
#undef TOAST_DEBUG
static void toast_delete_datum(Relation rel, Datum value);
static Datum toast_save_datum(Relation rel, Datum value);
static Datum toast_save_datum(Relation rel, Datum value, bool use_wal);
static varattrib *toast_fetch_datum(varattrib *attr);
static varattrib *toast_fetch_datum_slice(varattrib *attr,
int32 sliceoffset, int32 length);
@ -342,7 +342,7 @@ toast_delete(Relation rel, HeapTuple oldtup)
* ----------
*/
HeapTuple
toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, bool use_wal)
{
HeapTuple result_tuple;
TupleDesc tupleDesc;
@ -612,7 +612,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
i = biggest_attno;
old_value = toast_values[i];
toast_action[i] = 'p';
toast_values[i] = toast_save_datum(rel, toast_values[i]);
toast_values[i] = toast_save_datum(rel, toast_values[i], use_wal);
if (toast_free[i])
pfree(DatumGetPointer(old_value));
@ -724,7 +724,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
i = biggest_attno;
old_value = toast_values[i];
toast_action[i] = 'p';
toast_values[i] = toast_save_datum(rel, toast_values[i]);
toast_values[i] = toast_save_datum(rel, toast_values[i], use_wal);
if (toast_free[i])
pfree(DatumGetPointer(old_value));
@ -972,7 +972,7 @@ toast_compress_datum(Datum value)
* ----------
*/
static Datum
toast_save_datum(Relation rel, Datum value)
toast_save_datum(Relation rel, Datum value, bool use_wal)
{
Relation toastrel;
Relation toastidx;
@ -1057,7 +1057,7 @@ toast_save_datum(Relation rel, Datum value)
if (!HeapTupleIsValid(toasttup))
elog(ERROR, "failed to build TOAST tuple");
simple_heap_insert(toastrel, toasttup);
fast_heap_insert(toastrel, toasttup, use_wal);
/*
* Create the index entry. We cheat a little here by not using

5
src/backend/catalog/index.c
View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.276 2007/01/09 02:14:11 tgl Exp $
* $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.277 2007/01/25 02:17:26 momjian Exp $
*
*
* INTERFACE ROUTINES
@ -1245,6 +1245,9 @@ setNewRelfilenode(Relation relation)
heap_close(pg_class, RowExclusiveLock);
/* Remember we did this in current transaction, to allow later optimisations */
relation->rd_newRelfilenodeSubid = GetCurrentSubTransactionId();
/* Make sure the relfilenode change is visible */
CommandCounterIncrement();
}

53
src/backend/commands/copy.c
View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.274 2007/01/05 22:19:25 momjian Exp $
* $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.275 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
@ -1652,6 +1652,7 @@ CopyFrom(CopyState cstate)
ExprContext *econtext; /* used for ExecEvalExpr for default atts */
MemoryContext oldcontext = CurrentMemoryContext;
ErrorContextCallback errcontext;
bool use_wal = true; /* By default, we use WAL to log db changes */
Assert(cstate->rel);
@ -1843,6 +1844,28 @@ CopyFrom(CopyState cstate)
nfields = file_has_oids ? (attr_count + 1) : attr_count;
field_strings = (char **) palloc(nfields * sizeof(char *));
/*
* Check for performance optimization by avoiding WAL writes
*
* If archive logging is not be enabled *and* either
* - table is created in same transaction as this COPY
* - table data is now being written to new relfilenode
* then we can safely avoid writing WAL. Why?
* The data files for the table plus toast table/index, plus any indexes
* will all be dropped at the end of the transaction if it fails, so we
* do not need to worry about inconsistent states.
* As mentioned in comments in utils/rel.h, the in-same-transaction test is
* not completely reliable, since rd_createSubId can be reset to zero in
* certain cases before the end of the creating transaction.
* We are doing this for performance only, so we only need to know:
* if rd_createSubid != InvalidSubTransactionId then it is *always* just
* created. If we have PITR enabled, then we *must* use_wal
*/
if ((cstate->rel->rd_createSubid != InvalidSubTransactionId ||
cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId)
&& !XLogArchivingActive())
use_wal = false;
/* Initialize state variables */
cstate->fe_eof = false;
cstate->eol_type = EOL_UNKNOWN;
@ -2076,7 +2099,7 @@ CopyFrom(CopyState cstate)
ExecConstraints(resultRelInfo, slot, estate);
/* OK, store the tuple and create index entries for it */
simple_heap_insert(cstate->rel, tuple);
fast_heap_insert(cstate->rel, tuple, use_wal);
if (resultRelInfo->ri_NumIndices > 0)
ExecInsertIndexTuples(slot, &(tuple->t_self), estate, false);
@ -2093,6 +2116,32 @@ CopyFrom(CopyState cstate)
}
}
/*
* If we skipped writing WAL for heaps, then we need to sync
*/
if (!use_wal)
{
/* main heap */
heap_sync(cstate->rel);
/* main heap indexes, if any */
/* we always use WAL for index inserts, so no need to sync */
/* toast heap, if any */
if (OidIsValid(cstate->rel->rd_rel->reltoastrelid))
{
Relation toastrel;
toastrel = heap_open(cstate->rel->rd_rel->reltoastrelid,
AccessShareLock);
heap_sync(toastrel);
heap_close(toastrel, AccessShareLock);
}
/* toast index, if toast heap */
/* we always use WAL for index inserts, so no need to sync */
}
/* Done, clean up */
error_context_stack = errcontext.previous;

8
src/backend/executor/execMain.c
View File

@ -26,7 +26,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.283 2007/01/05 22:19:27 momjian Exp $
* $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.284 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
@ -2516,11 +2516,7 @@ CloseIntoRel(QueryDesc *queryDesc)
*/
if (!estate->es_into_relation_use_wal &&
!estate->es_into_relation_descriptor->rd_istemp)
{
FlushRelationBuffers(estate->es_into_relation_descriptor);
/* FlushRelationBuffers will have opened rd_smgr */
smgrimmedsync(estate->es_into_relation_descriptor->rd_smgr);
}
heap_sync(estate->es_into_relation_descriptor);
/* close rel, but keep lock until commit */
heap_close(estate->es_into_relation_descriptor, NoLock);

23
src/backend/utils/cache/relcache.c vendored
View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/utils/cache/relcache.c,v 1.254 2007/01/09 02:14:15 tgl Exp $
* $PostgreSQL: pgsql/src/backend/utils/cache/relcache.c,v 1.255 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
@ -836,6 +836,7 @@ RelationBuildDesc(Oid targetRelId, Relation oldrelation)
relation->rd_refcnt = 0;
relation->rd_isnailed = false;
relation->rd_createSubid = InvalidSubTransactionId;
relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
relation->rd_istemp = isTempNamespace(relation->rd_rel->relnamespace);
/*
@ -1358,6 +1359,7 @@ formrdesc(const char *relationName, Oid relationReltype,
*/
relation->rd_isnailed = true;
relation->rd_createSubid = InvalidSubTransactionId;
relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
relation->rd_istemp = false;
/*
@ -1769,6 +1771,7 @@ RelationClearRelation(Relation relation, bool rebuild)
Oid save_relid = RelationGetRelid(relation);
int old_refcnt = relation->rd_refcnt;
SubTransactionId old_createSubid = relation->rd_createSubid;
SubTransactionId old_newRelfilenodeSubid = relation->rd_newRelfilenodeSubid;
TupleDesc old_att = relation->rd_att;
RuleLock *old_rules = relation->rd_rules;
MemoryContext old_rulescxt = relation->rd_rulescxt;
@ -1787,6 +1790,8 @@ RelationClearRelation(Relation relation, bool rebuild)
}
relation->rd_refcnt = old_refcnt;
relation->rd_createSubid = old_createSubid;
relation->rd_newRelfilenodeSubid = old_newRelfilenodeSubid;
if (equalTupleDescs(old_att, relation->rd_att))
{
/* needn't flush typcache here */
@ -1827,7 +1832,8 @@ RelationFlushRelation(Relation relation)
{
bool rebuild;
if (relation->rd_createSubid != InvalidSubTransactionId)
if (relation->rd_createSubid != InvalidSubTransactionId ||
relation->rd_newRelfilenodeSubid != InvalidSubTransactionId)
{
/*
* New relcache entries are always rebuilt, not flushed; else we'd
@ -1909,6 +1915,9 @@ RelationCacheInvalidateEntry(Oid relationId)
* so we do not touch new-in-transaction relations; they cannot be targets
* of cross-backend SI updates (and our own updates now go through a
* separate linked list that isn't limited by the SI message buffer size).
* We don't do anything special for newRelfilenode-in-transaction relations,
* though since we have a lock on the relation nobody else should be
* generating cache invalidation messages for it anyhow.
*
* We do this in two phases: the first pass deletes deletable items, and
* the second one rebuilds the rebuildable items. This is essential for
@ -2085,6 +2094,7 @@ AtEOXact_RelationCache(bool isCommit)
continue;
}
}
relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
/*
* Flush any temporary index list.
@ -2146,6 +2156,13 @@ AtEOSubXact_RelationCache(bool isCommit, SubTransactionId mySubid,
continue;
}
}
if (relation->rd_newRelfilenodeSubid == mySubid)
{
if (isCommit)
relation->rd_newRelfilenodeSubid = parentSubid;
else
relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
}
/*
* Flush any temporary index list.
@ -2235,6 +2252,7 @@ RelationBuildLocalRelation(const char *relname,
/* it's being created in this transaction */
rel->rd_createSubid = GetCurrentSubTransactionId();
rel->rd_newRelfilenodeSubid = InvalidSubTransactionId;
/* must flag that we have rels created in this transaction */
need_eoxact_work = true;
@ -3392,6 +3410,7 @@ load_relcache_init_file(void)
rel->rd_indexlist = NIL;
rel->rd_oidindex = InvalidOid;
rel->rd_createSubid = InvalidSubTransactionId;
rel->rd_newRelfilenodeSubid = InvalidSubTransactionId;
rel->rd_amcache = NULL;
MemSet(&rel->pgstat_info, 0, sizeof(rel->pgstat_info));

7
src/include/access/heapam.h
View File

@ -7,7 +7,7 @@
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.119 2007/01/09 22:01:00 momjian Exp $
* $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.120 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
@ -178,6 +178,9 @@ extern void simple_heap_delete(Relation relation, ItemPointer tid);
extern void simple_heap_update(Relation relation, ItemPointer otid,
HeapTuple tup);
extern Oid fast_heap_insert(Relation relation, HeapTuple tup, bool use_wal);
extern void heap_markpos(HeapScanDesc scan);
extern void heap_restrpos(HeapScanDesc scan);
@ -236,4 +239,6 @@ extern MinimalTuple minimal_tuple_from_heap_tuple(HeapTuple htup);
extern HeapTuple heap_addheader(int natts, bool withoid,
Size structlen, void *structure);
extern void heap_sync(Relation relation);
#endif /* HEAPAM_H */

4
src/include/access/tuptoaster.h
View File

@ -6,7 +6,7 @@
*
* Copyright (c) 2000-2007, PostgreSQL Global Development Group
*
* $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.29 2007/01/05 22:19:51 momjian Exp $
* $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.30 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
@ -69,7 +69,7 @@
* ----------
*/
extern HeapTuple toast_insert_or_update(Relation rel,
HeapTuple newtup, HeapTuple oldtup);
HeapTuple newtup, HeapTuple oldtup, bool use_wal);
/* ----------
* toast_delete -

3
src/include/utils/rel.h
View File

@ -7,7 +7,7 @@
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* $PostgreSQL: pgsql/src/include/utils/rel.h,v 1.95 2007/01/09 02:14:16 tgl Exp $
* $PostgreSQL: pgsql/src/include/utils/rel.h,v 1.96 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
@ -137,6 +137,7 @@ typedef struct RelationData
char rd_indexvalid; /* state of rd_indexlist: 0 = not valid, 1 =
* valid, 2 = temporarily forced */
SubTransactionId rd_createSubid; /* rel was created in current xact */
SubTransactionId rd_newRelfilenodeSubid; /* rel had new relfilenode in current xact */
/*
* rd_createSubid is the ID of the highest subtransaction the rel has