Use multi-inserts for pg_attribute and pg_shdepend

For pg_attribute, this allows to insert at once a full set of attributes
for a relation (roughly 15% of WAL reduction in extreme cases).  For
pg_shdepend, this reduces the work done when creating new shared
dependencies from a database template.  The number of slots used for the
insertion is capped at 64kB of data inserted for both, depending on the
number of items to insert and the length of the rows involved.

More can be done for other catalogs, like pg_depend.  This part requires
a different approach as the number of slots to use depends also on the
number of entries discarded as pinned dependencies.  This is also
related to the rework or dependency handling for ALTER TABLE and CREATE
TABLE, mainly.

Author: Daniel Gustafsson
Reviewed-by: Andres Freund, Michael Paquier
Discussion: https://postgr.es/m/20190213182737.mxn6hkdxwrzgxk35@alap3.anarazel.de
This commit is contained in:
Michael Paquier 2020-07-31 10:54:26 +09:00
parent cab2556f3a
commit e3931d01f3
8 changed files with 223 additions and 117 deletions

View File

@ -2164,8 +2164,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
RelationPutHeapTuple(relation, buffer, heaptuples[ndone], false);
/*
* Note that heap_multi_insert is not used for catalog tuples yet, but
* this will cover the gap once that is the case.
* For logical decoding we need combocids to properly decode the
* catalog.
*/
if (needwal && need_cids)
log_heap_new_cid(relation, heaptuples[ndone]);
@ -2180,8 +2180,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
RelationPutHeapTuple(relation, buffer, heaptup, false);
/*
* We don't use heap_multi_insert for catalog tuples yet, but
* better be prepared...
* For logical decoding we need combocids to properly decode the
* catalog.
*/
if (needwal && need_cids)
log_heap_new_cid(relation, heaptup);

View File

@ -710,70 +710,122 @@ CheckAttributeType(const char *attname,
}
/*
* InsertPgAttributeTuple
* Construct and insert a new tuple in pg_attribute.
* Cap the maximum amount of bytes allocated for InsertPgAttributeTuples()
* slots.
*/
#define MAX_PGATTRIBUTE_INSERT_BYTES 65535
/*
* InsertPgAttributeTuples
* Construct and insert a set of tuples in pg_attribute.
*
* Caller has already opened and locked pg_attribute. new_attribute is the
* attribute to insert. attcacheoff is always initialized to -1, attacl,
* attfdwoptions and attmissingval are always initialized to NULL.
* Caller has already opened and locked pg_attribute. tupdesc contains the
* attributes to insert. attcacheoff is always initialized to -1, attacl,
* attfdwoptions and attmissingval are always initialized to NULL. attoptions
* must contain the same number of elements as tupdesc, or be NULL.
*
* indstate is the index state for CatalogTupleInsertWithInfo. It can be
* passed as NULL, in which case we'll fetch the necessary info. (Don't do
* this when inserting multiple attributes, because it's a tad more
* expensive.)
*
* new_rel_oid is the relation OID assigned to the attributes inserted.
* If set to InvalidOid, the relation OID from tupdesc is used instead.
*/
void
InsertPgAttributeTuple(Relation pg_attribute_rel,
Form_pg_attribute new_attribute,
Datum attoptions,
CatalogIndexState indstate)
InsertPgAttributeTuples(Relation pg_attribute_rel,
TupleDesc tupdesc,
Oid new_rel_oid,
Datum *attoptions,
CatalogIndexState indstate)
{
Datum values[Natts_pg_attribute];
bool nulls[Natts_pg_attribute];
HeapTuple tup;
TupleTableSlot **slot;
TupleDesc td;
int nslots;
int natts = 0;
int slotCount = 0;
bool close_index = false;
/* This is a tad tedious, but way cleaner than what we used to do... */
memset(values, 0, sizeof(values));
memset(nulls, false, sizeof(nulls));
td = RelationGetDescr(pg_attribute_rel);
values[Anum_pg_attribute_attrelid - 1] = ObjectIdGetDatum(new_attribute->attrelid);
values[Anum_pg_attribute_attname - 1] = NameGetDatum(&new_attribute->attname);
values[Anum_pg_attribute_atttypid - 1] = ObjectIdGetDatum(new_attribute->atttypid);
values[Anum_pg_attribute_attstattarget - 1] = Int32GetDatum(new_attribute->attstattarget);
values[Anum_pg_attribute_attlen - 1] = Int16GetDatum(new_attribute->attlen);
values[Anum_pg_attribute_attnum - 1] = Int16GetDatum(new_attribute->attnum);
values[Anum_pg_attribute_attndims - 1] = Int32GetDatum(new_attribute->attndims);
values[Anum_pg_attribute_attcacheoff - 1] = Int32GetDatum(-1);
values[Anum_pg_attribute_atttypmod - 1] = Int32GetDatum(new_attribute->atttypmod);
values[Anum_pg_attribute_attbyval - 1] = BoolGetDatum(new_attribute->attbyval);
values[Anum_pg_attribute_attstorage - 1] = CharGetDatum(new_attribute->attstorage);
values[Anum_pg_attribute_attalign - 1] = CharGetDatum(new_attribute->attalign);
values[Anum_pg_attribute_attnotnull - 1] = BoolGetDatum(new_attribute->attnotnull);
values[Anum_pg_attribute_atthasdef - 1] = BoolGetDatum(new_attribute->atthasdef);
values[Anum_pg_attribute_atthasmissing - 1] = BoolGetDatum(new_attribute->atthasmissing);
values[Anum_pg_attribute_attidentity - 1] = CharGetDatum(new_attribute->attidentity);
values[Anum_pg_attribute_attgenerated - 1] = CharGetDatum(new_attribute->attgenerated);
values[Anum_pg_attribute_attisdropped - 1] = BoolGetDatum(new_attribute->attisdropped);
values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(new_attribute->attislocal);
values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(new_attribute->attinhcount);
values[Anum_pg_attribute_attcollation - 1] = ObjectIdGetDatum(new_attribute->attcollation);
values[Anum_pg_attribute_attoptions - 1] = attoptions;
/* Initialize the number of slots to use */
nslots = Min(tupdesc->natts,
(MAX_PGATTRIBUTE_INSERT_BYTES / sizeof(FormData_pg_attribute)));
slot = palloc(sizeof(TupleTableSlot *) * nslots);
for (int i = 0; i < nslots; i++)
slot[i] = MakeSingleTupleTableSlot(td, &TTSOpsHeapTuple);
/* start out with empty permissions and empty options */
nulls[Anum_pg_attribute_attacl - 1] = true;
nulls[Anum_pg_attribute_attoptions - 1] = attoptions == (Datum) 0;
nulls[Anum_pg_attribute_attfdwoptions - 1] = true;
nulls[Anum_pg_attribute_attmissingval - 1] = true;
while (natts < tupdesc->natts)
{
Form_pg_attribute attrs = TupleDescAttr(tupdesc, natts);
tup = heap_form_tuple(RelationGetDescr(pg_attribute_rel), values, nulls);
ExecClearTuple(slot[slotCount]);
/* finally insert the new tuple, update the indexes, and clean up */
if (indstate != NULL)
CatalogTupleInsertWithInfo(pg_attribute_rel, tup, indstate);
else
CatalogTupleInsert(pg_attribute_rel, tup);
if (new_rel_oid != InvalidOid)
slot[slotCount]->tts_values[Anum_pg_attribute_attrelid - 1] = ObjectIdGetDatum(new_rel_oid);
else
slot[slotCount]->tts_values[Anum_pg_attribute_attrelid - 1] = ObjectIdGetDatum(attrs->attrelid);
heap_freetuple(tup);
slot[slotCount]->tts_values[Anum_pg_attribute_attname - 1] = NameGetDatum(&attrs->attname);
slot[slotCount]->tts_values[Anum_pg_attribute_atttypid - 1] = ObjectIdGetDatum(attrs->atttypid);
slot[slotCount]->tts_values[Anum_pg_attribute_attstattarget - 1] = Int32GetDatum(attrs->attstattarget);
slot[slotCount]->tts_values[Anum_pg_attribute_attlen - 1] = Int16GetDatum(attrs->attlen);
slot[slotCount]->tts_values[Anum_pg_attribute_attnum - 1] = Int16GetDatum(attrs->attnum);
slot[slotCount]->tts_values[Anum_pg_attribute_attndims - 1] = Int32GetDatum(attrs->attndims);
slot[slotCount]->tts_values[Anum_pg_attribute_attcacheoff - 1] = Int32GetDatum(-1);
slot[slotCount]->tts_values[Anum_pg_attribute_atttypmod - 1] = Int32GetDatum(attrs->atttypmod);
slot[slotCount]->tts_values[Anum_pg_attribute_attbyval - 1] = BoolGetDatum(attrs->attbyval);
slot[slotCount]->tts_values[Anum_pg_attribute_attstorage - 1] = CharGetDatum(attrs->attstorage);
slot[slotCount]->tts_values[Anum_pg_attribute_attalign - 1] = CharGetDatum(attrs->attalign);
slot[slotCount]->tts_values[Anum_pg_attribute_attnotnull - 1] = BoolGetDatum(attrs->attnotnull);
slot[slotCount]->tts_values[Anum_pg_attribute_atthasdef - 1] = BoolGetDatum(attrs->atthasdef);
slot[slotCount]->tts_values[Anum_pg_attribute_atthasmissing - 1] = BoolGetDatum(attrs->atthasmissing);
slot[slotCount]->tts_values[Anum_pg_attribute_attidentity - 1] = CharGetDatum(attrs->attidentity);
slot[slotCount]->tts_values[Anum_pg_attribute_attgenerated - 1] = CharGetDatum(attrs->attgenerated);
slot[slotCount]->tts_values[Anum_pg_attribute_attisdropped - 1] = BoolGetDatum(attrs->attisdropped);
slot[slotCount]->tts_values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(attrs->attislocal);
slot[slotCount]->tts_values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(attrs->attinhcount);
slot[slotCount]->tts_values[Anum_pg_attribute_attcollation - 1] = ObjectIdGetDatum(attrs->attcollation);
if (attoptions && attoptions[natts] != (Datum) 0)
slot[slotCount]->tts_values[Anum_pg_attribute_attoptions - 1] = attoptions[natts];
else
slot[slotCount]->tts_isnull[Anum_pg_attribute_attoptions - 1] = true;
/* start out with empty permissions and empty options */
slot[slotCount]->tts_isnull[Anum_pg_attribute_attacl - 1] = true;
slot[slotCount]->tts_isnull[Anum_pg_attribute_attfdwoptions - 1] = true;
slot[slotCount]->tts_isnull[Anum_pg_attribute_attmissingval - 1] = true;
ExecStoreVirtualTuple(slot[slotCount]);
slotCount++;
/*
* If slots are full or the end of processing has been reached, insert
* a batch of tuples.
*/
if (slotCount == nslots || natts == tupdesc->natts - 1)
{
/* fetch index info only when we know we need it */
if (!indstate)
{
indstate = CatalogOpenIndexes(pg_attribute_rel);
close_index = true;
}
/* insert the new tuples and update the indexes */
CatalogTuplesMultiInsertWithInfo(pg_attribute_rel, slot, slotCount,
indstate);
slotCount = 0;
}
natts++;
}
if (close_index)
CatalogCloseIndexes(indstate);
for (int i = 0; i < nslots; i++)
ExecDropSingleTupleTableSlot(slot[i]);
pfree(slot);
}
/* --------------------------------
@ -788,8 +840,6 @@ AddNewAttributeTuples(Oid new_rel_oid,
TupleDesc tupdesc,
char relkind)
{
Form_pg_attribute attr;
int i;
Relation rel;
CatalogIndexState indstate;
int natts = tupdesc->natts;
@ -803,30 +853,26 @@ AddNewAttributeTuples(Oid new_rel_oid,
indstate = CatalogOpenIndexes(rel);
/*
* First we add the user attributes. This is also a convenient place to
* add dependencies on their datatypes and collations.
*/
for (i = 0; i < natts; i++)
/* set stats detail level to a sane default */
for (int i = 0; i < natts; i++)
tupdesc->attrs[i].attstattarget = -1;
InsertPgAttributeTuples(rel, tupdesc, new_rel_oid, NULL, indstate);
/* add dependencies on their datatypes and collations */
for (int i = 0; i < natts; i++)
{
attr = TupleDescAttr(tupdesc, i);
/* Fill in the correct relation OID */
attr->attrelid = new_rel_oid;
/* Make sure this is OK, too */
attr->attstattarget = -1;
InsertPgAttributeTuple(rel, attr, (Datum) 0, indstate);
/* Add dependency info */
ObjectAddressSubSet(myself, RelationRelationId, new_rel_oid, i + 1);
ObjectAddressSet(referenced, TypeRelationId, attr->atttypid);
ObjectAddressSet(referenced, TypeRelationId,
tupdesc->attrs[i].atttypid);
recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL);
/* The default collation is pinned, so don't bother recording it */
if (OidIsValid(attr->attcollation) &&
attr->attcollation != DEFAULT_COLLATION_OID)
if (OidIsValid(tupdesc->attrs[i].attcollation) &&
tupdesc->attrs[i].attcollation != DEFAULT_COLLATION_OID)
{
ObjectAddressSet(referenced, CollationRelationId, attr->attcollation);
ObjectAddressSet(referenced, CollationRelationId,
tupdesc->attrs[i].attcollation);
recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL);
}
}
@ -838,17 +884,12 @@ AddNewAttributeTuples(Oid new_rel_oid,
*/
if (relkind != RELKIND_VIEW && relkind != RELKIND_COMPOSITE_TYPE)
{
for (i = 0; i < (int) lengthof(SysAtt); i++)
{
FormData_pg_attribute attStruct;
TupleDesc td;
memcpy(&attStruct, SysAtt[i], sizeof(FormData_pg_attribute));
td = CreateTupleDesc(lengthof(SysAtt), (FormData_pg_attribute **) &SysAtt);
/* Fill in the correct relation OID in the copied tuple */
attStruct.attrelid = new_rel_oid;
InsertPgAttributeTuple(rel, &attStruct, (Datum) 0, indstate);
}
InsertPgAttributeTuples(rel, td, new_rel_oid, NULL, indstate);
FreeTupleDesc(td);
}
/*

View File

@ -106,8 +106,7 @@ static TupleDesc ConstructTupleDescriptor(Relation heapRelation,
Oid *classObjectId);
static void InitializeAttributeOids(Relation indexRelation,
int numatts, Oid indexoid);
static void AppendAttributeTuples(Relation indexRelation, int numatts,
Datum *attopts);
static void AppendAttributeTuples(Relation indexRelation, Datum *attopts);
static void UpdateIndexRelation(Oid indexoid, Oid heapoid,
Oid parentIndexId,
IndexInfo *indexInfo,
@ -485,12 +484,11 @@ InitializeAttributeOids(Relation indexRelation,
* ----------------------------------------------------------------
*/
static void
AppendAttributeTuples(Relation indexRelation, int numatts, Datum *attopts)
AppendAttributeTuples(Relation indexRelation, Datum *attopts)
{
Relation pg_attribute;
CatalogIndexState indstate;
TupleDesc indexTupDesc;
int i;
/*
* open the attribute relation and its indexes
@ -504,15 +502,7 @@ AppendAttributeTuples(Relation indexRelation, int numatts, Datum *attopts)
*/
indexTupDesc = RelationGetDescr(indexRelation);
for (i = 0; i < numatts; i++)
{
Form_pg_attribute attr = TupleDescAttr(indexTupDesc, i);
Datum attoptions = attopts ? attopts[i] : (Datum) 0;
Assert(attr->attnum == i + 1);
InsertPgAttributeTuple(pg_attribute, attr, attoptions, indstate);
}
InsertPgAttributeTuples(pg_attribute, indexTupDesc, InvalidOid, attopts, indstate);
CatalogCloseIndexes(indstate);
@ -979,8 +969,7 @@ index_create(Relation heapRelation,
/*
* append ATTRIBUTE tuples for the index
*/
AppendAttributeTuples(indexRelation, indexInfo->ii_NumIndexAttrs,
indexInfo->ii_OpclassOptions);
AppendAttributeTuples(indexRelation, indexInfo->ii_OpclassOptions);
/* ----------------
* update pg_index

View File

@ -18,6 +18,7 @@
#include "access/genam.h"
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/xact.h"
#include "catalog/index.h"
#include "catalog/indexing.h"
#include "executor/executor.h"
@ -250,6 +251,41 @@ CatalogTupleInsertWithInfo(Relation heapRel, HeapTuple tup,
CatalogIndexInsert(indstate, tup);
}
/*
* CatalogTuplesMultiInsertWithInfo - as above, but for multiple tuples
*
* Insert multiple tuples into the given catalog relation at once, with an
* amortized cost of CatalogOpenIndexes.
*/
void
CatalogTuplesMultiInsertWithInfo(Relation heapRel, TupleTableSlot **slot,
int ntuples, CatalogIndexState indstate)
{
/* Nothing to do */
if (ntuples <= 0)
return;
heap_multi_insert(heapRel, slot, ntuples,
GetCurrentCommandId(true), 0, NULL);
/*
* There is no equivalent to heap_multi_insert for the catalog indexes, so
* we must loop over and insert individually.
*/
for (int i = 0; i < ntuples; i++)
{
bool should_free;
HeapTuple tuple;
tuple = ExecFetchSlotHeapTuple(slot[i], true, &should_free);
tuple->t_tableOid = slot[i]->tts_tableOid;
CatalogIndexInsert(indstate, tuple);
if (should_free)
heap_freetuple(tuple);
}
}
/*
* CatalogTupleUpdate - do heap and indexing work for updating a catalog tuple
*

View File

@ -785,6 +785,13 @@ checkSharedDependencies(Oid classId, Oid objectId,
return true;
}
/*
* Cap the maximum amount of bytes allocated for copyTemplateDependencies()
* slots.
*/
#define MAX_PGSHDEPEND_INSERT_BYTES 65535
/*
* copyTemplateDependencies
*
@ -799,14 +806,19 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId)
ScanKeyData key[1];
SysScanDesc scan;
HeapTuple tup;
int slotCount;
CatalogIndexState indstate;
Datum values[Natts_pg_shdepend];
bool nulls[Natts_pg_shdepend];
bool replace[Natts_pg_shdepend];
TupleTableSlot **slot;
int nslots;
sdepRel = table_open(SharedDependRelationId, RowExclusiveLock);
sdepDesc = RelationGetDescr(sdepRel);
nslots = MAX_PGSHDEPEND_INSERT_BYTES / sizeof(FormData_pg_shdepend);
slot = palloc(sizeof(TupleTableSlot *) * nslots);
for (int i = 0; i < nslots; i++)
slot[i] = MakeSingleTupleTableSlot(sdepDesc, &TTSOpsHeapTuple);
indstate = CatalogOpenIndexes(sdepRel);
/* Scan all entries with dbid = templateDbId */
@ -818,14 +830,6 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId)
scan = systable_beginscan(sdepRel, SharedDependDependerIndexId, true,
NULL, 1, key);
/* Set up to copy the tuples except for inserting newDbId */
memset(values, 0, sizeof(values));
memset(nulls, false, sizeof(nulls));
memset(replace, false, sizeof(replace));
replace[Anum_pg_shdepend_dbid - 1] = true;
values[Anum_pg_shdepend_dbid - 1] = ObjectIdGetDatum(newDbId);
/*
* Copy the entries of the original database, changing the database Id to
* that of the new database. Note that because we are not copying rows
@ -833,20 +837,46 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId)
* copy the ownership dependency of the template database itself; this is
* what we want.
*/
slotCount = 0;
while (HeapTupleIsValid(tup = systable_getnext(scan)))
{
HeapTuple newtup;
Form_pg_shdepend shdep;
newtup = heap_modify_tuple(tup, sdepDesc, values, nulls, replace);
CatalogTupleInsertWithInfo(sdepRel, newtup, indstate);
ExecClearTuple(slot[slotCount]);
heap_freetuple(newtup);
shdep = (Form_pg_shdepend) GETSTRUCT(tup);
slot[slotCount]->tts_values[Anum_pg_shdepend_dbid] = ObjectIdGetDatum(newDbId);
slot[slotCount]->tts_values[Anum_pg_shdepend_classid] = shdep->classid;
slot[slotCount]->tts_values[Anum_pg_shdepend_objid] = shdep->objid;
slot[slotCount]->tts_values[Anum_pg_shdepend_objsubid] = shdep->objsubid;
slot[slotCount]->tts_values[Anum_pg_shdepend_refclassid] = shdep->refclassid;
slot[slotCount]->tts_values[Anum_pg_shdepend_refobjid] = shdep->refobjid;
slot[slotCount]->tts_values[Anum_pg_shdepend_deptype] = shdep->deptype;
ExecStoreVirtualTuple(slot[slotCount]);
slotCount++;
/* If slots are full, insert a batch of tuples */
if (slotCount == nslots)
{
CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slotCount, indstate);
slotCount = 0;
}
}
/* Insert any tuples left in the buffer */
if (slotCount > 0)
CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slotCount, indstate);
systable_endscan(scan);
CatalogCloseIndexes(indstate);
table_close(sdepRel, RowExclusiveLock);
for (int i = 0; i < nslots; i++)
ExecDropSingleTupleTableSlot(slot[i]);
pfree(slot);
}
/*

View File

@ -5975,6 +5975,8 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel,
AlterTableCmd *childcmd;
AclResult aclresult;
ObjectAddress address;
TupleDesc tupdesc;
FormData_pg_attribute *aattr[] = {&attribute};
/* At top level, permission check was done in ATPrepCmd, else do it */
if (recursing)
@ -6128,11 +6130,13 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel,
attribute.attislocal = colDef->is_local;
attribute.attinhcount = colDef->inhcount;
attribute.attcollation = collOid;
/* attribute.attacl is handled by InsertPgAttributeTuple */
/* attribute.attacl is handled by InsertPgAttributeTuples() */
ReleaseSysCache(typeTuple);
InsertPgAttributeTuple(attrdesc, &attribute, (Datum) 0, NULL);
tupdesc = CreateTupleDesc(lengthof(aattr), (FormData_pg_attribute **) &aattr);
InsertPgAttributeTuples(attrdesc, tupdesc, myrelid, NULL, NULL);
table_close(attrdesc, RowExclusiveLock);

View File

@ -93,10 +93,11 @@ extern void heap_truncate_check_FKs(List *relations, bool tempTables);
extern List *heap_truncate_find_FKs(List *relationIds);
extern void InsertPgAttributeTuple(Relation pg_attribute_rel,
Form_pg_attribute new_attribute,
Datum attoptions,
CatalogIndexState indstate);
extern void InsertPgAttributeTuples(Relation pg_attribute_rel,
TupleDesc tupdesc,
Oid new_rel_oid,
Datum *attoptions,
CatalogIndexState indstate);
extern void InsertPgClassTuple(Relation pg_class_desc,
Relation new_rel_desc,

View File

@ -19,6 +19,7 @@
#define INDEXING_H
#include "access/htup.h"
#include "nodes/execnodes.h"
#include "utils/relcache.h"
/*
@ -36,6 +37,10 @@ extern void CatalogCloseIndexes(CatalogIndexState indstate);
extern void CatalogTupleInsert(Relation heapRel, HeapTuple tup);
extern void CatalogTupleInsertWithInfo(Relation heapRel, HeapTuple tup,
CatalogIndexState indstate);
extern void CatalogTuplesMultiInsertWithInfo(Relation heapRel,
TupleTableSlot **slot,
int ntuples,
CatalogIndexState indstate);
extern void CatalogTupleUpdate(Relation heapRel, ItemPointer otid,
HeapTuple tup);
extern void CatalogTupleUpdateWithInfo(Relation heapRel,