Fix CREATE INDEX progress reporting for multi-level partitioning.
The "partitions_total" and "partitions_done" fields were updated as though the current level of partitioning was the only one. In multi-level cases, not only could partitions_total change over the course of the command, but partitions_done could go backwards or exceed the currently-reported partitions_total. Fix by setting partitions_total to the total number of direct and indirect children once at command start, and then just incrementing partitions_done at appropriate points. Invent a new progress monitoring function "pgstat_progress_incr_param" to simplify doing the latter. We can avoid adding cost for the former when doing CREATE INDEX, because ProcessUtility already enumerates the children and it's pretty easy to pass the count down to DefineIndex. In principle the same could be done in ALTER TABLE, but that's structurally difficult; for now, just eat the cost of an extra find_all_inheritors scan in that case. Ilya Gladyshev and Justin Pryzby Discussion: https://postgr.es/m/a15f904a70924ffa4ca25c3c744cff31e0e6e143.camel@gmail.com
This commit is contained in:
parent
81a6d57e33
commit
27f5c712b2
|
@ -6817,7 +6817,7 @@ FROM pg_stat_get_backend_idset() AS backendid;
|
||||||
<structfield>pid</structfield> <type>integer</type>
|
<structfield>pid</structfield> <type>integer</type>
|
||||||
</para>
|
</para>
|
||||||
<para>
|
<para>
|
||||||
Process ID of backend.
|
Process ID of the backend creating indexes.
|
||||||
</para></entry>
|
</para></entry>
|
||||||
</row>
|
</row>
|
||||||
|
|
||||||
|
@ -6863,7 +6863,7 @@ FROM pg_stat_get_backend_idset() AS backendid;
|
||||||
<structfield>command</structfield> <type>text</type>
|
<structfield>command</structfield> <type>text</type>
|
||||||
</para>
|
</para>
|
||||||
<para>
|
<para>
|
||||||
The command that is running: <literal>CREATE INDEX</literal>,
|
Specific command type: <literal>CREATE INDEX</literal>,
|
||||||
<literal>CREATE INDEX CONCURRENTLY</literal>,
|
<literal>CREATE INDEX CONCURRENTLY</literal>,
|
||||||
<literal>REINDEX</literal>, or <literal>REINDEX CONCURRENTLY</literal>.
|
<literal>REINDEX</literal>, or <literal>REINDEX CONCURRENTLY</literal>.
|
||||||
</para></entry>
|
</para></entry>
|
||||||
|
@ -6946,9 +6946,10 @@ FROM pg_stat_get_backend_idset() AS backendid;
|
||||||
<structfield>partitions_total</structfield> <type>bigint</type>
|
<structfield>partitions_total</structfield> <type>bigint</type>
|
||||||
</para>
|
</para>
|
||||||
<para>
|
<para>
|
||||||
When creating an index on a partitioned table, this column is set to
|
Total number of partitions on which the index is to be created
|
||||||
the total number of partitions on which the index is to be created.
|
or attached, including both direct and indirect partitions.
|
||||||
This field is <literal>0</literal> during a <literal>REINDEX</literal>.
|
<literal>0</literal> during a <literal>REINDEX</literal>, or when
|
||||||
|
the index is not partitioned.
|
||||||
</para></entry>
|
</para></entry>
|
||||||
</row>
|
</row>
|
||||||
|
|
||||||
|
@ -6957,9 +6958,10 @@ FROM pg_stat_get_backend_idset() AS backendid;
|
||||||
<structfield>partitions_done</structfield> <type>bigint</type>
|
<structfield>partitions_done</structfield> <type>bigint</type>
|
||||||
</para>
|
</para>
|
||||||
<para>
|
<para>
|
||||||
When creating an index on a partitioned table, this column is set to
|
Number of partitions on which the index has already been created
|
||||||
the number of partitions on which the index has been created.
|
or attached, including both direct and indirect partitions.
|
||||||
This field is <literal>0</literal> during a <literal>REINDEX</literal>.
|
<literal>0</literal> during a <literal>REINDEX</literal>, or when
|
||||||
|
the index is not partitioned.
|
||||||
</para></entry>
|
</para></entry>
|
||||||
</row>
|
</row>
|
||||||
</tbody>
|
</tbody>
|
||||||
|
|
|
@ -306,6 +306,7 @@ Boot_DeclareIndexStmt:
|
||||||
$4,
|
$4,
|
||||||
InvalidOid,
|
InvalidOid,
|
||||||
InvalidOid,
|
InvalidOid,
|
||||||
|
-1,
|
||||||
false,
|
false,
|
||||||
false,
|
false,
|
||||||
false,
|
false,
|
||||||
|
@ -358,6 +359,7 @@ Boot_DeclareUniqueIndexStmt:
|
||||||
$5,
|
$5,
|
||||||
InvalidOid,
|
InvalidOid,
|
||||||
InvalidOid,
|
InvalidOid,
|
||||||
|
-1,
|
||||||
false,
|
false,
|
||||||
false,
|
false,
|
||||||
false,
|
false,
|
||||||
|
|
|
@ -513,6 +513,8 @@ WaitForOlderSnapshots(TransactionId limitXmin, bool progress)
|
||||||
* of a partitioned index.
|
* of a partitioned index.
|
||||||
* 'parentConstraintId': the OID of the parent constraint; InvalidOid if not
|
* 'parentConstraintId': the OID of the parent constraint; InvalidOid if not
|
||||||
* the child of a constraint (only used when recursing)
|
* the child of a constraint (only used when recursing)
|
||||||
|
* 'total_parts': total number of direct and indirect partitions of relation;
|
||||||
|
* pass -1 if not known or rel is not partitioned.
|
||||||
* 'is_alter_table': this is due to an ALTER rather than a CREATE operation.
|
* 'is_alter_table': this is due to an ALTER rather than a CREATE operation.
|
||||||
* 'check_rights': check for CREATE rights in namespace and tablespace. (This
|
* 'check_rights': check for CREATE rights in namespace and tablespace. (This
|
||||||
* should be true except when ALTER is deleting/recreating an index.)
|
* should be true except when ALTER is deleting/recreating an index.)
|
||||||
|
@ -530,6 +532,7 @@ DefineIndex(Oid relationId,
|
||||||
Oid indexRelationId,
|
Oid indexRelationId,
|
||||||
Oid parentIndexId,
|
Oid parentIndexId,
|
||||||
Oid parentConstraintId,
|
Oid parentConstraintId,
|
||||||
|
int total_parts,
|
||||||
bool is_alter_table,
|
bool is_alter_table,
|
||||||
bool check_rights,
|
bool check_rights,
|
||||||
bool check_not_in_use,
|
bool check_not_in_use,
|
||||||
|
@ -1225,8 +1228,37 @@ DefineIndex(Oid relationId,
|
||||||
Relation parentIndex;
|
Relation parentIndex;
|
||||||
TupleDesc parentDesc;
|
TupleDesc parentDesc;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Report the total number of partitions at the start of the
|
||||||
|
* command; don't update it when being called recursively.
|
||||||
|
*/
|
||||||
|
if (!OidIsValid(parentIndexId))
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
* When called by ProcessUtilitySlow, the number of partitions
|
||||||
|
* is passed in as an optimization; but other callers pass -1
|
||||||
|
* since they don't have the value handy. This should count
|
||||||
|
* partitions the same way, ie one less than the number of
|
||||||
|
* relations find_all_inheritors reports.
|
||||||
|
*
|
||||||
|
* We assume we needn't ask find_all_inheritors to take locks,
|
||||||
|
* because that should have happened already for all callers.
|
||||||
|
* Even if it did not, this is safe as long as we don't try to
|
||||||
|
* touch the partitions here; the worst consequence would be a
|
||||||
|
* bogus progress-reporting total.
|
||||||
|
*/
|
||||||
|
if (total_parts < 0)
|
||||||
|
{
|
||||||
|
List *children = find_all_inheritors(relationId,
|
||||||
|
NoLock, NULL);
|
||||||
|
|
||||||
|
total_parts = list_length(children) - 1;
|
||||||
|
list_free(children);
|
||||||
|
}
|
||||||
|
|
||||||
pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL,
|
pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL,
|
||||||
nparts);
|
total_parts);
|
||||||
|
}
|
||||||
|
|
||||||
/* Make a local copy of partdesc->oids[], just for safety */
|
/* Make a local copy of partdesc->oids[], just for safety */
|
||||||
memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts);
|
memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts);
|
||||||
|
@ -1353,6 +1385,18 @@ DefineIndex(Oid relationId,
|
||||||
invalidate_parent = true;
|
invalidate_parent = true;
|
||||||
|
|
||||||
found = true;
|
found = true;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Report this partition as processed. Note that if
|
||||||
|
* the partition has children itself, we'd ideally
|
||||||
|
* count the children and update the progress report
|
||||||
|
* for all of them; but that seems unduly expensive.
|
||||||
|
* Instead, the progress report will act like all such
|
||||||
|
* indirect children were processed in zero time at
|
||||||
|
* the end of the command.
|
||||||
|
*/
|
||||||
|
pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1);
|
||||||
|
|
||||||
/* keep lock till commit */
|
/* keep lock till commit */
|
||||||
index_close(cldidx, NoLock);
|
index_close(cldidx, NoLock);
|
||||||
break;
|
break;
|
||||||
|
@ -1432,14 +1476,13 @@ DefineIndex(Oid relationId,
|
||||||
InvalidOid, /* no predefined OID */
|
InvalidOid, /* no predefined OID */
|
||||||
indexRelationId, /* this is our child */
|
indexRelationId, /* this is our child */
|
||||||
createdConstraintId,
|
createdConstraintId,
|
||||||
|
-1,
|
||||||
is_alter_table, check_rights, check_not_in_use,
|
is_alter_table, check_rights, check_not_in_use,
|
||||||
skip_build, quiet);
|
skip_build, quiet);
|
||||||
SetUserIdAndSecContext(child_save_userid,
|
SetUserIdAndSecContext(child_save_userid,
|
||||||
child_save_sec_context);
|
child_save_sec_context);
|
||||||
}
|
}
|
||||||
|
|
||||||
pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
|
|
||||||
i + 1);
|
|
||||||
free_attrmap(attmap);
|
free_attrmap(attmap);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1479,6 +1522,12 @@ DefineIndex(Oid relationId,
|
||||||
table_close(rel, NoLock);
|
table_close(rel, NoLock);
|
||||||
if (!OidIsValid(parentIndexId))
|
if (!OidIsValid(parentIndexId))
|
||||||
pgstat_progress_end_command();
|
pgstat_progress_end_command();
|
||||||
|
else
|
||||||
|
{
|
||||||
|
/* Update progress for an intermediate partitioned index itself */
|
||||||
|
pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1);
|
||||||
|
}
|
||||||
|
|
||||||
return address;
|
return address;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1490,9 +1539,14 @@ DefineIndex(Oid relationId,
|
||||||
/* Close the heap and we're done, in the non-concurrent case */
|
/* Close the heap and we're done, in the non-concurrent case */
|
||||||
table_close(rel, NoLock);
|
table_close(rel, NoLock);
|
||||||
|
|
||||||
/* If this is the top-level index, we're done. */
|
/*
|
||||||
|
* If this is the top-level index, the command is done overall;
|
||||||
|
* otherwise, increment progress to report one child index is done.
|
||||||
|
*/
|
||||||
if (!OidIsValid(parentIndexId))
|
if (!OidIsValid(parentIndexId))
|
||||||
pgstat_progress_end_command();
|
pgstat_progress_end_command();
|
||||||
|
else
|
||||||
|
pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1);
|
||||||
|
|
||||||
return address;
|
return address;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1216,6 +1216,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId,
|
||||||
InvalidOid,
|
InvalidOid,
|
||||||
RelationGetRelid(idxRel),
|
RelationGetRelid(idxRel),
|
||||||
constraintOid,
|
constraintOid,
|
||||||
|
-1,
|
||||||
false, false, false, false, false);
|
false, false, false, false, false);
|
||||||
|
|
||||||
index_close(idxRel, AccessShareLock);
|
index_close(idxRel, AccessShareLock);
|
||||||
|
@ -8640,6 +8641,7 @@ ATExecAddIndex(AlteredTableInfo *tab, Relation rel,
|
||||||
InvalidOid, /* no predefined OID */
|
InvalidOid, /* no predefined OID */
|
||||||
InvalidOid, /* no parent index */
|
InvalidOid, /* no parent index */
|
||||||
InvalidOid, /* no parent constraint */
|
InvalidOid, /* no parent constraint */
|
||||||
|
-1, /* total_parts unknown */
|
||||||
true, /* is_alter_table */
|
true, /* is_alter_table */
|
||||||
check_rights,
|
check_rights,
|
||||||
false, /* check_not_in_use - we did it already */
|
false, /* check_not_in_use - we did it already */
|
||||||
|
@ -18106,6 +18108,7 @@ AttachPartitionEnsureIndexes(Relation rel, Relation attachrel)
|
||||||
DefineIndex(RelationGetRelid(attachrel), stmt, InvalidOid,
|
DefineIndex(RelationGetRelid(attachrel), stmt, InvalidOid,
|
||||||
RelationGetRelid(idxRel),
|
RelationGetRelid(idxRel),
|
||||||
conOid,
|
conOid,
|
||||||
|
-1,
|
||||||
true, false, false, false, false);
|
true, false, false, false, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1464,6 +1464,7 @@ ProcessUtilitySlow(ParseState *pstate,
|
||||||
IndexStmt *stmt = (IndexStmt *) parsetree;
|
IndexStmt *stmt = (IndexStmt *) parsetree;
|
||||||
Oid relid;
|
Oid relid;
|
||||||
LOCKMODE lockmode;
|
LOCKMODE lockmode;
|
||||||
|
int nparts = -1;
|
||||||
bool is_alter_table;
|
bool is_alter_table;
|
||||||
|
|
||||||
if (stmt->concurrent)
|
if (stmt->concurrent)
|
||||||
|
@ -1494,7 +1495,9 @@ ProcessUtilitySlow(ParseState *pstate,
|
||||||
*
|
*
|
||||||
* We also take the opportunity to verify that all
|
* We also take the opportunity to verify that all
|
||||||
* partitions are something we can put an index on, to
|
* partitions are something we can put an index on, to
|
||||||
* avoid building some indexes only to fail later.
|
* avoid building some indexes only to fail later. While
|
||||||
|
* at it, also count the partitions, so that DefineIndex
|
||||||
|
* needn't do a duplicative find_all_inheritors search.
|
||||||
*/
|
*/
|
||||||
if (stmt->relation->inh &&
|
if (stmt->relation->inh &&
|
||||||
get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE)
|
get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE)
|
||||||
|
@ -1505,7 +1508,8 @@ ProcessUtilitySlow(ParseState *pstate,
|
||||||
inheritors = find_all_inheritors(relid, lockmode, NULL);
|
inheritors = find_all_inheritors(relid, lockmode, NULL);
|
||||||
foreach(lc, inheritors)
|
foreach(lc, inheritors)
|
||||||
{
|
{
|
||||||
char relkind = get_rel_relkind(lfirst_oid(lc));
|
Oid partrelid = lfirst_oid(lc);
|
||||||
|
char relkind = get_rel_relkind(partrelid);
|
||||||
|
|
||||||
if (relkind != RELKIND_RELATION &&
|
if (relkind != RELKIND_RELATION &&
|
||||||
relkind != RELKIND_MATVIEW &&
|
relkind != RELKIND_MATVIEW &&
|
||||||
|
@ -1523,6 +1527,8 @@ ProcessUtilitySlow(ParseState *pstate,
|
||||||
errdetail("Table \"%s\" contains partitions that are foreign tables.",
|
errdetail("Table \"%s\" contains partitions that are foreign tables.",
|
||||||
stmt->relation->relname)));
|
stmt->relation->relname)));
|
||||||
}
|
}
|
||||||
|
/* count direct and indirect children, but not rel */
|
||||||
|
nparts = list_length(inheritors) - 1;
|
||||||
list_free(inheritors);
|
list_free(inheritors);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1548,6 +1554,7 @@ ProcessUtilitySlow(ParseState *pstate,
|
||||||
InvalidOid, /* no predefined OID */
|
InvalidOid, /* no predefined OID */
|
||||||
InvalidOid, /* no parent index */
|
InvalidOid, /* no parent index */
|
||||||
InvalidOid, /* no parent constraint */
|
InvalidOid, /* no parent constraint */
|
||||||
|
nparts, /* # of partitions, or -1 */
|
||||||
is_alter_table,
|
is_alter_table,
|
||||||
true, /* check_rights */
|
true, /* check_rights */
|
||||||
true, /* check_not_in_use */
|
true, /* check_not_in_use */
|
||||||
|
|
|
@ -58,6 +58,27 @@ pgstat_progress_update_param(int index, int64 val)
|
||||||
PGSTAT_END_WRITE_ACTIVITY(beentry);
|
PGSTAT_END_WRITE_ACTIVITY(beentry);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*-----------
|
||||||
|
* pgstat_progress_incr_param() -
|
||||||
|
*
|
||||||
|
* Increment index'th member in st_progress_param[] of own backend entry.
|
||||||
|
*-----------
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
pgstat_progress_incr_param(int index, int64 incr)
|
||||||
|
{
|
||||||
|
volatile PgBackendStatus *beentry = MyBEEntry;
|
||||||
|
|
||||||
|
Assert(index >= 0 && index < PGSTAT_NUM_PROGRESS_PARAM);
|
||||||
|
|
||||||
|
if (!beentry || !pgstat_track_activities)
|
||||||
|
return;
|
||||||
|
|
||||||
|
PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
|
||||||
|
beentry->st_progress_param[index] += incr;
|
||||||
|
PGSTAT_END_WRITE_ACTIVITY(beentry);
|
||||||
|
}
|
||||||
|
|
||||||
/*-----------
|
/*-----------
|
||||||
* pgstat_progress_update_multi_param() -
|
* pgstat_progress_update_multi_param() -
|
||||||
*
|
*
|
||||||
|
|
|
@ -29,6 +29,7 @@ extern ObjectAddress DefineIndex(Oid relationId,
|
||||||
Oid indexRelationId,
|
Oid indexRelationId,
|
||||||
Oid parentIndexId,
|
Oid parentIndexId,
|
||||||
Oid parentConstraintId,
|
Oid parentConstraintId,
|
||||||
|
int total_parts,
|
||||||
bool is_alter_table,
|
bool is_alter_table,
|
||||||
bool check_rights,
|
bool check_rights,
|
||||||
bool check_not_in_use,
|
bool check_not_in_use,
|
||||||
|
|
|
@ -36,6 +36,7 @@ typedef enum ProgressCommandType
|
||||||
extern void pgstat_progress_start_command(ProgressCommandType cmdtype,
|
extern void pgstat_progress_start_command(ProgressCommandType cmdtype,
|
||||||
Oid relid);
|
Oid relid);
|
||||||
extern void pgstat_progress_update_param(int index, int64 val);
|
extern void pgstat_progress_update_param(int index, int64 val);
|
||||||
|
extern void pgstat_progress_incr_param(int index, int64 incr);
|
||||||
extern void pgstat_progress_update_multi_param(int nparam, const int *index,
|
extern void pgstat_progress_update_multi_param(int nparam, const int *index,
|
||||||
const int64 *val);
|
const int64 *val);
|
||||||
extern void pgstat_progress_end_command(void);
|
extern void pgstat_progress_end_command(void);
|
||||||
|
|
Loading…
Reference in New Issue