Move parallel vacuum code to vacuumparallel.c.

This commit moves parallel vacuum related code to a new file
commands/vacuumparallel.c so that any table AM supporting indexes can
utilize parallel vacuum in order to call index AM callbacks (ambulkdelete
and amvacuumcleanup) with parallel workers.

Another reason for this refactoring is that the parallel vacuum isn't
specific to heap so it doesn't make sense to keep this code in
heap/vacuumlazy.c.

Author: Masahiko Sawada, based on suggestion from Andres Freund
Reviewed-by: Hou Zhijie, Amit Kapila, Haiying Tang
Discussion: https://www.postgresql.org/message-id/20211030212101.ae3qcouatwmy7tbr%40alap3.anarazel.de

src/backend/access/transam/parallel.c

@@ -14,7 +14,6 @@
 
 #include "postgres.h"
 
-#include "access/heapam.h"
 #include "access/nbtree.h"
 #include "access/parallel.h"
 #include "access/session.h"
@@ -25,6 +24,7 @@
 #include "catalog/pg_enum.h"
 #include "catalog/storage.h"
 #include "commands/async.h"
+#include "commands/vacuum.h"
 #include "executor/execParallel.h"
 #include "libpq/libpq.h"
 #include "libpq/pqformat.h"

src/backend/commands/Makefile

@@ -59,6 +59,7 @@ OBJS = \
 	typecmds.o \
 	user.o \
 	vacuum.o \
+	vacuumparallel.o \
 	variable.o \
 	view.o
 

src/backend/commands/vacuum.c

@@ -7,8 +7,9 @@
  * commands, (b) code to compute various vacuum thresholds, and (c) index
  * vacuum code.
  *
- * VACUUM for heap AM is implemented in vacuumlazy.c, ANALYZE in analyze.c, and
+ * VACUUM for heap AM is implemented in vacuumlazy.c, parallel vacuum in
- * VACUUM FULL is a variant of CLUSTER, handled in cluster.c.
+ * vacuumparallel.c, ANALYZE in analyze.c, and VACUUM FULL is a variant of
+ * CLUSTER, handled in cluster.c.
  *
  *
  * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group

src/include/access/heapam.h

@@ -198,7 +198,6 @@ extern void heap_get_root_tuples(Page page, OffsetNumber *root_offsets);
 struct VacuumParams;
 extern void heap_vacuum_rel(Relation rel,
 							struct VacuumParams *params, BufferAccessStrategy bstrategy);
-extern void parallel_vacuum_main(dsm_segment *seg, shm_toc *toc);
 
 /* in heap/heapam_visibility.c */
 extern bool HeapTupleSatisfiesVisibility(HeapTuple stup, Snapshot snapshot,

src/include/commands/vacuum.h

@@ -16,6 +16,7 @@
 
 #include "access/htup.h"
 #include "access/genam.h"
+#include "access/parallel.h"
 #include "catalog/pg_class.h"
 #include "catalog/pg_statistic.h"
 #include "catalog/pg_type.h"
@@ -63,6 +64,9 @@
 /* value for checking vacuum flags */
 #define VACUUM_OPTION_MAX_VALID_VALUE		((1 << 3) - 1)
 
+/* Abstract type for parallel vacuum state */
+typedef struct ParallelVacuumState ParallelVacuumState;
+
 /*----------
  * ANALYZE builds one of these structs for each attribute (column) that is
  * to be analyzed.  The struct and subsidiary data are in anl_context,
@@ -305,6 +309,22 @@ extern IndexBulkDeleteResult *vac_cleanup_one_index(IndexVacuumInfo *ivinfo,
 													IndexBulkDeleteResult *istat);
 extern Size vac_max_items_to_alloc_size(int max_items);
 
+/* in commands/vacuumparallel.c */
+extern ParallelVacuumState *parallel_vacuum_init(Relation rel, Relation *indrels,
+												 int nindexes, int nrequested_workers,
+												 int max_items, int elevel,
+												 BufferAccessStrategy bstrategy);
+extern void parallel_vacuum_end(ParallelVacuumState *pvs, IndexBulkDeleteResult **istats);
+extern VacDeadItems *parallel_vacuum_get_dead_items(ParallelVacuumState *pvs);
+extern void parallel_vacuum_bulkdel_all_indexes(ParallelVacuumState *pvs,
+												long num_table_tuples,
+												int num_index_scans);
+extern void parallel_vacuum_cleanup_all_indexes(ParallelVacuumState *pvs,
+												long num_table_tuples,
+												int num_index_scans,
+												bool estimated_count);
+extern void parallel_vacuum_main(dsm_segment *seg, shm_toc *toc);
+
 /* in commands/analyze.c */
 extern void analyze_rel(Oid relid, RangeVar *relation,
 						VacuumParams *params, List *va_cols, bool in_outer_xact,

src/tools/pgindent/typedefs.list

@@ -1306,13 +1306,8 @@ LPWSTR
 LSEG
 LUID
 LVPagePruneState
-LVParallelIndStats
-LVParallelIndVacStatus
-LVParallelState
 LVRelState
 LVSavedErrInfo
-LVShared
-LVSharedIndStats
 LWLock
 LWLockHandle
 LWLockMode
@@ -1775,7 +1770,10 @@ PTIterationArray
 PTOKEN_PRIVILEGES
 PTOKEN_USER
 PUTENVPROC
+PVIndStats
+PvIndVacStatus
 PVOID
+PVShared
 PX_Alias
 PX_Cipher
 PX_Combo
@@ -1809,6 +1807,7 @@ ParallelSlotResultHandler
 ParallelState
 ParallelTableScanDesc
 ParallelTableScanDescData
+ParallelVacuumState
 ParallelWorkerContext
 ParallelWorkerInfo
 Param