2017-01-19 18:00:00 +01:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* subscriptioncmds.c
|
|
|
|
* subscription catalog manipulation functions
|
|
|
|
*
|
2022-01-08 01:04:57 +01:00
|
|
|
* Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
|
2017-01-25 18:32:05 +01:00
|
|
|
* Portions Copyright (c) 1994, Regents of the University of California
|
2017-01-19 18:00:00 +01:00
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
|
|
|
* subscriptioncmds.c
|
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include "access/htup_details.h"
|
2019-01-21 19:18:20 +01:00
|
|
|
#include "access/table.h"
|
2017-03-04 05:25:34 +01:00
|
|
|
#include "access/xact.h"
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
#include "catalog/catalog.h"
|
2017-03-23 13:36:36 +01:00
|
|
|
#include "catalog/dependency.h"
|
2017-01-19 18:00:00 +01:00
|
|
|
#include "catalog/indexing.h"
|
2017-03-23 13:36:36 +01:00
|
|
|
#include "catalog/namespace.h"
|
2017-01-19 18:00:00 +01:00
|
|
|
#include "catalog/objectaccess.h"
|
|
|
|
#include "catalog/objectaddress.h"
|
|
|
|
#include "catalog/pg_subscription.h"
|
2017-03-23 13:36:36 +01:00
|
|
|
#include "catalog/pg_subscription_rel.h"
|
2019-11-12 04:00:16 +01:00
|
|
|
#include "catalog/pg_type.h"
|
2017-01-19 18:00:00 +01:00
|
|
|
#include "commands/defrem.h"
|
|
|
|
#include "commands/event_trigger.h"
|
|
|
|
#include "commands/subscriptioncmds.h"
|
2017-05-17 04:57:16 +02:00
|
|
|
#include "executor/executor.h"
|
2019-11-12 04:00:16 +01:00
|
|
|
#include "miscadmin.h"
|
2017-03-23 13:36:36 +01:00
|
|
|
#include "nodes/makefuncs.h"
|
2021-11-30 04:24:30 +01:00
|
|
|
#include "pgstat.h"
|
2017-01-19 18:00:00 +01:00
|
|
|
#include "replication/logicallauncher.h"
|
|
|
|
#include "replication/origin.h"
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
#include "replication/slot.h"
|
2017-01-19 18:00:00 +01:00
|
|
|
#include "replication/walreceiver.h"
|
2017-03-23 13:36:36 +01:00
|
|
|
#include "replication/walsender.h"
|
2017-01-19 18:00:00 +01:00
|
|
|
#include "replication/worker_internal.h"
|
|
|
|
#include "storage/lmgr.h"
|
2020-03-10 10:22:52 +01:00
|
|
|
#include "utils/acl.h"
|
2017-01-19 18:00:00 +01:00
|
|
|
#include "utils/builtins.h"
|
2017-04-14 19:58:46 +02:00
|
|
|
#include "utils/guc.h"
|
2017-03-23 13:36:36 +01:00
|
|
|
#include "utils/lsyscache.h"
|
2017-01-19 18:00:00 +01:00
|
|
|
#include "utils/memutils.h"
|
|
|
|
#include "utils/syscache.h"
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
/*
|
|
|
|
* Options that can be specified by the user in CREATE/ALTER SUBSCRIPTION
|
|
|
|
* command.
|
|
|
|
*/
|
|
|
|
#define SUBOPT_CONNECT 0x00000001
|
|
|
|
#define SUBOPT_ENABLED 0x00000002
|
|
|
|
#define SUBOPT_CREATE_SLOT 0x00000004
|
|
|
|
#define SUBOPT_SLOT_NAME 0x00000008
|
|
|
|
#define SUBOPT_COPY_DATA 0x00000010
|
|
|
|
#define SUBOPT_SYNCHRONOUS_COMMIT 0x00000020
|
|
|
|
#define SUBOPT_REFRESH 0x00000040
|
|
|
|
#define SUBOPT_BINARY 0x00000080
|
|
|
|
#define SUBOPT_STREAMING 0x00000100
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
#define SUBOPT_TWOPHASE_COMMIT 0x00000200
|
2021-07-06 04:16:50 +02:00
|
|
|
|
|
|
|
/* check if the 'val' has 'bits' set */
|
|
|
|
#define IsSet(val, bits) (((val) & (bits)) == (bits))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Structure to hold a bitmap representing the user-provided CREATE/ALTER
|
|
|
|
* SUBSCRIPTION command options and the parsed/default values of each of them.
|
|
|
|
*/
|
|
|
|
typedef struct SubOpts
|
|
|
|
{
|
|
|
|
bits32 specified_opts;
|
|
|
|
char *slot_name;
|
|
|
|
char *synchronous_commit;
|
|
|
|
bool connect;
|
|
|
|
bool enabled;
|
|
|
|
bool create_slot;
|
|
|
|
bool copy_data;
|
|
|
|
bool refresh;
|
|
|
|
bool binary;
|
|
|
|
bool streaming;
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
bool twophase;
|
2021-07-06 04:16:50 +02:00
|
|
|
} SubOpts;
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
static List *fetch_table_list(WalReceiverConn *wrconn, List *publications);
|
2021-04-06 10:44:26 +02:00
|
|
|
static void check_duplicates_in_publist(List *publist, Datum *datums);
|
|
|
|
static List *merge_publications(List *oldpublist, List *newpublist, bool addpub, const char *subname);
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err);
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
/*
|
|
|
|
* Common option parsing function for CREATE and ALTER SUBSCRIPTION commands.
|
|
|
|
*
|
|
|
|
* Since not all options can be specified in both commands, this function
|
2021-07-06 04:16:50 +02:00
|
|
|
* will report an error if mutually exclusive options are specified.
|
2017-01-19 18:00:00 +01:00
|
|
|
*/
|
|
|
|
static void
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
parse_subscription_options(ParseState *pstate, List *stmt_options,
|
|
|
|
bits32 supported_opts, SubOpts *opts)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
|
|
|
ListCell *lc;
|
2017-05-15 19:59:58 +02:00
|
|
|
|
2021-12-08 04:36:31 +01:00
|
|
|
/* Start out with cleared opts. */
|
|
|
|
memset(opts, 0, sizeof(SubOpts));
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
/* caller must expect some option */
|
|
|
|
Assert(supported_opts != 0);
|
|
|
|
|
|
|
|
/* If connect option is supported, these others also need to be. */
|
|
|
|
Assert(!IsSet(supported_opts, SUBOPT_CONNECT) ||
|
|
|
|
IsSet(supported_opts, SUBOPT_ENABLED | SUBOPT_CREATE_SLOT |
|
|
|
|
SUBOPT_COPY_DATA));
|
|
|
|
|
|
|
|
/* Set default values for the boolean supported options. */
|
|
|
|
if (IsSet(supported_opts, SUBOPT_CONNECT))
|
|
|
|
opts->connect = true;
|
|
|
|
if (IsSet(supported_opts, SUBOPT_ENABLED))
|
|
|
|
opts->enabled = true;
|
|
|
|
if (IsSet(supported_opts, SUBOPT_CREATE_SLOT))
|
|
|
|
opts->create_slot = true;
|
|
|
|
if (IsSet(supported_opts, SUBOPT_COPY_DATA))
|
|
|
|
opts->copy_data = true;
|
|
|
|
if (IsSet(supported_opts, SUBOPT_REFRESH))
|
|
|
|
opts->refresh = true;
|
|
|
|
if (IsSet(supported_opts, SUBOPT_BINARY))
|
|
|
|
opts->binary = false;
|
|
|
|
if (IsSet(supported_opts, SUBOPT_STREAMING))
|
|
|
|
opts->streaming = false;
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
if (IsSet(supported_opts, SUBOPT_TWOPHASE_COMMIT))
|
|
|
|
opts->twophase = false;
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
/* Parse options */
|
2021-07-06 04:16:50 +02:00
|
|
|
foreach(lc, stmt_options)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
|
|
|
DefElem *defel = (DefElem *) lfirst(lc);
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(supported_opts, SUBOPT_CONNECT) &&
|
|
|
|
strcmp(defel->defname, "connect") == 0)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_CONNECT))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_CONNECT;
|
|
|
|
opts->connect = defGetBoolean(defel);
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
2021-07-06 04:16:50 +02:00
|
|
|
else if (IsSet(supported_opts, SUBOPT_ENABLED) &&
|
|
|
|
strcmp(defel->defname, "enabled") == 0)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_ENABLED))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_ENABLED;
|
|
|
|
opts->enabled = defGetBoolean(defel);
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
2021-07-06 04:16:50 +02:00
|
|
|
else if (IsSet(supported_opts, SUBOPT_CREATE_SLOT) &&
|
|
|
|
strcmp(defel->defname, "create_slot") == 0)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_CREATE_SLOT))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_CREATE_SLOT;
|
|
|
|
opts->create_slot = defGetBoolean(defel);
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
2021-07-06 04:16:50 +02:00
|
|
|
else if (IsSet(supported_opts, SUBOPT_SLOT_NAME) &&
|
|
|
|
strcmp(defel->defname, "slot_name") == 0)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_SLOT_NAME))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_SLOT_NAME;
|
|
|
|
opts->slot_name = defGetString(defel);
|
2017-05-09 16:20:42 +02:00
|
|
|
|
|
|
|
/* Setting slot_name = NONE is treated as no slot name. */
|
2021-07-06 04:16:50 +02:00
|
|
|
if (strcmp(opts->slot_name, "none") == 0)
|
|
|
|
opts->slot_name = NULL;
|
2021-07-19 07:06:15 +02:00
|
|
|
else
|
|
|
|
ReplicationSlotValidateName(opts->slot_name, ERROR);
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
2021-07-06 04:16:50 +02:00
|
|
|
else if (IsSet(supported_opts, SUBOPT_COPY_DATA) &&
|
|
|
|
strcmp(defel->defname, "copy_data") == 0)
|
2017-03-23 13:36:36 +01:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_COPY_DATA))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_COPY_DATA;
|
|
|
|
opts->copy_data = defGetBoolean(defel);
|
2017-03-23 13:36:36 +01:00
|
|
|
}
|
2021-07-06 04:16:50 +02:00
|
|
|
else if (IsSet(supported_opts, SUBOPT_SYNCHRONOUS_COMMIT) &&
|
|
|
|
strcmp(defel->defname, "synchronous_commit") == 0)
|
2017-04-14 19:58:46 +02:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_SYNCHRONOUS_COMMIT))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
2017-04-14 19:58:46 +02:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_SYNCHRONOUS_COMMIT;
|
|
|
|
opts->synchronous_commit = defGetString(defel);
|
2017-04-14 19:58:46 +02:00
|
|
|
|
|
|
|
/* Test if the given value is valid for synchronous_commit GUC. */
|
2021-07-06 04:16:50 +02:00
|
|
|
(void) set_config_option("synchronous_commit", opts->synchronous_commit,
|
2017-04-14 19:58:46 +02:00
|
|
|
PGC_BACKEND, PGC_S_TEST, GUC_ACTION_SET,
|
|
|
|
false, 0, false);
|
|
|
|
}
|
2021-07-06 04:16:50 +02:00
|
|
|
else if (IsSet(supported_opts, SUBOPT_REFRESH) &&
|
|
|
|
strcmp(defel->defname, "refresh") == 0)
|
2017-06-06 03:37:00 +02:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_REFRESH))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
2017-06-06 03:37:00 +02:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_REFRESH;
|
|
|
|
opts->refresh = defGetBoolean(defel);
|
2017-06-06 03:37:00 +02:00
|
|
|
}
|
2021-07-06 04:16:50 +02:00
|
|
|
else if (IsSet(supported_opts, SUBOPT_BINARY) &&
|
|
|
|
strcmp(defel->defname, "binary") == 0)
|
2020-07-18 18:44:51 +02:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_BINARY))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
2020-07-18 18:44:51 +02:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_BINARY;
|
|
|
|
opts->binary = defGetBoolean(defel);
|
2020-07-18 18:44:51 +02:00
|
|
|
}
|
2021-07-06 04:16:50 +02:00
|
|
|
else if (IsSet(supported_opts, SUBOPT_STREAMING) &&
|
|
|
|
strcmp(defel->defname, "streaming") == 0)
|
Add support for streaming to built-in logical replication.
To add support for streaming of in-progress transactions into the
built-in logical replication, we need to do three things:
* Extend the logical replication protocol, so identify in-progress
transactions, and allow adding additional bits of information (e.g.
XID of subtransactions).
* Modify the output plugin (pgoutput) to implement the new stream
API callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle streamed
in-progress transaction by spilling the data to disk and then
replaying them on commit.
We however must explicitly disable streaming replication during
replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover we don't have a replication connection open so we
don't have where to send the data anyway.
Author: Tomas Vondra, Dilip Kumar and Amit Kapila
Reviewed-by: Amit Kapila, Kuntal Ghosh and Ajin Cherian
Tested-by: Neha Sharma, Mahendra Singh Thalor and Ajin Cherian
Discussion: https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com
2020-09-03 04:24:07 +02:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_STREAMING))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
Add support for streaming to built-in logical replication.
To add support for streaming of in-progress transactions into the
built-in logical replication, we need to do three things:
* Extend the logical replication protocol, so identify in-progress
transactions, and allow adding additional bits of information (e.g.
XID of subtransactions).
* Modify the output plugin (pgoutput) to implement the new stream
API callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle streamed
in-progress transaction by spilling the data to disk and then
replaying them on commit.
We however must explicitly disable streaming replication during
replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover we don't have a replication connection open so we
don't have where to send the data anyway.
Author: Tomas Vondra, Dilip Kumar and Amit Kapila
Reviewed-by: Amit Kapila, Kuntal Ghosh and Ajin Cherian
Tested-by: Neha Sharma, Mahendra Singh Thalor and Ajin Cherian
Discussion: https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com
2020-09-03 04:24:07 +02:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->specified_opts |= SUBOPT_STREAMING;
|
|
|
|
opts->streaming = defGetBoolean(defel);
|
Add support for streaming to built-in logical replication.
To add support for streaming of in-progress transactions into the
built-in logical replication, we need to do three things:
* Extend the logical replication protocol, so identify in-progress
transactions, and allow adding additional bits of information (e.g.
XID of subtransactions).
* Modify the output plugin (pgoutput) to implement the new stream
API callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle streamed
in-progress transaction by spilling the data to disk and then
replaying them on commit.
We however must explicitly disable streaming replication during
replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover we don't have a replication connection open so we
don't have where to send the data anyway.
Author: Tomas Vondra, Dilip Kumar and Amit Kapila
Reviewed-by: Amit Kapila, Kuntal Ghosh and Ajin Cherian
Tested-by: Neha Sharma, Mahendra Singh Thalor and Ajin Cherian
Discussion: https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com
2020-09-03 04:24:07 +02:00
|
|
|
}
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
else if (strcmp(defel->defname, "two_phase") == 0)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Do not allow toggling of two_phase option. Doing so could cause
|
|
|
|
* missing of transactions and lead to an inconsistent replica.
|
|
|
|
* See comments atop worker.c
|
|
|
|
*
|
|
|
|
* Note: Unsupported twophase indicates that this call originated
|
|
|
|
* from AlterSubscription.
|
|
|
|
*/
|
|
|
|
if (!IsSet(supported_opts, SUBOPT_TWOPHASE_COMMIT))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
errmsg("unrecognized subscription parameter: \"%s\"", defel->defname)));
|
|
|
|
|
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_TWOPHASE_COMMIT))
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
errorConflictingDefElem(defel, pstate);
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
|
|
|
|
opts->specified_opts |= SUBOPT_TWOPHASE_COMMIT;
|
|
|
|
opts->twophase = defGetBoolean(defel);
|
|
|
|
}
|
2017-01-19 18:00:00 +01:00
|
|
|
else
|
2017-05-18 02:47:37 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2019-05-17 00:50:56 +02:00
|
|
|
errmsg("unrecognized subscription parameter: \"%s\"", defel->defname)));
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
2017-03-23 13:36:36 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We've been explicitly asked to not connect, that requires some
|
|
|
|
* additional processing.
|
|
|
|
*/
|
2021-07-06 04:16:50 +02:00
|
|
|
if (!opts->connect && IsSet(supported_opts, SUBOPT_CONNECT))
|
2017-03-23 13:36:36 +01:00
|
|
|
{
|
|
|
|
/* Check for incompatible options from the user. */
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts->enabled &&
|
|
|
|
IsSet(opts->specified_opts, SUBOPT_ENABLED))
|
2017-03-23 13:36:36 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2019-05-17 00:50:56 +02:00
|
|
|
/*- translator: both %s are strings of the form "option = value" */
|
|
|
|
errmsg("%s and %s are mutually exclusive options",
|
|
|
|
"connect = false", "enabled = true")));
|
2017-03-23 13:36:36 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts->create_slot &&
|
|
|
|
IsSet(opts->specified_opts, SUBOPT_CREATE_SLOT))
|
2017-03-23 13:36:36 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2019-05-17 00:50:56 +02:00
|
|
|
errmsg("%s and %s are mutually exclusive options",
|
|
|
|
"connect = false", "create_slot = true")));
|
2017-03-23 13:36:36 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts->copy_data &&
|
|
|
|
IsSet(opts->specified_opts, SUBOPT_COPY_DATA))
|
2017-03-23 13:36:36 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2019-05-17 00:50:56 +02:00
|
|
|
errmsg("%s and %s are mutually exclusive options",
|
|
|
|
"connect = false", "copy_data = true")));
|
2017-03-23 13:36:36 +01:00
|
|
|
|
|
|
|
/* Change the defaults of other options. */
|
2021-07-06 04:16:50 +02:00
|
|
|
opts->enabled = false;
|
|
|
|
opts->create_slot = false;
|
|
|
|
opts->copy_data = false;
|
2017-03-23 13:36:36 +01:00
|
|
|
}
|
2017-05-09 16:20:42 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Do additional checking for disallowed combination when slot_name = NONE
|
|
|
|
* was used.
|
|
|
|
*/
|
2021-07-06 04:16:50 +02:00
|
|
|
if (!opts->slot_name &&
|
|
|
|
IsSet(opts->specified_opts, SUBOPT_SLOT_NAME))
|
2017-05-09 16:20:42 +02:00
|
|
|
{
|
2021-12-08 04:36:31 +01:00
|
|
|
if (opts->enabled)
|
|
|
|
{
|
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_ENABLED))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
/*- translator: both %s are strings of the form "option = value" */
|
|
|
|
errmsg("%s and %s are mutually exclusive options",
|
|
|
|
"slot_name = NONE", "enabled = true")));
|
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
/*- translator: both %s are strings of the form "option = value" */
|
|
|
|
errmsg("subscription with %s must also set %s",
|
|
|
|
"slot_name = NONE", "enabled = false")));
|
|
|
|
}
|
2017-05-18 02:47:37 +02:00
|
|
|
|
2021-12-08 04:36:31 +01:00
|
|
|
if (opts->create_slot)
|
|
|
|
{
|
|
|
|
if (IsSet(opts->specified_opts, SUBOPT_CREATE_SLOT))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
/*- translator: both %s are strings of the form "option = value" */
|
|
|
|
errmsg("%s and %s are mutually exclusive options",
|
|
|
|
"slot_name = NONE", "create_slot = true")));
|
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
/*- translator: both %s are strings of the form "option = value" */
|
|
|
|
errmsg("subscription with %s must also set %s",
|
|
|
|
"slot_name = NONE", "create_slot = false")));
|
|
|
|
}
|
2017-05-09 16:20:42 +02:00
|
|
|
}
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2017-09-23 21:16:48 +02:00
|
|
|
* Auxiliary function to build a text array out of a list of String nodes.
|
2017-01-19 18:00:00 +01:00
|
|
|
*/
|
|
|
|
static Datum
|
|
|
|
publicationListToArray(List *publist)
|
|
|
|
{
|
|
|
|
ArrayType *arr;
|
|
|
|
Datum *datums;
|
|
|
|
MemoryContext memcxt;
|
|
|
|
MemoryContext oldcxt;
|
|
|
|
|
|
|
|
/* Create memory context for temporary allocations. */
|
|
|
|
memcxt = AllocSetContextCreate(CurrentMemoryContext,
|
|
|
|
"publicationListToArray to array",
|
Rethink MemoryContext creation to improve performance.
This patch makes a number of interrelated changes to reduce the overhead
involved in creating/deleting memory contexts. The key ideas are:
* Include the AllocSetContext header of an aset.c context in its first
malloc request, rather than allocating it separately in TopMemoryContext.
This means that we now always create an initial or "keeper" block in an
aset, even if it never receives any allocation requests.
* Create freelists in which we can save and recycle recently-destroyed
asets (this idea is due to Robert Haas).
* In the common case where the name of a context is a constant string,
just store a pointer to it in the context header, rather than copying
the string.
The first change eliminates a palloc/pfree cycle per context, and
also avoids bloat in TopMemoryContext, at the price that creating
a context now involves a malloc/free cycle even if the context never
receives any allocations. That would be a loser for some common
usage patterns, but recycling short-lived contexts via the freelist
eliminates that pain.
Avoiding copying constant strings not only saves strlen() and strcpy()
overhead, but is an essential part of the freelist optimization because
it makes the context header size constant. Currently we make no
attempt to use the freelist for contexts with non-constant names.
(Perhaps someday we'll need to think harder about that, but in current
usage, most contexts with custom names are long-lived anyway.)
The freelist management in this initial commit is pretty simplistic,
and we might want to refine it later --- but in common workloads that
will never matter because the freelists will never get full anyway.
To create a context with a non-constant name, one is now required to
call AllocSetContextCreateExtended and specify the MEMCONTEXT_COPY_NAME
option. AllocSetContextCreate becomes a wrapper macro, and it includes
a test that will complain about non-string-literal context name
parameters on gcc and similar compilers.
An unfortunate side effect of making AllocSetContextCreate a macro is
that one is now *required* to use the size parameter abstraction macros
(ALLOCSET_DEFAULT_SIZES and friends) with it; the pre-9.6 habit of
writing out individual size parameters no longer works unless you
switch to AllocSetContextCreateExtended.
Internally to the memory-context-related modules, the context creation
APIs are simplified, removing the rather baroque original design whereby
a context-type module called mcxt.c which then called back into the
context-type module. That saved a bit of code duplication, but not much,
and it prevented context-type modules from exercising control over the
allocation of context headers.
In passing, I converted the test-and-elog validation of aset size
parameters into Asserts to save a few more cycles. The original thought
was that callers might compute size parameters on the fly, but in practice
nobody does that, so it's useless to expend cycles on checking those
numbers in production builds.
Also, mark the memory context method-pointer structs "const",
just for cleanliness.
Discussion: https://postgr.es/m/2264.1512870796@sss.pgh.pa.us
2017-12-13 19:55:12 +01:00
|
|
|
ALLOCSET_DEFAULT_SIZES);
|
2017-01-19 18:00:00 +01:00
|
|
|
oldcxt = MemoryContextSwitchTo(memcxt);
|
|
|
|
|
2017-09-23 21:16:48 +02:00
|
|
|
datums = (Datum *) palloc(sizeof(Datum) * list_length(publist));
|
|
|
|
|
2021-04-06 10:44:26 +02:00
|
|
|
check_duplicates_in_publist(publist, datums);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
MemoryContextSwitchTo(oldcxt);
|
|
|
|
|
|
|
|
arr = construct_array(datums, list_length(publist),
|
2020-03-04 16:34:25 +01:00
|
|
|
TEXTOID, -1, false, TYPALIGN_INT);
|
2017-09-23 21:16:48 +02:00
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
MemoryContextDelete(memcxt);
|
|
|
|
|
|
|
|
return PointerGetDatum(arr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create new subscription.
|
|
|
|
*/
|
|
|
|
ObjectAddress
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
|
|
|
|
bool isTopLevel)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
|
|
|
Relation rel;
|
|
|
|
ObjectAddress myself;
|
|
|
|
Oid subid;
|
|
|
|
bool nulls[Natts_pg_subscription];
|
|
|
|
Datum values[Natts_pg_subscription];
|
2017-01-20 20:45:02 +01:00
|
|
|
Oid owner = GetUserId();
|
2017-01-19 18:00:00 +01:00
|
|
|
HeapTuple tup;
|
|
|
|
char *conninfo;
|
|
|
|
char originname[NAMEDATALEN];
|
|
|
|
List *publications;
|
2021-07-06 04:16:50 +02:00
|
|
|
bits32 supported_opts;
|
|
|
|
SubOpts opts = {0};
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-03-04 05:25:34 +01:00
|
|
|
/*
|
|
|
|
* Parse and check options.
|
2017-05-17 02:36:35 +02:00
|
|
|
*
|
2017-03-04 05:25:34 +01:00
|
|
|
* Connection and publication should not be specified here.
|
|
|
|
*/
|
2021-07-06 04:16:50 +02:00
|
|
|
supported_opts = (SUBOPT_CONNECT | SUBOPT_ENABLED | SUBOPT_CREATE_SLOT |
|
|
|
|
SUBOPT_SLOT_NAME | SUBOPT_COPY_DATA |
|
|
|
|
SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY |
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT);
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
parse_subscription_options(pstate, stmt->options, supported_opts, &opts);
|
2017-03-04 05:25:34 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Since creating a replication slot is not transactional, rolling back
|
|
|
|
* the transaction leaves the created replication slot. So we cannot run
|
|
|
|
* CREATE SUBSCRIPTION inside a transaction block if creating a
|
|
|
|
* replication slot.
|
|
|
|
*/
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.create_slot)
|
2018-02-17 02:44:15 +01:00
|
|
|
PreventInTransactionBlock(isTopLevel, "CREATE SUBSCRIPTION ... WITH (create_slot = true)");
|
2017-03-04 05:25:34 +01:00
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
if (!superuser())
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
|
2020-01-30 17:32:04 +01:00
|
|
|
errmsg("must be superuser to create subscriptions")));
|
2017-01-19 18:00:00 +01:00
|
|
|
|
Add an enforcement mechanism for global object names in regression tests.
In commit 18555b132 we tentatively established a rule that regression
tests should use names containing "regression" for databases, and names
starting with "regress_" for all other globally-visible object names, so
as to circumscribe the side-effects that "make installcheck" could have
on an existing installation.
This commit adds a simple enforcement mechanism for that rule: if the code
is compiled with ENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS defined, it
will emit a warning (not an error) whenever a database, role, tablespace,
subscription, or replication origin name is created that doesn't obey the
rule. Running one or more buildfarm members with that symbol defined
should be enough to catch new violations, at least in the regular
regression tests. Most TAP tests wouldn't notice such warnings, but
that's actually fine because TAP tests don't execute against an existing
server anyway.
Since it's already the case that running src/test/modules/ tests in
installcheck mode is deprecated, we can use that as a home for tests
that seem unsafe to run against an existing server, such as tests that
might have side-effects on existing roles. Document that (though this
commit doesn't in itself make it any less safe than before).
Update regress.sgml to define these restrictions more clearly, and
to clean up assorted lack-of-up-to-date-ness in its descriptions of
the available regression tests.
Discussion: https://postgr.es/m/16638.1468620817@sss.pgh.pa.us
2019-06-29 17:34:00 +02:00
|
|
|
/*
|
|
|
|
* If built with appropriate switch, whine when regression-testing
|
|
|
|
* conventions for subscription names are violated.
|
|
|
|
*/
|
|
|
|
#ifdef ENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
|
|
|
|
if (strncmp(stmt->subname, "regress_", 8) != 0)
|
|
|
|
elog(WARNING, "subscriptions created by regression test cases should have names starting with \"regress_\"");
|
|
|
|
#endif
|
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
rel = table_open(SubscriptionRelationId, RowExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
/* Check if name is used */
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
subid = GetSysCacheOid2(SUBSCRIPTIONNAME, Anum_pg_subscription_oid,
|
|
|
|
MyDatabaseId, CStringGetDatum(stmt->subname));
|
2017-01-19 18:00:00 +01:00
|
|
|
if (OidIsValid(subid))
|
|
|
|
{
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_DUPLICATE_OBJECT),
|
|
|
|
errmsg("subscription \"%s\" already exists",
|
|
|
|
stmt->subname)));
|
|
|
|
}
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (!IsSet(opts.specified_opts, SUBOPT_SLOT_NAME) &&
|
|
|
|
opts.slot_name == NULL)
|
|
|
|
opts.slot_name = stmt->subname;
|
2017-05-09 16:20:42 +02:00
|
|
|
|
2017-04-14 19:58:46 +02:00
|
|
|
/* The default for synchronous_commit of subscriptions is off. */
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.synchronous_commit == NULL)
|
|
|
|
opts.synchronous_commit = "off";
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
conninfo = stmt->conninfo;
|
|
|
|
publications = stmt->publication;
|
|
|
|
|
|
|
|
/* Load the library providing us libpq calls. */
|
|
|
|
load_file("libpqwalreceiver", false);
|
|
|
|
|
|
|
|
/* Check the connection info string. */
|
|
|
|
walrcv_check_conninfo(conninfo);
|
|
|
|
|
|
|
|
/* Everything ok, form a new tuple. */
|
|
|
|
memset(values, 0, sizeof(values));
|
|
|
|
memset(nulls, false, sizeof(nulls));
|
|
|
|
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
subid = GetNewOidWithIndex(rel, SubscriptionObjectIndexId,
|
|
|
|
Anum_pg_subscription_oid);
|
|
|
|
values[Anum_pg_subscription_oid - 1] = ObjectIdGetDatum(subid);
|
2017-01-19 18:00:00 +01:00
|
|
|
values[Anum_pg_subscription_subdbid - 1] = ObjectIdGetDatum(MyDatabaseId);
|
|
|
|
values[Anum_pg_subscription_subname - 1] =
|
|
|
|
DirectFunctionCall1(namein, CStringGetDatum(stmt->subname));
|
2017-01-20 20:45:02 +01:00
|
|
|
values[Anum_pg_subscription_subowner - 1] = ObjectIdGetDatum(owner);
|
2021-07-06 04:16:50 +02:00
|
|
|
values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(opts.enabled);
|
|
|
|
values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary);
|
|
|
|
values[Anum_pg_subscription_substream - 1] = BoolGetDatum(opts.streaming);
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
values[Anum_pg_subscription_subtwophasestate - 1] =
|
|
|
|
CharGetDatum(opts.twophase ?
|
|
|
|
LOGICALREP_TWOPHASE_STATE_PENDING :
|
|
|
|
LOGICALREP_TWOPHASE_STATE_DISABLED);
|
2017-01-19 18:00:00 +01:00
|
|
|
values[Anum_pg_subscription_subconninfo - 1] =
|
|
|
|
CStringGetTextDatum(conninfo);
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.slot_name)
|
2017-05-09 16:20:42 +02:00
|
|
|
values[Anum_pg_subscription_subslotname - 1] =
|
2021-07-06 04:16:50 +02:00
|
|
|
DirectFunctionCall1(namein, CStringGetDatum(opts.slot_name));
|
2017-05-09 16:20:42 +02:00
|
|
|
else
|
|
|
|
nulls[Anum_pg_subscription_subslotname - 1] = true;
|
2017-04-14 19:58:46 +02:00
|
|
|
values[Anum_pg_subscription_subsynccommit - 1] =
|
2021-07-06 04:16:50 +02:00
|
|
|
CStringGetTextDatum(opts.synchronous_commit);
|
2017-01-19 18:00:00 +01:00
|
|
|
values[Anum_pg_subscription_subpublications - 1] =
|
|
|
|
publicationListToArray(publications);
|
|
|
|
|
|
|
|
tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
|
|
|
|
|
|
|
|
/* Insert tuple into catalog. */
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
CatalogTupleInsert(rel, tup);
|
2017-01-19 18:00:00 +01:00
|
|
|
heap_freetuple(tup);
|
|
|
|
|
2017-01-20 20:45:02 +01:00
|
|
|
recordDependencyOnOwner(SubscriptionRelationId, subid, owner);
|
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
snprintf(originname, sizeof(originname), "pg_%u", subid);
|
|
|
|
replorigin_create(originname);
|
|
|
|
|
|
|
|
/*
|
2017-03-23 13:36:36 +01:00
|
|
|
* Connect to remote side to execute requested commands and fetch table
|
|
|
|
* info.
|
2017-01-19 18:00:00 +01:00
|
|
|
*/
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.connect)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
|
|
|
char *err;
|
|
|
|
WalReceiverConn *wrconn;
|
2017-03-23 13:36:36 +01:00
|
|
|
List *tables;
|
|
|
|
ListCell *lc;
|
|
|
|
char table_state;
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
/* Try to connect to the publisher. */
|
|
|
|
wrconn = walrcv_connect(conninfo, true, stmt->subname, &err);
|
|
|
|
if (!wrconn)
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
|
|
|
errmsg("could not connect to the publisher: %s", err)));
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-01-25 16:47:53 +01:00
|
|
|
PG_TRY();
|
|
|
|
{
|
2017-03-23 13:36:36 +01:00
|
|
|
/*
|
|
|
|
* Set sync state based on if we were asked to do data copy or
|
|
|
|
* not.
|
2017-03-14 22:13:56 +01:00
|
|
|
*/
|
2021-07-06 04:16:50 +02:00
|
|
|
table_state = opts.copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY;
|
2017-03-23 13:36:36 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the table list from publisher and build local table status
|
|
|
|
* info.
|
|
|
|
*/
|
|
|
|
tables = fetch_table_list(wrconn, publications);
|
|
|
|
foreach(lc, tables)
|
|
|
|
{
|
|
|
|
RangeVar *rv = (RangeVar *) lfirst(lc);
|
|
|
|
Oid relid;
|
|
|
|
|
2017-03-24 19:44:11 +01:00
|
|
|
relid = RangeVarGetRelid(rv, AccessShareLock, false);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
2017-05-17 04:57:16 +02:00
|
|
|
/* Check for supported relkind. */
|
|
|
|
CheckSubscriptionRelkind(get_rel_relkind(relid),
|
|
|
|
rv->schemaname, rv->relname);
|
|
|
|
|
2018-04-06 16:00:26 +02:00
|
|
|
AddSubscriptionRelState(subid, relid, table_state,
|
|
|
|
InvalidXLogRecPtr);
|
2017-03-23 13:36:36 +01:00
|
|
|
}
|
|
|
|
|
2017-04-21 14:35:24 +02:00
|
|
|
/*
|
|
|
|
* If requested, create permanent slot for the subscription. We
|
|
|
|
* won't use the initial snapshot for anything, so no need to
|
|
|
|
* export it.
|
|
|
|
*/
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.create_slot)
|
2017-04-21 14:35:24 +02:00
|
|
|
{
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
bool twophase_enabled = false;
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
Assert(opts.slot_name);
|
2017-05-09 16:20:42 +02:00
|
|
|
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
/*
|
|
|
|
* Even if two_phase is set, don't create the slot with
|
|
|
|
* two-phase enabled. Will enable it once all the tables are
|
|
|
|
* synced and ready. This avoids race-conditions like prepared
|
|
|
|
* transactions being skipped due to changes not being applied
|
|
|
|
* due to checks in should_apply_changes_for_rel() when
|
|
|
|
* tablesync for the corresponding tables are in progress. See
|
|
|
|
* comments atop worker.c.
|
|
|
|
*
|
|
|
|
* Note that if tables were specified but copy_data is false
|
|
|
|
* then it is safe to enable two_phase up-front because those
|
|
|
|
* tables are already initially in READY state. When the
|
|
|
|
* subscription has no tables, we leave the twophase state as
|
|
|
|
* PENDING, to allow ALTER SUBSCRIPTION ... REFRESH
|
|
|
|
* PUBLICATION to work.
|
|
|
|
*/
|
|
|
|
if (opts.twophase && !opts.copy_data && tables != NIL)
|
|
|
|
twophase_enabled = true;
|
|
|
|
|
|
|
|
walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled,
|
2020-01-11 09:00:19 +01:00
|
|
|
CRS_NOEXPORT_SNAPSHOT, NULL);
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
|
|
|
|
if (twophase_enabled)
|
|
|
|
UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED);
|
|
|
|
|
2017-04-21 14:35:24 +02:00
|
|
|
ereport(NOTICE,
|
|
|
|
(errmsg("created replication slot \"%s\" on publisher",
|
2021-07-06 04:16:50 +02:00
|
|
|
opts.slot_name)));
|
2017-04-21 14:35:24 +02:00
|
|
|
}
|
2017-01-25 16:47:53 +01:00
|
|
|
}
|
2019-11-01 11:09:52 +01:00
|
|
|
PG_FINALLY();
|
2017-01-25 16:47:53 +01:00
|
|
|
{
|
|
|
|
walrcv_disconnect(wrconn);
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
2017-03-23 13:36:36 +01:00
|
|
|
else
|
|
|
|
ereport(WARNING,
|
2019-05-17 00:50:56 +02:00
|
|
|
/* translator: %s is an SQL ALTER statement */
|
|
|
|
(errmsg("tables were not subscribed, you will have to run %s to subscribe the tables",
|
|
|
|
"ALTER SUBSCRIPTION ... REFRESH PUBLICATION")));
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, RowExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.enabled)
|
2017-05-02 04:50:32 +02:00
|
|
|
ApplyLauncherWakeupAtCommit();
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
ObjectAddressSet(myself, SubscriptionRelationId, subid);
|
|
|
|
|
|
|
|
InvokeObjectPostCreateHook(SubscriptionRelationId, subid, 0);
|
|
|
|
|
|
|
|
return myself;
|
|
|
|
}
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
static void
|
|
|
|
AlterSubscription_refresh(Subscription *sub, bool copy_data)
|
|
|
|
{
|
|
|
|
char *err;
|
|
|
|
List *pubrel_names;
|
|
|
|
List *subrel_states;
|
|
|
|
Oid *subrel_local_oids;
|
|
|
|
Oid *pubrel_local_oids;
|
|
|
|
ListCell *lc;
|
|
|
|
int off;
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
int remove_rel_len;
|
|
|
|
Relation rel = NULL;
|
|
|
|
typedef struct SubRemoveRels
|
|
|
|
{
|
|
|
|
Oid relid;
|
|
|
|
char state;
|
|
|
|
} SubRemoveRels;
|
|
|
|
SubRemoveRels *sub_remove_rels;
|
2021-05-07 17:46:37 +02:00
|
|
|
WalReceiverConn *wrconn;
|
2017-03-23 13:36:36 +01:00
|
|
|
|
|
|
|
/* Load the library providing us libpq calls. */
|
|
|
|
load_file("libpqwalreceiver", false);
|
|
|
|
|
2021-05-07 17:46:37 +02:00
|
|
|
/* Try to connect to the publisher. */
|
|
|
|
wrconn = walrcv_connect(sub->conninfo, true, sub->name, &err);
|
|
|
|
if (!wrconn)
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
|
|
|
errmsg("could not connect to the publisher: %s", err)));
|
2021-05-07 17:46:37 +02:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
PG_TRY();
|
|
|
|
{
|
|
|
|
/* Get the table list from publisher. */
|
|
|
|
pubrel_names = fetch_table_list(wrconn, sub->publications);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
/* Get local table list. */
|
|
|
|
subrel_states = GetSubscriptionRelations(sub->oid);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
/*
|
|
|
|
* Build qsorted array of local table oids for faster lookup. This can
|
|
|
|
* potentially contain all tables in the database so speed of lookup
|
|
|
|
* is important.
|
|
|
|
*/
|
|
|
|
subrel_local_oids = palloc(list_length(subrel_states) * sizeof(Oid));
|
|
|
|
off = 0;
|
|
|
|
foreach(lc, subrel_states)
|
|
|
|
{
|
|
|
|
SubscriptionRelState *relstate = (SubscriptionRelState *) lfirst(lc);
|
2017-05-17 22:31:56 +02:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
subrel_local_oids[off++] = relstate->relid;
|
|
|
|
}
|
|
|
|
qsort(subrel_local_oids, list_length(subrel_states),
|
|
|
|
sizeof(Oid), oid_cmp);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Rels that we want to remove from subscription and drop any slots
|
|
|
|
* and origins corresponding to them.
|
|
|
|
*/
|
|
|
|
sub_remove_rels = palloc(list_length(subrel_states) * sizeof(SubRemoveRels));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Walk over the remote tables and try to match them to locally known
|
|
|
|
* tables. If the table is not known locally create a new state for
|
|
|
|
* it.
|
|
|
|
*
|
|
|
|
* Also builds array of local oids of remote tables for the next step.
|
|
|
|
*/
|
|
|
|
off = 0;
|
|
|
|
pubrel_local_oids = palloc(list_length(pubrel_names) * sizeof(Oid));
|
|
|
|
|
|
|
|
foreach(lc, pubrel_names)
|
|
|
|
{
|
|
|
|
RangeVar *rv = (RangeVar *) lfirst(lc);
|
|
|
|
Oid relid;
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
relid = RangeVarGetRelid(rv, AccessShareLock, false);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
/* Check for supported relkind. */
|
|
|
|
CheckSubscriptionRelkind(get_rel_relkind(relid),
|
|
|
|
rv->schemaname, rv->relname);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
pubrel_local_oids[off++] = relid;
|
2017-05-17 04:57:16 +02:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
if (!bsearch(&relid, subrel_local_oids,
|
|
|
|
list_length(subrel_states), sizeof(Oid), oid_cmp))
|
|
|
|
{
|
|
|
|
AddSubscriptionRelState(sub->oid, relid,
|
|
|
|
copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
|
|
|
|
InvalidXLogRecPtr);
|
|
|
|
ereport(DEBUG1,
|
2021-02-17 11:24:46 +01:00
|
|
|
(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
rv->schemaname, rv->relname, sub->name)));
|
|
|
|
}
|
|
|
|
}
|
2017-05-17 04:57:16 +02:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
/*
|
|
|
|
* Next remove state for tables we should not care about anymore using
|
|
|
|
* the data we collected above
|
|
|
|
*/
|
|
|
|
qsort(pubrel_local_oids, list_length(pubrel_names),
|
|
|
|
sizeof(Oid), oid_cmp);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
remove_rel_len = 0;
|
|
|
|
for (off = 0; off < list_length(subrel_states); off++)
|
2017-03-23 13:36:36 +01:00
|
|
|
{
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
Oid relid = subrel_local_oids[off];
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
if (!bsearch(&relid, pubrel_local_oids,
|
|
|
|
list_length(pubrel_names), sizeof(Oid), oid_cmp))
|
|
|
|
{
|
|
|
|
char state;
|
|
|
|
XLogRecPtr statelsn;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Lock pg_subscription_rel with AccessExclusiveLock to
|
|
|
|
* prevent any race conditions with the apply worker
|
|
|
|
* re-launching workers at the same time this code is trying
|
|
|
|
* to remove those tables.
|
|
|
|
*
|
|
|
|
* Even if new worker for this particular rel is restarted it
|
|
|
|
* won't be able to make any progress as we hold exclusive
|
|
|
|
* lock on subscription_rel till the transaction end. It will
|
|
|
|
* simply exit as there is no corresponding rel entry.
|
|
|
|
*
|
|
|
|
* This locking also ensures that the state of rels won't
|
|
|
|
* change till we are done with this refresh operation.
|
|
|
|
*/
|
|
|
|
if (!rel)
|
|
|
|
rel = table_open(SubscriptionRelRelationId, AccessExclusiveLock);
|
|
|
|
|
|
|
|
/* Last known rel state. */
|
|
|
|
state = GetSubscriptionRelState(sub->oid, relid, &statelsn);
|
|
|
|
|
|
|
|
sub_remove_rels[remove_rel_len].relid = relid;
|
|
|
|
sub_remove_rels[remove_rel_len++].state = state;
|
|
|
|
|
|
|
|
RemoveSubscriptionRel(sub->oid, relid);
|
|
|
|
|
|
|
|
logicalrep_worker_stop(sub->oid, relid);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For READY state, we would have already dropped the
|
|
|
|
* tablesync origin.
|
|
|
|
*/
|
|
|
|
if (state != SUBREL_STATE_READY)
|
|
|
|
{
|
|
|
|
char originname[NAMEDATALEN];
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Drop the tablesync's origin tracking if exists.
|
|
|
|
*
|
|
|
|
* It is possible that the origin is not yet created for
|
|
|
|
* tablesync worker, this can happen for the states before
|
|
|
|
* SUBREL_STATE_FINISHEDCOPY. The apply worker can also
|
|
|
|
* concurrently try to drop the origin and by this time
|
|
|
|
* the origin might be already removed. For these reasons,
|
|
|
|
* passing missing_ok = true.
|
|
|
|
*/
|
2021-02-15 02:58:02 +01:00
|
|
|
ReplicationOriginNameForTablesync(sub->oid, relid, originname,
|
|
|
|
sizeof(originname));
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
replorigin_drop_by_name(originname, true, false);
|
|
|
|
}
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
ereport(DEBUG1,
|
2021-02-17 11:24:46 +01:00
|
|
|
(errmsg_internal("table \"%s.%s\" removed from subscription \"%s\"",
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
get_namespace_name(get_rel_namespace(relid)),
|
|
|
|
get_rel_name(relid),
|
|
|
|
sub->name)));
|
|
|
|
}
|
|
|
|
}
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
/*
|
|
|
|
* Drop the tablesync slots associated with removed tables. This has
|
|
|
|
* to be at the end because otherwise if there is an error while doing
|
|
|
|
* the database operations we won't be able to rollback dropped slots.
|
|
|
|
*/
|
|
|
|
for (off = 0; off < remove_rel_len; off++)
|
2017-03-23 13:36:36 +01:00
|
|
|
{
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
|
|
|
|
sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
|
|
|
|
{
|
|
|
|
char syncslotname[NAMEDATALEN] = {0};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For READY/SYNCDONE states we know the tablesync slot has
|
|
|
|
* already been dropped by the tablesync worker.
|
|
|
|
*
|
|
|
|
* For other states, there is no certainty, maybe the slot
|
|
|
|
* does not exist yet. Also, if we fail after removing some of
|
|
|
|
* the slots, next time, it will again try to drop already
|
|
|
|
* dropped slots and fail. For these reasons, we allow
|
|
|
|
* missing_ok = true for the drop.
|
|
|
|
*/
|
2021-02-15 02:58:02 +01:00
|
|
|
ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
|
|
|
|
syncslotname, sizeof(syncslotname));
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
|
|
|
|
}
|
2017-03-23 13:36:36 +01:00
|
|
|
}
|
|
|
|
}
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
PG_FINALLY();
|
|
|
|
{
|
2021-05-07 17:46:37 +02:00
|
|
|
walrcv_disconnect(wrconn);
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
}
|
|
|
|
PG_END_TRY();
|
|
|
|
|
|
|
|
if (rel)
|
|
|
|
table_close(rel, NoLock);
|
2017-03-23 13:36:36 +01:00
|
|
|
}
|
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
/*
|
|
|
|
* Alter the existing subscription.
|
|
|
|
*/
|
|
|
|
ObjectAddress
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
|
|
|
|
bool isTopLevel)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
|
|
|
Relation rel;
|
|
|
|
ObjectAddress myself;
|
|
|
|
bool nulls[Natts_pg_subscription];
|
|
|
|
bool replaces[Natts_pg_subscription];
|
|
|
|
Datum values[Natts_pg_subscription];
|
|
|
|
HeapTuple tup;
|
|
|
|
Oid subid;
|
2017-03-23 13:36:36 +01:00
|
|
|
bool update_tuple = false;
|
2017-05-09 16:20:42 +02:00
|
|
|
Subscription *sub;
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
Form_pg_subscription form;
|
2021-07-06 04:16:50 +02:00
|
|
|
bits32 supported_opts;
|
|
|
|
SubOpts opts = {0};
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
rel = table_open(SubscriptionRelationId, RowExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
/* Fetch the existing tuple. */
|
|
|
|
tup = SearchSysCacheCopy2(SUBSCRIPTIONNAME, MyDatabaseId,
|
|
|
|
CStringGetDatum(stmt->subname));
|
|
|
|
|
|
|
|
if (!HeapTupleIsValid(tup))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_UNDEFINED_OBJECT),
|
|
|
|
errmsg("subscription \"%s\" does not exist",
|
|
|
|
stmt->subname)));
|
|
|
|
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
form = (Form_pg_subscription) GETSTRUCT(tup);
|
|
|
|
subid = form->oid;
|
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
/* must be owner */
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
if (!pg_subscription_ownercheck(subid, GetUserId()))
|
2017-12-02 15:26:34 +01:00
|
|
|
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_SUBSCRIPTION,
|
2017-01-19 18:00:00 +01:00
|
|
|
stmt->subname);
|
|
|
|
|
2017-05-09 16:20:42 +02:00
|
|
|
sub = GetSubscription(subid, false);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-07-04 04:47:06 +02:00
|
|
|
/* Lock the subscription so nobody else can do anything with it. */
|
|
|
|
LockSharedObject(SubscriptionRelationId, subid, 0, AccessExclusiveLock);
|
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
/* Form a new tuple. */
|
|
|
|
memset(values, 0, sizeof(values));
|
|
|
|
memset(nulls, false, sizeof(nulls));
|
|
|
|
memset(replaces, false, sizeof(replaces));
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
switch (stmt->kind)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
2017-03-23 13:36:36 +01:00
|
|
|
case ALTER_SUBSCRIPTION_OPTIONS:
|
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
supported_opts = (SUBOPT_SLOT_NAME |
|
|
|
|
SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY |
|
|
|
|
SUBOPT_STREAMING);
|
|
|
|
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
parse_subscription_options(pstate, stmt->options,
|
|
|
|
supported_opts, &opts);
|
2021-07-06 04:16:50 +02:00
|
|
|
|
|
|
|
if (IsSet(opts.specified_opts, SUBOPT_SLOT_NAME))
|
2017-04-14 19:58:46 +02:00
|
|
|
{
|
2021-07-19 05:02:37 +02:00
|
|
|
/*
|
|
|
|
* The subscription must be disabled to allow slot_name as
|
|
|
|
* 'none', otherwise, the apply worker will repeatedly try
|
|
|
|
* to stream the data using that slot_name which neither
|
|
|
|
* exists on the publisher nor the user will be allowed to
|
|
|
|
* create it.
|
|
|
|
*/
|
2021-07-06 04:16:50 +02:00
|
|
|
if (sub->enabled && !opts.slot_name)
|
2017-05-09 16:20:42 +02:00
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
|
2019-05-17 00:50:56 +02:00
|
|
|
errmsg("cannot set %s for enabled subscription",
|
|
|
|
"slot_name = NONE")));
|
2017-05-09 16:20:42 +02:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.slot_name)
|
2017-05-09 16:20:42 +02:00
|
|
|
values[Anum_pg_subscription_subslotname - 1] =
|
2021-07-06 04:16:50 +02:00
|
|
|
DirectFunctionCall1(namein, CStringGetDatum(opts.slot_name));
|
2017-05-09 16:20:42 +02:00
|
|
|
else
|
|
|
|
nulls[Anum_pg_subscription_subslotname - 1] = true;
|
2017-04-14 19:58:46 +02:00
|
|
|
replaces[Anum_pg_subscription_subslotname - 1] = true;
|
|
|
|
}
|
2017-05-09 16:20:42 +02:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.synchronous_commit)
|
2017-04-14 19:58:46 +02:00
|
|
|
{
|
|
|
|
values[Anum_pg_subscription_subsynccommit - 1] =
|
2021-07-06 04:16:50 +02:00
|
|
|
CStringGetTextDatum(opts.synchronous_commit);
|
2017-04-14 19:58:46 +02:00
|
|
|
replaces[Anum_pg_subscription_subsynccommit - 1] = true;
|
|
|
|
}
|
2017-03-23 13:36:36 +01:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts.specified_opts, SUBOPT_BINARY))
|
2020-07-18 18:44:51 +02:00
|
|
|
{
|
|
|
|
values[Anum_pg_subscription_subbinary - 1] =
|
2021-07-06 04:16:50 +02:00
|
|
|
BoolGetDatum(opts.binary);
|
2020-07-18 18:44:51 +02:00
|
|
|
replaces[Anum_pg_subscription_subbinary - 1] = true;
|
|
|
|
}
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (IsSet(opts.specified_opts, SUBOPT_STREAMING))
|
Add support for streaming to built-in logical replication.
To add support for streaming of in-progress transactions into the
built-in logical replication, we need to do three things:
* Extend the logical replication protocol, so identify in-progress
transactions, and allow adding additional bits of information (e.g.
XID of subtransactions).
* Modify the output plugin (pgoutput) to implement the new stream
API callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle streamed
in-progress transaction by spilling the data to disk and then
replaying them on commit.
We however must explicitly disable streaming replication during
replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover we don't have a replication connection open so we
don't have where to send the data anyway.
Author: Tomas Vondra, Dilip Kumar and Amit Kapila
Reviewed-by: Amit Kapila, Kuntal Ghosh and Ajin Cherian
Tested-by: Neha Sharma, Mahendra Singh Thalor and Ajin Cherian
Discussion: https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com
2020-09-03 04:24:07 +02:00
|
|
|
{
|
|
|
|
values[Anum_pg_subscription_substream - 1] =
|
2021-07-06 04:16:50 +02:00
|
|
|
BoolGetDatum(opts.streaming);
|
Add support for streaming to built-in logical replication.
To add support for streaming of in-progress transactions into the
built-in logical replication, we need to do three things:
* Extend the logical replication protocol, so identify in-progress
transactions, and allow adding additional bits of information (e.g.
XID of subtransactions).
* Modify the output plugin (pgoutput) to implement the new stream
API callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle streamed
in-progress transaction by spilling the data to disk and then
replaying them on commit.
We however must explicitly disable streaming replication during
replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover we don't have a replication connection open so we
don't have where to send the data anyway.
Author: Tomas Vondra, Dilip Kumar and Amit Kapila
Reviewed-by: Amit Kapila, Kuntal Ghosh and Ajin Cherian
Tested-by: Neha Sharma, Mahendra Singh Thalor and Ajin Cherian
Discussion: https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com
2020-09-03 04:24:07 +02:00
|
|
|
replaces[Anum_pg_subscription_substream - 1] = true;
|
|
|
|
}
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
update_tuple = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
case ALTER_SUBSCRIPTION_ENABLED:
|
|
|
|
{
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
parse_subscription_options(pstate, stmt->options,
|
|
|
|
SUBOPT_ENABLED, &opts);
|
2021-07-06 04:16:50 +02:00
|
|
|
Assert(IsSet(opts.specified_opts, SUBOPT_ENABLED));
|
|
|
|
|
|
|
|
if (!sub->slotname && opts.enabled)
|
2017-05-09 16:20:42 +02:00
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
|
2017-05-09 16:20:42 +02:00
|
|
|
errmsg("cannot enable subscription that does not have a slot name")));
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
values[Anum_pg_subscription_subenabled - 1] =
|
2021-07-06 04:16:50 +02:00
|
|
|
BoolGetDatum(opts.enabled);
|
2017-03-23 13:36:36 +01:00
|
|
|
replaces[Anum_pg_subscription_subenabled - 1] = true;
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.enabled)
|
2017-04-25 20:40:33 +02:00
|
|
|
ApplyLauncherWakeupAtCommit();
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
update_tuple = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
case ALTER_SUBSCRIPTION_CONNECTION:
|
2017-05-08 20:01:00 +02:00
|
|
|
/* Load the library providing us libpq calls. */
|
|
|
|
load_file("libpqwalreceiver", false);
|
|
|
|
/* Check the connection info string. */
|
|
|
|
walrcv_check_conninfo(stmt->conninfo);
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
values[Anum_pg_subscription_subconninfo - 1] =
|
|
|
|
CStringGetTextDatum(stmt->conninfo);
|
|
|
|
replaces[Anum_pg_subscription_subconninfo - 1] = true;
|
|
|
|
update_tuple = true;
|
|
|
|
break;
|
|
|
|
|
2021-04-06 10:44:26 +02:00
|
|
|
case ALTER_SUBSCRIPTION_SET_PUBLICATION:
|
2017-03-23 13:36:36 +01:00
|
|
|
{
|
2021-07-06 04:16:50 +02:00
|
|
|
supported_opts = SUBOPT_COPY_DATA | SUBOPT_REFRESH;
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
parse_subscription_options(pstate, stmt->options,
|
|
|
|
supported_opts, &opts);
|
2021-07-06 04:16:50 +02:00
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
values[Anum_pg_subscription_subpublications - 1] =
|
|
|
|
publicationListToArray(stmt->publication);
|
|
|
|
replaces[Anum_pg_subscription_subpublications - 1] = true;
|
|
|
|
|
|
|
|
update_tuple = true;
|
|
|
|
|
|
|
|
/* Refresh if user asked us to. */
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.refresh)
|
2017-03-23 13:36:36 +01:00
|
|
|
{
|
2017-05-09 16:20:42 +02:00
|
|
|
if (!sub->enabled)
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
|
2017-06-06 03:37:00 +02:00
|
|
|
errmsg("ALTER SUBSCRIPTION with refresh is not allowed for disabled subscriptions"),
|
|
|
|
errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION ... WITH (refresh = false).")));
|
2017-05-09 16:20:42 +02:00
|
|
|
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
/*
|
|
|
|
* See ALTER_SUBSCRIPTION_REFRESH for details why this is
|
|
|
|
* not allowed.
|
|
|
|
*/
|
|
|
|
if (sub->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED && opts.copy_data)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"),
|
|
|
|
errhint("Use ALTER SUBSCRIPTION ...SET PUBLICATION with refresh = false, or with copy_data = false"
|
|
|
|
", or use DROP/CREATE SUBSCRIPTION.")));
|
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh");
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
/* Make sure refresh sees the new list of publications. */
|
|
|
|
sub->publications = stmt->publication;
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
AlterSubscription_refresh(sub, opts.copy_data);
|
2017-03-23 13:36:36 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2021-04-06 10:44:26 +02:00
|
|
|
case ALTER_SUBSCRIPTION_ADD_PUBLICATION:
|
|
|
|
case ALTER_SUBSCRIPTION_DROP_PUBLICATION:
|
|
|
|
{
|
|
|
|
List *publist;
|
2021-07-06 04:16:50 +02:00
|
|
|
bool isadd = stmt->kind == ALTER_SUBSCRIPTION_ADD_PUBLICATION;
|
2021-04-06 10:44:26 +02:00
|
|
|
|
2021-08-24 04:55:21 +02:00
|
|
|
supported_opts = SUBOPT_REFRESH | SUBOPT_COPY_DATA;
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
parse_subscription_options(pstate, stmt->options,
|
|
|
|
supported_opts, &opts);
|
2021-06-25 09:51:24 +02:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
publist = merge_publications(sub->publications, stmt->publication, isadd, stmt->subname);
|
2021-04-06 10:44:26 +02:00
|
|
|
values[Anum_pg_subscription_subpublications - 1] =
|
|
|
|
publicationListToArray(publist);
|
|
|
|
replaces[Anum_pg_subscription_subpublications - 1] = true;
|
|
|
|
|
|
|
|
update_tuple = true;
|
|
|
|
|
|
|
|
/* Refresh if user asked us to. */
|
2021-07-06 04:16:50 +02:00
|
|
|
if (opts.refresh)
|
2021-04-06 10:44:26 +02:00
|
|
|
{
|
|
|
|
if (!sub->enabled)
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
|
2021-04-06 10:44:26 +02:00
|
|
|
errmsg("ALTER SUBSCRIPTION with refresh is not allowed for disabled subscriptions"),
|
|
|
|
errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION ... WITH (refresh = false).")));
|
|
|
|
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
/*
|
|
|
|
* See ALTER_SUBSCRIPTION_REFRESH for details why this is
|
|
|
|
* not allowed.
|
|
|
|
*/
|
|
|
|
if (sub->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED && opts.copy_data)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"),
|
|
|
|
errhint("Use ALTER SUBSCRIPTION ...SET PUBLICATION with refresh = false, or with copy_data = false"
|
|
|
|
", or use DROP/CREATE SUBSCRIPTION.")));
|
|
|
|
|
2021-04-06 10:44:26 +02:00
|
|
|
PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh");
|
|
|
|
|
2021-08-24 04:55:21 +02:00
|
|
|
/* Refresh the new list of publications. */
|
|
|
|
sub->publications = publist;
|
2021-04-06 10:44:26 +02:00
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
AlterSubscription_refresh(sub, opts.copy_data);
|
2021-04-06 10:44:26 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
case ALTER_SUBSCRIPTION_REFRESH:
|
|
|
|
{
|
2017-05-09 16:20:42 +02:00
|
|
|
if (!sub->enabled)
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
|
2017-05-09 16:20:42 +02:00
|
|
|
errmsg("ALTER SUBSCRIPTION ... REFRESH is not allowed for disabled subscriptions")));
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Improve reporting of "conflicting or redundant options" errors.
When reporting "conflicting or redundant options" errors, try to
ensure that errposition() is used, to help the user identify the
offending option.
Formerly, errposition() was invoked in less than 60% of cases. This
patch raises that to over 90%, but there remain a few places where the
ParseState is not readily available. Using errdetail() might improve
the error in such cases, but that is left as a task for the future.
Additionally, since this error is thrown from over 100 places in the
codebase, introduce a dedicated function to throw it, reducing code
duplication.
Extracted from a slightly larger patch by Vignesh C. Reviewed by
Bharath Rupireddy, Alvaro Herrera, Dilip Kumar, Hou Zhijie, Peter
Smith, Daniel Gustafsson, Julien Rouhaud and me.
Discussion: https://postgr.es/m/CALDaNm33FFSS5tVyvmkoK2cCMuDVxcui=gFrjti9ROfynqSAGA@mail.gmail.com
2021-07-15 09:49:45 +02:00
|
|
|
parse_subscription_options(pstate, stmt->options,
|
|
|
|
SUBOPT_COPY_DATA, &opts);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
Add support for prepared transactions to built-in logical replication.
To add support for streaming transactions at prepare time into the
built-in logical replication, we need to do the following things:
* Modify the output plugin (pgoutput) to implement the new two-phase API
callbacks, by leveraging the extended replication protocol.
* Modify the replication apply worker, to properly handle two-phase
transactions by replaying them on prepare.
* Add a new SUBSCRIPTION option "two_phase" to allow users to enable
two-phase transactions. We enable the two_phase once the initial data sync
is over.
We however must explicitly disable replication of two-phase transactions
during replication slot creation, even if the plugin supports it. We
don't need to replicate the changes accumulated during this phase,
and moreover, we don't have a replication connection open so we don't know
where to send the data anyway.
The streaming option is not allowed with this new two_phase option. This
can be done as a separate patch.
We don't allow to toggle two_phase option of a subscription because it can
lead to an inconsistent replica. For the same reason, we don't allow to
refresh the publication once the two_phase is enabled for a subscription
unless copy_data option is false.
Author: Peter Smith, Ajin Cherian and Amit Kapila based on previous work by Nikhil Sontakke and Stas Kelvich
Reviewed-by: Amit Kapila, Sawada Masahiko, Vignesh C, Dilip Kumar, Takamichi Osumi, Greg Nancarrow
Tested-By: Haiying Tang
Discussion: https://postgr.es/m/02DA5F5E-CECE-4D9C-8B4B-418077E2C010@postgrespro.ru
Discussion: https://postgr.es/m/CAA4eK1+opiV4aFTmWWUF9h_32=HfPOW9vZASHarT0UA5oBrtGw@mail.gmail.com
2021-07-14 04:03:50 +02:00
|
|
|
/*
|
|
|
|
* The subscription option "two_phase" requires that
|
|
|
|
* replication has passed the initial table synchronization
|
|
|
|
* phase before the two_phase becomes properly enabled.
|
|
|
|
*
|
|
|
|
* But, having reached this two-phase commit "enabled" state
|
|
|
|
* we must not allow any subsequent table initialization to
|
|
|
|
* occur. So the ALTER SUBSCRIPTION ... REFRESH is disallowed
|
|
|
|
* when the user had requested two_phase = on mode.
|
|
|
|
*
|
|
|
|
* The exception to this restriction is when copy_data =
|
|
|
|
* false, because when copy_data is false the tablesync will
|
|
|
|
* start already in READY state and will exit directly without
|
|
|
|
* doing anything.
|
|
|
|
*
|
|
|
|
* For more details see comments atop worker.c.
|
|
|
|
*/
|
|
|
|
if (sub->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED && opts.copy_data)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"),
|
|
|
|
errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false"
|
|
|
|
", or use DROP/CREATE SUBSCRIPTION.")));
|
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH");
|
|
|
|
|
2021-07-06 04:16:50 +02:00
|
|
|
AlterSubscription_refresh(sub, opts.copy_data);
|
2017-03-23 13:36:36 +01:00
|
|
|
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
default:
|
|
|
|
elog(ERROR, "unrecognized ALTER SUBSCRIPTION kind %d",
|
|
|
|
stmt->kind);
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
/* Update the catalog if needed. */
|
|
|
|
if (update_tuple)
|
|
|
|
{
|
|
|
|
tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
|
|
|
|
replaces);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
CatalogTupleUpdate(rel, &tup->t_self, tup);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
heap_freetuple(tup);
|
|
|
|
}
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, RowExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
ObjectAddressSet(myself, SubscriptionRelationId, subid);
|
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
InvokeObjectPostAlterHook(SubscriptionRelationId, subid, 0);
|
|
|
|
|
|
|
|
return myself;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Drop a subscription
|
|
|
|
*/
|
|
|
|
void
|
2017-03-04 05:25:34 +01:00
|
|
|
DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
|
|
|
Relation rel;
|
|
|
|
ObjectAddress myself;
|
|
|
|
HeapTuple tup;
|
|
|
|
Oid subid;
|
|
|
|
Datum datum;
|
|
|
|
bool isnull;
|
|
|
|
char *subname;
|
|
|
|
char *conninfo;
|
|
|
|
char *slotname;
|
2017-08-05 03:14:35 +02:00
|
|
|
List *subworkers;
|
|
|
|
ListCell *lc;
|
2017-01-19 18:00:00 +01:00
|
|
|
char originname[NAMEDATALEN];
|
|
|
|
char *err = NULL;
|
2021-05-07 17:46:37 +02:00
|
|
|
WalReceiverConn *wrconn;
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
Form_pg_subscription form;
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
List *rstates;
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-03-08 17:44:23 +01:00
|
|
|
/*
|
|
|
|
* Lock pg_subscription with AccessExclusiveLock to ensure that the
|
|
|
|
* launcher doesn't restart new worker during dropping the subscription
|
|
|
|
*/
|
2019-01-21 19:32:19 +01:00
|
|
|
rel = table_open(SubscriptionRelationId, AccessExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
tup = SearchSysCache2(SUBSCRIPTIONNAME, MyDatabaseId,
|
|
|
|
CStringGetDatum(stmt->subname));
|
|
|
|
|
|
|
|
if (!HeapTupleIsValid(tup))
|
|
|
|
{
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, NoLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
if (!stmt->missing_ok)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_UNDEFINED_OBJECT),
|
|
|
|
errmsg("subscription \"%s\" does not exist",
|
|
|
|
stmt->subname)));
|
|
|
|
else
|
|
|
|
ereport(NOTICE,
|
|
|
|
(errmsg("subscription \"%s\" does not exist, skipping",
|
|
|
|
stmt->subname)));
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
form = (Form_pg_subscription) GETSTRUCT(tup);
|
|
|
|
subid = form->oid;
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
/* must be owner */
|
|
|
|
if (!pg_subscription_ownercheck(subid, GetUserId()))
|
2017-12-02 15:26:34 +01:00
|
|
|
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_SUBSCRIPTION,
|
2017-01-19 18:00:00 +01:00
|
|
|
stmt->subname);
|
|
|
|
|
|
|
|
/* DROP hook for the subscription being removed */
|
|
|
|
InvokeObjectDropHook(SubscriptionRelationId, subid, 0);
|
|
|
|
|
|
|
|
/*
|
2017-02-06 10:33:58 +01:00
|
|
|
* Lock the subscription so nobody else can do anything with it (including
|
2017-01-19 18:00:00 +01:00
|
|
|
* the replication workers).
|
|
|
|
*/
|
|
|
|
LockSharedObject(SubscriptionRelationId, subid, 0, AccessExclusiveLock);
|
|
|
|
|
|
|
|
/* Get subname */
|
|
|
|
datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
|
|
|
|
Anum_pg_subscription_subname, &isnull);
|
|
|
|
Assert(!isnull);
|
|
|
|
subname = pstrdup(NameStr(*DatumGetName(datum)));
|
|
|
|
|
|
|
|
/* Get conninfo */
|
|
|
|
datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
|
|
|
|
Anum_pg_subscription_subconninfo, &isnull);
|
|
|
|
Assert(!isnull);
|
2017-04-14 18:54:09 +02:00
|
|
|
conninfo = TextDatumGetCString(datum);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
/* Get slotname */
|
|
|
|
datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
|
|
|
|
Anum_pg_subscription_subslotname, &isnull);
|
2017-05-09 16:20:42 +02:00
|
|
|
if (!isnull)
|
|
|
|
slotname = pstrdup(NameStr(*DatumGetName(datum)));
|
|
|
|
else
|
|
|
|
slotname = NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Since dropping a replication slot is not transactional, the replication
|
|
|
|
* slot stays dropped even if the transaction rolls back. So we cannot
|
|
|
|
* run DROP SUBSCRIPTION inside a transaction block if dropping the
|
2021-11-30 04:24:30 +01:00
|
|
|
* replication slot. Also, in this case, we report a message for dropping
|
|
|
|
* the subscription to the stats collector.
|
2017-05-09 16:20:42 +02:00
|
|
|
*
|
|
|
|
* XXX The command name should really be something like "DROP SUBSCRIPTION
|
|
|
|
* of a subscription that is associated with a replication slot", but we
|
|
|
|
* don't have the proper facilities for that.
|
|
|
|
*/
|
|
|
|
if (slotname)
|
2018-02-17 02:44:15 +01:00
|
|
|
PreventInTransactionBlock(isTopLevel, "DROP SUBSCRIPTION");
|
2017-05-09 16:20:42 +02:00
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
ObjectAddressSet(myself, SubscriptionRelationId, subid);
|
|
|
|
EventTriggerSQLDropAddObject(&myself, true, true);
|
|
|
|
|
|
|
|
/* Remove the tuple from catalog. */
|
2017-02-01 22:13:30 +01:00
|
|
|
CatalogTupleDelete(rel, &tup->t_self);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
ReleaseSysCache(tup);
|
|
|
|
|
2017-08-05 03:14:35 +02:00
|
|
|
/*
|
Fix DROP SUBSCRIPTION hang
When ALTER SUBSCRIPTION DISABLE is run in the same transaction before
DROP SUBSCRIPTION, the latter will hang because workers will still be
running, not having seen the DISABLE committed, and DROP SUBSCRIPTION
will wait until the workers have vacated the replication origin slots.
Previously, DROP SUBSCRIPTION killed the logical replication workers
immediately only if it was going to drop the replication slot, otherwise
it scheduled the worker killing for the end of the transaction, as a
result of 7e174fa793a2df89fe03d002a5087ef67abcdde8. This, however,
causes the present problem. To fix, kill the workers immediately in all
cases. This covers all cases: A subscription that doesn't have a
replication slot must be disabled. It was either disabled in the same
transaction, or it was already disabled before the current transaction,
but then there shouldn't be any workers left and this won't make a
difference.
Reported-by: Arseny Sher <a.sher@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/87mv6av84w.fsf%40ars-thinkpad
2017-09-18 03:37:02 +02:00
|
|
|
* Stop all the subscription workers immediately.
|
|
|
|
*
|
|
|
|
* This is necessary if we are dropping the replication slot, so that the
|
|
|
|
* slot becomes accessible.
|
|
|
|
*
|
|
|
|
* It is also necessary if the subscription is disabled and was disabled
|
|
|
|
* in the same transaction. Then the workers haven't seen the disabling
|
|
|
|
* yet and will still be running, leading to hangs later when we want to
|
|
|
|
* drop the replication origin. If the subscription was disabled before
|
|
|
|
* this transaction, then there shouldn't be any workers left, so this
|
|
|
|
* won't make a difference.
|
2017-08-05 03:14:35 +02:00
|
|
|
*
|
|
|
|
* New workers won't be started because we hold an exclusive lock on the
|
|
|
|
* subscription till the end of the transaction.
|
|
|
|
*/
|
|
|
|
LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
|
|
|
|
subworkers = logicalrep_workers_find(subid, false);
|
|
|
|
LWLockRelease(LogicalRepWorkerLock);
|
|
|
|
foreach(lc, subworkers)
|
|
|
|
{
|
|
|
|
LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc);
|
2017-08-14 23:29:33 +02:00
|
|
|
|
Fix DROP SUBSCRIPTION hang
When ALTER SUBSCRIPTION DISABLE is run in the same transaction before
DROP SUBSCRIPTION, the latter will hang because workers will still be
running, not having seen the DISABLE committed, and DROP SUBSCRIPTION
will wait until the workers have vacated the replication origin slots.
Previously, DROP SUBSCRIPTION killed the logical replication workers
immediately only if it was going to drop the replication slot, otherwise
it scheduled the worker killing for the end of the transaction, as a
result of 7e174fa793a2df89fe03d002a5087ef67abcdde8. This, however,
causes the present problem. To fix, kill the workers immediately in all
cases. This covers all cases: A subscription that doesn't have a
replication slot must be disabled. It was either disabled in the same
transaction, or it was already disabled before the current transaction,
but then there shouldn't be any workers left and this won't make a
difference.
Reported-by: Arseny Sher <a.sher@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/87mv6av84w.fsf%40ars-thinkpad
2017-09-18 03:37:02 +02:00
|
|
|
logicalrep_worker_stop(w->subid, w->relid);
|
2017-08-05 03:14:35 +02:00
|
|
|
}
|
|
|
|
list_free(subworkers);
|
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
/*
|
|
|
|
* Cleanup of tablesync replication origins.
|
|
|
|
*
|
|
|
|
* Any READY-state relations would already have dealt with clean-ups.
|
|
|
|
*
|
|
|
|
* Note that the state can't change because we have already stopped both
|
|
|
|
* the apply and tablesync workers and they can't restart because of
|
|
|
|
* exclusive lock on the subscription.
|
|
|
|
*/
|
|
|
|
rstates = GetSubscriptionNotReadyRelations(subid);
|
|
|
|
foreach(lc, rstates)
|
|
|
|
{
|
|
|
|
SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
|
|
|
|
Oid relid = rstate->relid;
|
|
|
|
|
|
|
|
/* Only cleanup resources of tablesync workers */
|
|
|
|
if (!OidIsValid(relid))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Drop the tablesync's origin tracking if exists.
|
|
|
|
*
|
|
|
|
* It is possible that the origin is not yet created for tablesync
|
|
|
|
* worker so passing missing_ok = true. This can happen for the states
|
|
|
|
* before SUBREL_STATE_FINISHEDCOPY.
|
|
|
|
*/
|
2021-02-15 02:58:02 +01:00
|
|
|
ReplicationOriginNameForTablesync(subid, relid, originname,
|
|
|
|
sizeof(originname));
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
replorigin_drop_by_name(originname, true, false);
|
|
|
|
}
|
|
|
|
|
2017-01-20 20:45:02 +01:00
|
|
|
/* Clean up dependencies */
|
|
|
|
deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
|
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
/* Remove any associated relation synchronization states. */
|
|
|
|
RemoveSubscriptionRel(subid, InvalidOid);
|
|
|
|
|
2017-01-19 18:00:00 +01:00
|
|
|
/* Remove the origin tracking if exists. */
|
|
|
|
snprintf(originname, sizeof(originname), "pg_%u", subid);
|
2021-02-10 02:47:09 +01:00
|
|
|
replorigin_drop_by_name(originname, true, false);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-05-09 16:20:42 +02:00
|
|
|
/*
|
|
|
|
* If there is no slot associated with the subscription, we can finish
|
|
|
|
* here.
|
|
|
|
*/
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
if (!slotname && rstates == NIL)
|
2017-01-19 18:00:00 +01:00
|
|
|
{
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, NoLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
* Try to acquire the connection necessary for dropping slots.
|
|
|
|
*
|
|
|
|
* Note: If the slotname is NONE/NULL then we allow the command to finish
|
|
|
|
* and users need to manually cleanup the apply and tablesync worker slots
|
|
|
|
* later.
|
|
|
|
*
|
|
|
|
* This has to be at the end because otherwise if there is an error while
|
|
|
|
* doing the database operations we won't be able to rollback dropped
|
|
|
|
* slot.
|
2017-01-19 18:00:00 +01:00
|
|
|
*/
|
|
|
|
load_file("libpqwalreceiver", false);
|
|
|
|
|
|
|
|
wrconn = walrcv_connect(conninfo, true, subname, &err);
|
|
|
|
if (wrconn == NULL)
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
{
|
|
|
|
if (!slotname)
|
|
|
|
{
|
|
|
|
/* be tidy */
|
|
|
|
list_free(rstates);
|
|
|
|
table_close(rel, NoLock);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
ReportSlotConnectionError(rstates, subid, slotname, err);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
PG_TRY();
|
|
|
|
{
|
|
|
|
foreach(lc, rstates)
|
|
|
|
{
|
|
|
|
SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
|
|
|
|
Oid relid = rstate->relid;
|
|
|
|
|
|
|
|
/* Only cleanup resources of tablesync workers */
|
|
|
|
if (!OidIsValid(relid))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Drop the tablesync slots associated with removed tables.
|
|
|
|
*
|
|
|
|
* For SYNCDONE/READY states, the tablesync slot is known to have
|
|
|
|
* already been dropped by the tablesync worker.
|
|
|
|
*
|
|
|
|
* For other states, there is no certainty, maybe the slot does
|
|
|
|
* not exist yet. Also, if we fail after removing some of the
|
|
|
|
* slots, next time, it will again try to drop already dropped
|
|
|
|
* slots and fail. For these reasons, we allow missing_ok = true
|
|
|
|
* for the drop.
|
|
|
|
*/
|
|
|
|
if (rstate->state != SUBREL_STATE_SYNCDONE)
|
|
|
|
{
|
|
|
|
char syncslotname[NAMEDATALEN] = {0};
|
|
|
|
|
2021-02-15 02:58:02 +01:00
|
|
|
ReplicationSlotNameForTablesync(subid, relid, syncslotname,
|
|
|
|
sizeof(syncslotname));
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
list_free(rstates);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there is a slot associated with the subscription, then drop the
|
|
|
|
* replication slot at the publisher.
|
|
|
|
*/
|
|
|
|
if (slotname)
|
|
|
|
ReplicationSlotDropAtPubNode(wrconn, slotname, false);
|
|
|
|
|
|
|
|
}
|
|
|
|
PG_FINALLY();
|
|
|
|
{
|
|
|
|
walrcv_disconnect(wrconn);
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
|
|
|
|
2021-11-30 04:24:30 +01:00
|
|
|
/*
|
|
|
|
* Send a message for dropping this subscription to the stats collector.
|
|
|
|
* We can safely report dropping the subscription statistics here if the
|
|
|
|
* subscription is associated with a replication slot since we cannot run
|
|
|
|
* DROP SUBSCRIPTION inside a transaction block. Subscription statistics
|
|
|
|
* will be removed later by (auto)vacuum either if it's not associated
|
|
|
|
* with a replication slot or if the message for dropping the subscription
|
|
|
|
* gets lost.
|
|
|
|
*/
|
|
|
|
if (slotname)
|
|
|
|
pgstat_report_subscription_drop(subid);
|
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
table_close(rel, NoLock);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Drop the replication slot at the publisher node using the replication
|
|
|
|
* connection.
|
|
|
|
*
|
|
|
|
* missing_ok - if true then only issue a LOG message if the slot doesn't
|
|
|
|
* exist.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok)
|
|
|
|
{
|
|
|
|
StringInfoData cmd;
|
|
|
|
|
|
|
|
Assert(wrconn);
|
|
|
|
|
|
|
|
load_file("libpqwalreceiver", false);
|
|
|
|
|
|
|
|
initStringInfo(&cmd);
|
|
|
|
appendStringInfo(&cmd, "DROP_REPLICATION_SLOT %s WAIT", quote_identifier(slotname));
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2017-03-08 15:43:38 +01:00
|
|
|
PG_TRY();
|
|
|
|
{
|
2017-03-23 13:36:36 +01:00
|
|
|
WalRcvExecResult *res;
|
2017-05-17 22:31:56 +02:00
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
res = walrcv_exec(wrconn, cmd.data, 0, NULL);
|
|
|
|
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
if (res->status == WALRCV_OK_COMMAND)
|
|
|
|
{
|
|
|
|
/* NOTICE. Success. */
|
|
|
|
ereport(NOTICE,
|
|
|
|
(errmsg("dropped replication slot \"%s\" on publisher",
|
|
|
|
slotname)));
|
|
|
|
}
|
|
|
|
else if (res->status == WALRCV_ERROR &&
|
|
|
|
missing_ok &&
|
|
|
|
res->sqlstate == ERRCODE_UNDEFINED_OBJECT)
|
|
|
|
{
|
|
|
|
/* LOG. Error, but missing_ok = true. */
|
|
|
|
ereport(LOG,
|
2021-03-31 21:25:53 +02:00
|
|
|
(errmsg("could not drop replication slot \"%s\" on publisher: %s",
|
|
|
|
slotname, res->err)));
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
}
|
2017-03-08 15:43:38 +01:00
|
|
|
else
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
{
|
|
|
|
/* ERROR. */
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
|
|
|
errmsg("could not drop replication slot \"%s\" on publisher: %s",
|
2021-03-31 21:25:53 +02:00
|
|
|
slotname, res->err)));
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
}
|
2017-03-23 13:36:36 +01:00
|
|
|
|
|
|
|
walrcv_clear_result(res);
|
2017-03-08 15:43:38 +01:00
|
|
|
}
|
2019-11-01 11:09:52 +01:00
|
|
|
PG_FINALLY();
|
2017-02-21 19:36:02 +01:00
|
|
|
{
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
pfree(cmd.data);
|
2017-02-21 19:36:02 +01:00
|
|
|
}
|
2017-03-08 15:43:38 +01:00
|
|
|
PG_END_TRY();
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Internal workhorse for changing a subscription owner
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
AlterSubscriptionOwner_internal(Relation rel, HeapTuple tup, Oid newOwnerId)
|
|
|
|
{
|
|
|
|
Form_pg_subscription form;
|
|
|
|
|
|
|
|
form = (Form_pg_subscription) GETSTRUCT(tup);
|
|
|
|
|
|
|
|
if (form->subowner == newOwnerId)
|
|
|
|
return;
|
|
|
|
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
if (!pg_subscription_ownercheck(form->oid, GetUserId()))
|
2017-12-02 15:26:34 +01:00
|
|
|
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_SUBSCRIPTION,
|
2017-01-19 18:00:00 +01:00
|
|
|
NameStr(form->subname));
|
|
|
|
|
|
|
|
/* New owner must be a superuser */
|
|
|
|
if (!superuser_arg(newOwnerId))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
|
|
|
|
errmsg("permission denied to change owner of subscription \"%s\"",
|
|
|
|
NameStr(form->subname)),
|
2017-04-17 20:19:39 +02:00
|
|
|
errhint("The owner of a subscription must be a superuser.")));
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
form->subowner = newOwnerId;
|
2017-01-31 22:42:24 +01:00
|
|
|
CatalogTupleUpdate(rel, &tup->t_self, tup);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
/* Update owner dependency reference */
|
|
|
|
changeDependencyOnOwner(SubscriptionRelationId,
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
form->oid,
|
2017-01-19 18:00:00 +01:00
|
|
|
newOwnerId);
|
|
|
|
|
|
|
|
InvokeObjectPostAlterHook(SubscriptionRelationId,
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
form->oid, 0);
|
Respect permissions within logical replication.
Prevent logical replication workers from performing insert, update,
delete, truncate, or copy commands on tables unless the subscription
owner has permission to do so.
Prevent subscription owners from circumventing row-level security by
forbidding replication into tables with row-level security policies
which the subscription owner is subject to, without regard to whether
the policy would ordinarily allow the INSERT, UPDATE, DELETE or
TRUNCATE which is being replicated. This seems sufficient for now, as
superusers, roles with bypassrls, and target table owners should still
be able to replicate despite RLS policies. We can revisit the
question of applying row-level security policies on a per-row basis if
this restriction proves too severe in practice.
Author: Mark Dilger
Reviewed-by: Jeff Davis, Andrew Dunstan, Ronan Dunklau
Discussion: https://postgr.es/m/9DFC88D3-1300-4DE8-ACBC-4CEF84399A53%40enterprisedb.com
2022-01-08 02:38:20 +01:00
|
|
|
|
|
|
|
ApplyLauncherWakeupAtCommit();
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Change subscription owner -- by name
|
|
|
|
*/
|
|
|
|
ObjectAddress
|
|
|
|
AlterSubscriptionOwner(const char *name, Oid newOwnerId)
|
|
|
|
{
|
|
|
|
Oid subid;
|
|
|
|
HeapTuple tup;
|
|
|
|
Relation rel;
|
|
|
|
ObjectAddress address;
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
Form_pg_subscription form;
|
2017-01-19 18:00:00 +01:00
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
rel = table_open(SubscriptionRelationId, RowExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
tup = SearchSysCacheCopy2(SUBSCRIPTIONNAME, MyDatabaseId,
|
|
|
|
CStringGetDatum(name));
|
|
|
|
|
|
|
|
if (!HeapTupleIsValid(tup))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_UNDEFINED_OBJECT),
|
|
|
|
errmsg("subscription \"%s\" does not exist", name)));
|
|
|
|
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
form = (Form_pg_subscription) GETSTRUCT(tup);
|
|
|
|
subid = form->oid;
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
AlterSubscriptionOwner_internal(rel, tup, newOwnerId);
|
|
|
|
|
|
|
|
ObjectAddressSet(address, SubscriptionRelationId, subid);
|
|
|
|
|
|
|
|
heap_freetuple(tup);
|
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, RowExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
return address;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Change subscription owner -- by OID
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId)
|
|
|
|
{
|
|
|
|
HeapTuple tup;
|
|
|
|
Relation rel;
|
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
rel = table_open(SubscriptionRelationId, RowExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
|
|
|
|
tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
|
|
|
|
|
|
|
|
if (!HeapTupleIsValid(tup))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_UNDEFINED_OBJECT),
|
|
|
|
errmsg("subscription with OID %u does not exist", subid)));
|
|
|
|
|
|
|
|
AlterSubscriptionOwner_internal(rel, tup, newOwnerId);
|
|
|
|
|
|
|
|
heap_freetuple(tup);
|
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, RowExclusiveLock);
|
2017-01-19 18:00:00 +01:00
|
|
|
}
|
2017-03-23 13:36:36 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the list of tables which belong to specified publications on the
|
|
|
|
* publisher connection.
|
|
|
|
*/
|
|
|
|
static List *
|
|
|
|
fetch_table_list(WalReceiverConn *wrconn, List *publications)
|
|
|
|
{
|
|
|
|
WalRcvExecResult *res;
|
|
|
|
StringInfoData cmd;
|
|
|
|
TupleTableSlot *slot;
|
|
|
|
Oid tableRow[2] = {TEXTOID, TEXTOID};
|
|
|
|
ListCell *lc;
|
|
|
|
bool first;
|
|
|
|
List *tablelist = NIL;
|
|
|
|
|
|
|
|
Assert(list_length(publications) > 0);
|
|
|
|
|
|
|
|
initStringInfo(&cmd);
|
2017-08-16 05:34:39 +02:00
|
|
|
appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
|
|
|
|
" FROM pg_catalog.pg_publication_tables t\n"
|
|
|
|
" WHERE t.pubname IN (");
|
2017-03-23 13:36:36 +01:00
|
|
|
first = true;
|
|
|
|
foreach(lc, publications)
|
|
|
|
{
|
|
|
|
char *pubname = strVal(lfirst(lc));
|
|
|
|
|
|
|
|
if (first)
|
|
|
|
first = false;
|
|
|
|
else
|
|
|
|
appendStringInfoString(&cmd, ", ");
|
|
|
|
|
2017-08-16 05:34:39 +02:00
|
|
|
appendStringInfoString(&cmd, quote_literal_cstr(pubname));
|
2017-03-23 13:36:36 +01:00
|
|
|
}
|
2017-08-16 05:34:39 +02:00
|
|
|
appendStringInfoChar(&cmd, ')');
|
2017-03-23 13:36:36 +01:00
|
|
|
|
|
|
|
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
|
|
|
|
pfree(cmd.data);
|
|
|
|
|
|
|
|
if (res->status != WALRCV_OK_TUPLES)
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
|
|
|
errmsg("could not receive list of replicated tables from the publisher: %s",
|
2017-03-23 13:36:36 +01:00
|
|
|
res->err)));
|
|
|
|
|
|
|
|
/* Process tables. */
|
Introduce notion of different types of slots (without implementing them).
Upcoming work intends to allow pluggable ways to introduce new ways of
storing table data. Accessing those table access methods from the
executor requires TupleTableSlots to be carry tuples in the native
format of such storage methods; otherwise there'll be a significant
conversion overhead.
Different access methods will require different data to store tuples
efficiently (just like virtual, minimal, heap already require fields
in TupleTableSlot). To allow that without requiring additional pointer
indirections, we want to have different structs (embedding
TupleTableSlot) for different types of slots. Thus different types of
slots are needed, which requires adapting creators of slots.
The slot that most efficiently can represent a type of tuple in an
executor node will often depend on the type of slot a child node
uses. Therefore we need to track the type of slot is returned by
nodes, so parent slots can create slots based on that.
Relatedly, JIT compilation of tuple deforming needs to know which type
of slot a certain expression refers to, so it can create an
appropriate deforming function for the type of tuple in the slot.
But not all nodes will only return one type of slot, e.g. an append
node will potentially return different types of slots for each of its
subplans.
Therefore add function that allows to query the type of a node's
result slot, and whether it'll always be the same type (whether it's
fixed). This can be queried using ExecGetResultSlotOps().
The scan, result, inner, outer type of slots are automatically
inferred from ExecInitScanTupleSlot(), ExecInitResultSlot(),
left/right subtrees respectively. If that's not correct for a node,
that can be overwritten using new fields in PlanState.
This commit does not introduce the actually abstracted implementation
of different kind of TupleTableSlots, that will be left for a followup
commit. The different types of slots introduced will, for now, still
use the same backing implementation.
While this already partially invalidates the big comment in
tuptable.h, it seems to make more sense to update it later, when the
different TupleTableSlot implementations actually exist.
Author: Ashutosh Bapat and Andres Freund, with changes by Amit Khandekar
Discussion: https://postgr.es/m/20181105210039.hh4vvi4vwoq5ba2q@alap3.anarazel.de
2018-11-16 07:00:30 +01:00
|
|
|
slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
|
2017-03-23 13:36:36 +01:00
|
|
|
while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
|
|
|
|
{
|
|
|
|
char *nspname;
|
|
|
|
char *relname;
|
|
|
|
bool isnull;
|
|
|
|
RangeVar *rv;
|
|
|
|
|
|
|
|
nspname = TextDatumGetCString(slot_getattr(slot, 1, &isnull));
|
|
|
|
Assert(!isnull);
|
|
|
|
relname = TextDatumGetCString(slot_getattr(slot, 2, &isnull));
|
|
|
|
Assert(!isnull);
|
|
|
|
|
2021-01-16 05:45:32 +01:00
|
|
|
rv = makeRangeVar(nspname, relname, -1);
|
2017-03-23 13:36:36 +01:00
|
|
|
tablelist = lappend(tablelist, rv);
|
|
|
|
|
|
|
|
ExecClearTuple(slot);
|
|
|
|
}
|
|
|
|
ExecDropSingleTupleTableSlot(slot);
|
|
|
|
|
|
|
|
walrcv_clear_result(res);
|
|
|
|
|
|
|
|
return tablelist;
|
|
|
|
}
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This is to report the connection failure while dropping replication slots.
|
|
|
|
* Here, we report the WARNING for all tablesync slots so that user can drop
|
|
|
|
* them manually, if required.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
foreach(lc, rstates)
|
|
|
|
{
|
|
|
|
SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
|
|
|
|
Oid relid = rstate->relid;
|
|
|
|
|
|
|
|
/* Only cleanup resources of tablesync workers */
|
|
|
|
if (!OidIsValid(relid))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Caller needs to ensure that relstate doesn't change underneath us.
|
|
|
|
* See DropSubscription where we get the relstates.
|
|
|
|
*/
|
|
|
|
if (rstate->state != SUBREL_STATE_SYNCDONE)
|
|
|
|
{
|
|
|
|
char syncslotname[NAMEDATALEN] = {0};
|
|
|
|
|
2021-02-15 02:58:02 +01:00
|
|
|
ReplicationSlotNameForTablesync(subid, relid, syncslotname,
|
|
|
|
sizeof(syncslotname));
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
elog(WARNING, "could not drop tablesync replication slot \"%s\"",
|
|
|
|
syncslotname);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
|
|
|
errmsg("could not connect to publisher when attempting to "
|
2021-03-31 21:25:53 +02:00
|
|
|
"drop replication slot \"%s\": %s", slotname, err),
|
Allow multiple xacts during table sync in logical replication.
For the initial table data synchronization in logical replication, we use
a single transaction to copy the entire table and then synchronize the
position in the stream with the main apply worker.
There are multiple downsides of this approach: (a) We have to perform the
entire copy operation again if there is any error (network breakdown,
error in the database operation, etc.) while we synchronize the WAL
position between tablesync worker and apply worker; this will be onerous
especially for large copies, (b) Using a single transaction in the
synchronization-phase (where we can receive WAL from multiple
transactions) will have the risk of exceeding the CID limit, (c) The slot
will hold the WAL till the entire sync is complete because we never commit
till the end.
This patch solves all the above downsides by allowing multiple
transactions during the tablesync phase. The initial copy is done in a
single transaction and after that, we commit each transaction as we
receive. To allow recovery after any error or crash, we use a permanent
slot and origin to track the progress. The slot and origin will be removed
once we finish the synchronization of the table. We also remove slot and
origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or
ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not
finished.
The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and
ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true
cannot be executed inside a transaction block because they can now drop
the slots for which we have no provision to rollback.
This will also open up the path for logical replication of 2PC
transactions on the subscriber side. Previously, we can't do that because
of the requirement of maintaining a single transaction in tablesync
workers.
Bump catalog version due to change of state in the catalog
(pg_subscription_rel).
Author: Peter Smith, Amit Kapila, and Takamichi Osumi
Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila
Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
2021-02-12 03:11:51 +01:00
|
|
|
/* translator: %s is an SQL ALTER command */
|
|
|
|
errhint("Use %s to disassociate the subscription from the slot.",
|
|
|
|
"ALTER SUBSCRIPTION ... SET (slot_name = NONE)")));
|
|
|
|
}
|
2021-04-06 10:44:26 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Check for duplicates in the given list of publications and error out if
|
|
|
|
* found one. Add publications to datums as text datums, if datums is not
|
|
|
|
* NULL.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
check_duplicates_in_publist(List *publist, Datum *datums)
|
|
|
|
{
|
|
|
|
ListCell *cell;
|
|
|
|
int j = 0;
|
|
|
|
|
|
|
|
foreach(cell, publist)
|
|
|
|
{
|
|
|
|
char *name = strVal(lfirst(cell));
|
|
|
|
ListCell *pcell;
|
|
|
|
|
|
|
|
foreach(pcell, publist)
|
|
|
|
{
|
|
|
|
char *pname = strVal(lfirst(pcell));
|
|
|
|
|
|
|
|
if (pcell == cell)
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (strcmp(name, pname) == 0)
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_DUPLICATE_OBJECT),
|
2021-04-06 10:44:26 +02:00
|
|
|
errmsg("publication name \"%s\" used more than once",
|
|
|
|
pname)));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (datums)
|
|
|
|
datums[j++] = CStringGetTextDatum(name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Merge current subscription's publications and user-specified publications
|
|
|
|
* from ADD/DROP PUBLICATIONS.
|
|
|
|
*
|
|
|
|
* If addpub is true, we will add the list of publications into oldpublist.
|
|
|
|
* Otherwise, we will delete the list of publications from oldpublist. The
|
|
|
|
* returned list is a copy, oldpublist itself is not changed.
|
|
|
|
*
|
|
|
|
* subname is the subscription name, for error messages.
|
|
|
|
*/
|
|
|
|
static List *
|
|
|
|
merge_publications(List *oldpublist, List *newpublist, bool addpub, const char *subname)
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
oldpublist = list_copy(oldpublist);
|
|
|
|
|
|
|
|
check_duplicates_in_publist(newpublist, NULL);
|
|
|
|
|
|
|
|
foreach(lc, newpublist)
|
|
|
|
{
|
|
|
|
char *name = strVal(lfirst(lc));
|
|
|
|
ListCell *lc2;
|
|
|
|
bool found = false;
|
|
|
|
|
|
|
|
foreach(lc2, oldpublist)
|
|
|
|
{
|
|
|
|
char *pubname = strVal(lfirst(lc2));
|
|
|
|
|
|
|
|
if (strcmp(name, pubname) == 0)
|
|
|
|
{
|
|
|
|
found = true;
|
|
|
|
if (addpub)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_DUPLICATE_OBJECT),
|
|
|
|
errmsg("publication \"%s\" is already in subscription \"%s\"",
|
|
|
|
name, subname)));
|
|
|
|
else
|
|
|
|
oldpublist = foreach_delete_current(oldpublist, lc2);
|
|
|
|
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (addpub && !found)
|
|
|
|
oldpublist = lappend(oldpublist, makeString(name));
|
|
|
|
else if (!addpub && !found)
|
|
|
|
ereport(ERROR,
|
2021-06-16 17:52:05 +02:00
|
|
|
(errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
|
2021-04-06 10:44:26 +02:00
|
|
|
errmsg("publication \"%s\" is not in subscription \"%s\"",
|
|
|
|
name, subname)));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* XXX Probably no strong reason for this, but for now it's to make ALTER
|
|
|
|
* SUBSCRIPTION ... DROP PUBLICATION consistent with SET PUBLICATION.
|
|
|
|
*/
|
|
|
|
if (!oldpublist)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
|
2021-06-25 09:51:24 +02:00
|
|
|
errmsg("cannot drop all the publications from a subscription")));
|
2021-04-06 10:44:26 +02:00
|
|
|
|
|
|
|
return oldpublist;
|
|
|
|
}
|