2013-02-21 11:26:23 +01:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* postgres_fdw.c
|
|
|
|
* Foreign-data wrapper for remote PostgreSQL servers
|
|
|
|
*
|
2021-01-02 19:06:25 +01:00
|
|
|
* Portions Copyright (c) 2012-2021, PostgreSQL Global Development Group
|
2013-02-21 11:26:23 +01:00
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
|
|
|
* contrib/postgres_fdw/postgres_fdw.c
|
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
#include "postgres.h"
|
|
|
|
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
#include <limits.h>
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
#include "access/htup_details.h"
|
2013-03-10 19:14:53 +01:00
|
|
|
#include "access/sysattr.h"
|
2019-01-21 19:18:20 +01:00
|
|
|
#include "access/table.h"
|
2017-03-10 05:36:44 +01:00
|
|
|
#include "catalog/pg_class.h"
|
2013-02-21 11:26:23 +01:00
|
|
|
#include "commands/defrem.h"
|
|
|
|
#include "commands/explain.h"
|
|
|
|
#include "commands/vacuum.h"
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
#include "executor/execAsync.h"
|
2013-02-21 11:26:23 +01:00
|
|
|
#include "foreign/fdwapi.h"
|
|
|
|
#include "funcapi.h"
|
|
|
|
#include "miscadmin.h"
|
2013-03-10 19:14:53 +01:00
|
|
|
#include "nodes/makefuncs.h"
|
2013-03-22 00:43:59 +01:00
|
|
|
#include "nodes/nodeFuncs.h"
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
#include "optimizer/appendinfo.h"
|
2016-10-21 15:54:29 +02:00
|
|
|
#include "optimizer/clauses.h"
|
2019-06-11 06:39:31 +02:00
|
|
|
#include "optimizer/cost.h"
|
2019-01-29 21:48:51 +01:00
|
|
|
#include "optimizer/optimizer.h"
|
2013-02-21 11:26:23 +01:00
|
|
|
#include "optimizer/pathnode.h"
|
2013-03-22 00:43:59 +01:00
|
|
|
#include "optimizer/paths.h"
|
2013-02-21 11:26:23 +01:00
|
|
|
#include "optimizer/planmain.h"
|
2020-11-24 09:45:00 +01:00
|
|
|
#include "optimizer/prep.h"
|
2013-03-22 00:43:59 +01:00
|
|
|
#include "optimizer/restrictinfo.h"
|
2016-02-09 20:00:50 +01:00
|
|
|
#include "optimizer/tlist.h"
|
2013-02-21 11:26:23 +01:00
|
|
|
#include "parser/parsetree.h"
|
2019-10-23 05:56:22 +02:00
|
|
|
#include "postgres_fdw.h"
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
#include "storage/latch.h"
|
2013-03-10 19:14:53 +01:00
|
|
|
#include "utils/builtins.h"
|
2018-07-29 03:30:48 +02:00
|
|
|
#include "utils/float.h"
|
2013-03-12 02:31:28 +01:00
|
|
|
#include "utils/guc.h"
|
2013-02-21 11:26:23 +01:00
|
|
|
#include "utils/lsyscache.h"
|
|
|
|
#include "utils/memutils.h"
|
2014-11-15 01:48:53 +01:00
|
|
|
#include "utils/rel.h"
|
2015-05-15 04:02:54 +02:00
|
|
|
#include "utils/sampling.h"
|
2016-10-21 15:54:29 +02:00
|
|
|
#include "utils/selfuncs.h"
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
PG_MODULE_MAGIC;
|
|
|
|
|
|
|
|
/* Default CPU cost to start up a foreign query. */
|
|
|
|
#define DEFAULT_FDW_STARTUP_COST 100.0
|
|
|
|
|
|
|
|
/* Default CPU cost to process 1 row (above and beyond cpu_tuple_cost). */
|
|
|
|
#define DEFAULT_FDW_TUPLE_COST 0.01
|
|
|
|
|
2015-11-03 18:46:06 +01:00
|
|
|
/* If no remote estimates, assume a sort costs 20% extra */
|
|
|
|
#define DEFAULT_FDW_SORT_MULTIPLIER 1.2
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
2013-03-10 19:14:53 +01:00
|
|
|
* Indexes of FDW-private information stored in fdw_private lists.
|
2013-02-21 11:26:23 +01:00
|
|
|
*
|
2013-03-10 19:14:53 +01:00
|
|
|
* These items are indexed with the enum FdwScanPrivateIndex, so an item
|
|
|
|
* can be fetched with list_nth(). For example, to get the SELECT statement:
|
|
|
|
* sql = strVal(list_nth(fdw_private, FdwScanPrivateSelectSql));
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
enum FdwScanPrivateIndex
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
|
|
|
/* SQL statement to execute remotely (as a String node) */
|
2013-03-22 05:31:11 +01:00
|
|
|
FdwScanPrivateSelectSql,
|
|
|
|
/* Integer list of attribute numbers retrieved by the SELECT */
|
2016-02-03 15:01:59 +01:00
|
|
|
FdwScanPrivateRetrievedAttrs,
|
|
|
|
/* Integer representing the desired fetch_size */
|
2016-02-09 20:00:50 +01:00
|
|
|
FdwScanPrivateFetchSize,
|
|
|
|
|
|
|
|
/*
|
|
|
|
* String describing join i.e. names of relations being joined and types
|
|
|
|
* of join, added when the scan is join
|
|
|
|
*/
|
|
|
|
FdwScanPrivateRelations
|
2013-03-10 19:14:53 +01:00
|
|
|
};
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/*
|
|
|
|
* Similarly, this enum describes what's kept in the fdw_private list for
|
|
|
|
* a ModifyTable node referencing a postgres_fdw foreign table. We store:
|
|
|
|
*
|
|
|
|
* 1) INSERT/UPDATE/DELETE statement text to be sent to the remote server
|
|
|
|
* 2) Integer list of target attribute numbers for INSERT/UPDATE
|
|
|
|
* (NIL for a DELETE)
|
2021-01-20 23:05:46 +01:00
|
|
|
* 3) Length till the end of VALUES clause for INSERT
|
|
|
|
* (-1 for a DELETE/UPDATE)
|
|
|
|
* 4) Boolean flag showing if the remote query has a RETURNING clause
|
|
|
|
* 5) Integer list of attribute numbers retrieved by RETURNING, if any
|
2013-03-10 19:14:53 +01:00
|
|
|
*/
|
|
|
|
enum FdwModifyPrivateIndex
|
|
|
|
{
|
|
|
|
/* SQL statement to execute remotely (as a String node) */
|
|
|
|
FdwModifyPrivateUpdateSql,
|
|
|
|
/* Integer list of target attribute numbers for INSERT/UPDATE */
|
|
|
|
FdwModifyPrivateTargetAttnums,
|
2021-01-20 23:05:46 +01:00
|
|
|
/* Length till the end of VALUES clause (as an integer Value node) */
|
|
|
|
FdwModifyPrivateLen,
|
2013-03-10 19:14:53 +01:00
|
|
|
/* has-returning flag (as an integer Value node) */
|
2013-03-22 05:31:11 +01:00
|
|
|
FdwModifyPrivateHasReturning,
|
|
|
|
/* Integer list of attribute numbers retrieved by RETURNING */
|
|
|
|
FdwModifyPrivateRetrievedAttrs
|
2013-02-21 11:26:23 +01:00
|
|
|
};
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* Similarly, this enum describes what's kept in the fdw_private list for
|
|
|
|
* a ForeignScan node that modifies a foreign table directly. We store:
|
|
|
|
*
|
|
|
|
* 1) UPDATE/DELETE statement text to be sent to the remote server
|
|
|
|
* 2) Boolean flag showing if the remote query has a RETURNING clause
|
|
|
|
* 3) Integer list of attribute numbers retrieved by RETURNING, if any
|
|
|
|
* 4) Boolean flag showing if we set the command es_processed
|
|
|
|
*/
|
|
|
|
enum FdwDirectModifyPrivateIndex
|
|
|
|
{
|
|
|
|
/* SQL statement to execute remotely (as a String node) */
|
|
|
|
FdwDirectModifyPrivateUpdateSql,
|
|
|
|
/* has-returning flag (as an integer Value node) */
|
|
|
|
FdwDirectModifyPrivateHasReturning,
|
|
|
|
/* Integer list of attribute numbers retrieved by RETURNING */
|
|
|
|
FdwDirectModifyPrivateRetrievedAttrs,
|
|
|
|
/* set-processed flag (as an integer Value node) */
|
|
|
|
FdwDirectModifyPrivateSetProcessed
|
|
|
|
};
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* Execution state of a foreign scan using postgres_fdw.
|
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
typedef struct PgFdwScanState
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
2016-02-09 20:00:50 +01:00
|
|
|
Relation rel; /* relcache entry for the foreign table. NULL
|
|
|
|
* for a foreign join scan. */
|
|
|
|
TupleDesc tupdesc; /* tuple descriptor of scan */
|
2013-02-21 11:26:23 +01:00
|
|
|
AttInMetadata *attinmeta; /* attribute datatype conversion metadata */
|
|
|
|
|
2013-03-22 05:31:11 +01:00
|
|
|
/* extracted fdw_private data */
|
|
|
|
char *query; /* text of SELECT command */
|
|
|
|
List *retrieved_attrs; /* list of retrieved attribute numbers */
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* for remote query execution */
|
|
|
|
PGconn *conn; /* connection for the scan */
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
PgFdwConnState *conn_state; /* extra per-connection state */
|
2013-02-21 11:26:23 +01:00
|
|
|
unsigned int cursor_number; /* quasi-unique ID for my cursor */
|
|
|
|
bool cursor_exists; /* have we created the cursor? */
|
|
|
|
int numParams; /* number of parameters passed to query */
|
2013-03-22 00:43:59 +01:00
|
|
|
FmgrInfo *param_flinfo; /* output conversion functions for them */
|
|
|
|
List *param_exprs; /* executable expressions for param values */
|
|
|
|
const char **param_values; /* textual values of query parameters */
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* for storing result tuples */
|
|
|
|
HeapTuple *tuples; /* array of currently-retrieved tuples */
|
|
|
|
int num_tuples; /* # of tuples in array */
|
|
|
|
int next_tuple; /* index of next one to return */
|
|
|
|
|
|
|
|
/* batch-level state, for optimizing rewinds and avoiding useless fetch */
|
|
|
|
int fetch_ct_2; /* Min(# of fetches done, 2) */
|
|
|
|
bool eof_reached; /* true if last fetch reached EOF */
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/* for asynchronous execution */
|
|
|
|
bool async_capable; /* engage asynchronous-capable logic? */
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/* working memory contexts */
|
|
|
|
MemoryContext batch_cxt; /* context holding current batch of tuples */
|
|
|
|
MemoryContext temp_cxt; /* context for per-tuple temporary data */
|
2016-02-03 15:01:59 +01:00
|
|
|
|
|
|
|
int fetch_size; /* number of tuples per fetch */
|
2013-03-10 19:14:53 +01:00
|
|
|
} PgFdwScanState;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Execution state of a foreign insert/update/delete operation.
|
|
|
|
*/
|
|
|
|
typedef struct PgFdwModifyState
|
|
|
|
{
|
|
|
|
Relation rel; /* relcache entry for the foreign table */
|
|
|
|
AttInMetadata *attinmeta; /* attribute datatype conversion metadata */
|
|
|
|
|
|
|
|
/* for remote query execution */
|
|
|
|
PGconn *conn; /* connection for the scan */
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
PgFdwConnState *conn_state; /* extra per-connection state */
|
2013-03-10 19:14:53 +01:00
|
|
|
char *p_name; /* name of prepared statement, if created */
|
|
|
|
|
|
|
|
/* extracted fdw_private data */
|
|
|
|
char *query; /* text of INSERT/UPDATE/DELETE command */
|
2021-01-20 23:05:46 +01:00
|
|
|
char *orig_query; /* original text of INSERT command */
|
2013-03-10 19:14:53 +01:00
|
|
|
List *target_attrs; /* list of target attribute numbers */
|
2021-01-20 23:05:46 +01:00
|
|
|
int values_end; /* length up to the end of VALUES */
|
|
|
|
int batch_size; /* value of FDW option "batch_size" */
|
2013-03-10 19:14:53 +01:00
|
|
|
bool has_returning; /* is there a RETURNING clause? */
|
2013-03-22 05:31:11 +01:00
|
|
|
List *retrieved_attrs; /* attr numbers retrieved by RETURNING */
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
/* info about parameters for prepared statement */
|
|
|
|
AttrNumber ctidAttno; /* attnum of input resjunk ctid column */
|
|
|
|
int p_nums; /* number of parameters to transmit */
|
|
|
|
FmgrInfo *p_flinfo; /* output conversion functions for them */
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
/* batch operation stuff */
|
|
|
|
int num_slots; /* number of slots to insert */
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/* working memory context */
|
|
|
|
MemoryContext temp_cxt; /* context for per-tuple temporary data */
|
2019-04-24 11:31:50 +02:00
|
|
|
|
|
|
|
/* for update row movement if subplan result rel */
|
|
|
|
struct PgFdwModifyState *aux_fmstate; /* foreign-insert state, if
|
|
|
|
* created */
|
2013-03-10 19:14:53 +01:00
|
|
|
} PgFdwModifyState;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* Execution state of a foreign scan that modifies a foreign table directly.
|
|
|
|
*/
|
|
|
|
typedef struct PgFdwDirectModifyState
|
|
|
|
{
|
|
|
|
Relation rel; /* relcache entry for the foreign table */
|
|
|
|
AttInMetadata *attinmeta; /* attribute datatype conversion metadata */
|
|
|
|
|
|
|
|
/* extracted fdw_private data */
|
|
|
|
char *query; /* text of UPDATE/DELETE command */
|
|
|
|
bool has_returning; /* is there a RETURNING clause? */
|
|
|
|
List *retrieved_attrs; /* attr numbers retrieved by RETURNING */
|
|
|
|
bool set_processed; /* do we set the command es_processed? */
|
|
|
|
|
|
|
|
/* for remote query execution */
|
|
|
|
PGconn *conn; /* connection for the update */
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
PgFdwConnState *conn_state; /* extra per-connection state */
|
2016-03-18 18:48:58 +01:00
|
|
|
int numParams; /* number of parameters passed to query */
|
|
|
|
FmgrInfo *param_flinfo; /* output conversion functions for them */
|
|
|
|
List *param_exprs; /* executable expressions for param values */
|
|
|
|
const char **param_values; /* textual values of query parameters */
|
|
|
|
|
|
|
|
/* for storing result tuples */
|
|
|
|
PGresult *result; /* result for query */
|
|
|
|
int num_tuples; /* # of result tuples */
|
|
|
|
int next_tuple; /* index of next one to return */
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
Relation resultRel; /* relcache entry for the target relation */
|
|
|
|
AttrNumber *attnoMap; /* array of attnums of input user columns */
|
|
|
|
AttrNumber ctidAttno; /* attnum of input ctid column */
|
|
|
|
AttrNumber oidAttno; /* attnum of input oid column */
|
|
|
|
bool hasSystemCols; /* are there system columns of resultRel? */
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/* working memory context */
|
|
|
|
MemoryContext temp_cxt; /* context for per-tuple temporary data */
|
|
|
|
} PgFdwDirectModifyState;
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* Workspace for analyzing a foreign table.
|
|
|
|
*/
|
|
|
|
typedef struct PgFdwAnalyzeState
|
|
|
|
{
|
|
|
|
Relation rel; /* relcache entry for the foreign table */
|
|
|
|
AttInMetadata *attinmeta; /* attribute datatype conversion metadata */
|
2013-03-22 05:31:11 +01:00
|
|
|
List *retrieved_attrs; /* attr numbers retrieved by query */
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* collected sample rows */
|
|
|
|
HeapTuple *rows; /* array of size targrows */
|
|
|
|
int targrows; /* target # of sample rows */
|
|
|
|
int numrows; /* # of sample rows collected */
|
|
|
|
|
|
|
|
/* for random sampling */
|
|
|
|
double samplerows; /* # of rows fetched */
|
|
|
|
double rowstoskip; /* # of rows to skip before next sample */
|
2015-05-15 04:02:54 +02:00
|
|
|
ReservoirStateData rstate; /* state for reservoir sampling */
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* working memory contexts */
|
|
|
|
MemoryContext anl_cxt; /* context for per-analyze lifespan data */
|
|
|
|
MemoryContext temp_cxt; /* context for per-tuple temporary data */
|
|
|
|
} PgFdwAnalyzeState;
|
|
|
|
|
2019-04-02 12:20:30 +02:00
|
|
|
/*
|
|
|
|
* This enum describes what's kept in the fdw_private list for a ForeignPath.
|
|
|
|
* We store:
|
|
|
|
*
|
|
|
|
* 1) Boolean flag showing if the remote query has the final sort
|
2019-04-02 13:30:45 +02:00
|
|
|
* 2) Boolean flag showing if the remote query has the LIMIT clause
|
2019-04-02 12:20:30 +02:00
|
|
|
*/
|
|
|
|
enum FdwPathPrivateIndex
|
|
|
|
{
|
|
|
|
/* has-final-sort flag (as an integer Value node) */
|
2019-04-02 13:30:45 +02:00
|
|
|
FdwPathPrivateHasFinalSort,
|
|
|
|
/* has-limit flag (as an integer Value node) */
|
|
|
|
FdwPathPrivateHasLimit
|
2019-04-02 12:20:30 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Struct for extra information passed to estimate_path_cost_size() */
|
|
|
|
typedef struct
|
|
|
|
{
|
|
|
|
PathTarget *target;
|
|
|
|
bool has_final_sort;
|
2019-04-02 13:30:45 +02:00
|
|
|
bool has_limit;
|
|
|
|
double limit_tuples;
|
|
|
|
int64 count_est;
|
|
|
|
int64 offset_est;
|
2019-04-02 12:20:30 +02:00
|
|
|
} PgFdwPathExtraData;
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* Identify the attribute where data conversion fails.
|
|
|
|
*/
|
|
|
|
typedef struct ConversionLocation
|
|
|
|
{
|
2016-02-09 20:00:50 +01:00
|
|
|
Relation rel; /* foreign table's relcache entry. */
|
2013-02-21 11:26:23 +01:00
|
|
|
AttrNumber cur_attno; /* attribute number being processed, or 0 */
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* In case of foreign join push down, fdw_scan_tlist is used to identify
|
|
|
|
* the Var node corresponding to the error location and
|
|
|
|
* fsstate->ss.ps.state gives access to the RTEs of corresponding relation
|
|
|
|
* to get the relation name and attribute name.
|
|
|
|
*/
|
|
|
|
ForeignScanState *fsstate;
|
2013-02-21 11:26:23 +01:00
|
|
|
} ConversionLocation;
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/* Callback argument for ec_member_matches_foreign */
|
|
|
|
typedef struct
|
|
|
|
{
|
|
|
|
Expr *current; /* current expr, or NULL if not yet found */
|
|
|
|
List *already_used; /* expressions already dealt with */
|
|
|
|
} ec_member_foreign_arg;
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* SQL functions
|
|
|
|
*/
|
|
|
|
PG_FUNCTION_INFO_V1(postgres_fdw_handler);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* FDW callback routines
|
|
|
|
*/
|
|
|
|
static void postgresGetForeignRelSize(PlannerInfo *root,
|
|
|
|
RelOptInfo *baserel,
|
|
|
|
Oid foreigntableid);
|
|
|
|
static void postgresGetForeignPaths(PlannerInfo *root,
|
|
|
|
RelOptInfo *baserel,
|
|
|
|
Oid foreigntableid);
|
|
|
|
static ForeignScan *postgresGetForeignPlan(PlannerInfo *root,
|
2017-09-06 23:52:08 +02:00
|
|
|
RelOptInfo *foreignrel,
|
2013-02-21 11:26:23 +01:00
|
|
|
Oid foreigntableid,
|
|
|
|
ForeignPath *best_path,
|
|
|
|
List *tlist,
|
Allow foreign and custom joins to handle EvalPlanQual rechecks.
Commit e7cb7ee14555cc9c5773e2c102efd6371f6f2005 provided basic
infrastructure for allowing a foreign data wrapper or custom scan
provider to replace a join of one or more tables with a scan.
However, this infrastructure failed to take into account the need
for possible EvalPlanQual rechecks, and ExecScanFetch would fail
an assertion (or just overwrite memory) if such a check was attempted
for a plan containing a pushed-down join. To fix, adjust the EPQ
machinery to skip some processing steps when scanrelid == 0, making
those the responsibility of scan's recheck method, which also has
the responsibility in this case of correctly populating the relevant
slot.
To allow foreign scans to gain control in the right place to make
use of this new facility, add a new, optional RecheckForeignScan
method. Also, allow a foreign scan to have a child plan, which can
be used to correctly populate the slot (or perhaps for something
else, but this is the only use currently envisioned).
KaiGai Kohei, reviewed by Robert Haas, Etsuro Fujita, and Kyotaro
Horiguchi.
2015-12-08 18:31:03 +01:00
|
|
|
List *scan_clauses,
|
|
|
|
Plan *outer_plan);
|
2013-02-21 11:26:23 +01:00
|
|
|
static void postgresBeginForeignScan(ForeignScanState *node, int eflags);
|
|
|
|
static TupleTableSlot *postgresIterateForeignScan(ForeignScanState *node);
|
|
|
|
static void postgresReScanForeignScan(ForeignScanState *node);
|
|
|
|
static void postgresEndForeignScan(ForeignScanState *node);
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
static void postgresAddForeignUpdateTargets(PlannerInfo *root,
|
|
|
|
Index rtindex,
|
2013-03-10 19:14:53 +01:00
|
|
|
RangeTblEntry *target_rte,
|
|
|
|
Relation target_relation);
|
|
|
|
static List *postgresPlanForeignModify(PlannerInfo *root,
|
|
|
|
ModifyTable *plan,
|
|
|
|
Index resultRelation,
|
|
|
|
int subplan_index);
|
|
|
|
static void postgresBeginForeignModify(ModifyTableState *mtstate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
List *fdw_private,
|
|
|
|
int subplan_index,
|
|
|
|
int eflags);
|
|
|
|
static TupleTableSlot *postgresExecForeignInsert(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
TupleTableSlot *slot,
|
|
|
|
TupleTableSlot *planSlot);
|
2021-01-20 23:05:46 +01:00
|
|
|
static TupleTableSlot **postgresExecForeignBatchInsert(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
TupleTableSlot **slots,
|
|
|
|
TupleTableSlot **planSlots,
|
|
|
|
int *numSlots);
|
|
|
|
static int postgresGetForeignModifyBatchSize(ResultRelInfo *resultRelInfo);
|
2013-03-10 19:14:53 +01:00
|
|
|
static TupleTableSlot *postgresExecForeignUpdate(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
TupleTableSlot *slot,
|
|
|
|
TupleTableSlot *planSlot);
|
|
|
|
static TupleTableSlot *postgresExecForeignDelete(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
TupleTableSlot *slot,
|
|
|
|
TupleTableSlot *planSlot);
|
|
|
|
static void postgresEndForeignModify(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo);
|
2018-04-07 01:16:11 +02:00
|
|
|
static void postgresBeginForeignInsert(ModifyTableState *mtstate,
|
|
|
|
ResultRelInfo *resultRelInfo);
|
|
|
|
static void postgresEndForeignInsert(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo);
|
2013-06-12 23:52:54 +02:00
|
|
|
static int postgresIsForeignRelUpdatable(Relation rel);
|
2016-03-18 18:48:58 +01:00
|
|
|
static bool postgresPlanDirectModify(PlannerInfo *root,
|
|
|
|
ModifyTable *plan,
|
|
|
|
Index resultRelation,
|
|
|
|
int subplan_index);
|
|
|
|
static void postgresBeginDirectModify(ForeignScanState *node, int eflags);
|
|
|
|
static TupleTableSlot *postgresIterateDirectModify(ForeignScanState *node);
|
|
|
|
static void postgresEndDirectModify(ForeignScanState *node);
|
2013-03-10 19:14:53 +01:00
|
|
|
static void postgresExplainForeignScan(ForeignScanState *node,
|
|
|
|
ExplainState *es);
|
|
|
|
static void postgresExplainForeignModify(ModifyTableState *mtstate,
|
|
|
|
ResultRelInfo *rinfo,
|
|
|
|
List *fdw_private,
|
|
|
|
int subplan_index,
|
|
|
|
ExplainState *es);
|
2016-03-18 18:48:58 +01:00
|
|
|
static void postgresExplainDirectModify(ForeignScanState *node,
|
|
|
|
ExplainState *es);
|
Allow TRUNCATE command to truncate foreign tables.
This commit introduces new foreign data wrapper API for TRUNCATE.
It extends TRUNCATE command so that it accepts foreign tables as
the targets to truncate and invokes that API. Also it extends postgres_fdw
so that it can issue TRUNCATE command to foreign servers, by adding
new routine for that TRUNCATE API.
The information about options specified in TRUNCATE command, e.g.,
ONLY, CACADE, etc is passed to FDW via API. The list of foreign tables to
truncate is also passed to FDW. FDW truncates the foreign data sources
that the passed foreign tables specify, based on those information.
For example, postgres_fdw constructs TRUNCATE command using them
and issues it to the foreign server.
For performance, TRUNCATE command invokes the FDW routine for
TRUNCATE once per foreign server that foreign tables to truncate belong to.
Author: Kazutaka Onishi, Kohei KaiGai, slightly modified by Fujii Masao
Reviewed-by: Bharath Rupireddy, Michael Paquier, Zhihong Yu, Alvaro Herrera, Stephen Frost, Ashutosh Bapat, Amit Langote, Daniel Gustafsson, Ibrar Ahmed, Fujii Masao
Discussion: https://postgr.es/m/CAOP8fzb_gkReLput7OvOK+8NHgw-RKqNv59vem7=524krQTcWA@mail.gmail.com
Discussion: https://postgr.es/m/CAJuF6cMWDDqU-vn_knZgma+2GMaout68YUgn1uyDnexRhqqM5Q@mail.gmail.com
2021-04-08 13:56:08 +02:00
|
|
|
static void postgresExecForeignTruncate(List *rels,
|
|
|
|
DropBehavior behavior,
|
|
|
|
bool restart_seqs);
|
2013-02-21 11:26:23 +01:00
|
|
|
static bool postgresAnalyzeForeignTable(Relation relation,
|
|
|
|
AcquireSampleRowsFunc *func,
|
|
|
|
BlockNumber *totalpages);
|
2014-07-10 21:01:31 +02:00
|
|
|
static List *postgresImportForeignSchema(ImportForeignSchemaStmt *stmt,
|
|
|
|
Oid serverOid);
|
2016-02-09 20:00:50 +01:00
|
|
|
static void postgresGetForeignJoinPaths(PlannerInfo *root,
|
|
|
|
RelOptInfo *joinrel,
|
|
|
|
RelOptInfo *outerrel,
|
|
|
|
RelOptInfo *innerrel,
|
|
|
|
JoinType jointype,
|
|
|
|
JoinPathExtraData *extra);
|
|
|
|
static bool postgresRecheckForeignScan(ForeignScanState *node,
|
|
|
|
TupleTableSlot *slot);
|
2016-10-21 15:54:29 +02:00
|
|
|
static void postgresGetForeignUpperPaths(PlannerInfo *root,
|
|
|
|
UpperRelationKind stage,
|
|
|
|
RelOptInfo *input_rel,
|
2018-04-02 16:51:50 +02:00
|
|
|
RelOptInfo *output_rel,
|
|
|
|
void *extra);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
static bool postgresIsForeignPathAsyncCapable(ForeignPath *path);
|
|
|
|
static void postgresForeignAsyncRequest(AsyncRequest *areq);
|
|
|
|
static void postgresForeignAsyncConfigureWait(AsyncRequest *areq);
|
|
|
|
static void postgresForeignAsyncNotify(AsyncRequest *areq);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Helper functions
|
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
static void estimate_path_cost_size(PlannerInfo *root,
|
2018-01-11 17:53:59 +01:00
|
|
|
RelOptInfo *foreignrel,
|
|
|
|
List *param_join_conds,
|
2015-11-03 18:46:06 +01:00
|
|
|
List *pathkeys,
|
2019-04-02 12:20:30 +02:00
|
|
|
PgFdwPathExtraData *fpextra,
|
2013-03-22 00:43:59 +01:00
|
|
|
double *p_rows, int *p_width,
|
|
|
|
Cost *p_startup_cost, Cost *p_total_cost);
|
2013-02-21 11:26:23 +01:00
|
|
|
static void get_remote_estimate(const char *sql,
|
|
|
|
PGconn *conn,
|
|
|
|
double *rows,
|
|
|
|
int *width,
|
|
|
|
Cost *startup_cost,
|
|
|
|
Cost *total_cost);
|
2019-04-02 12:20:30 +02:00
|
|
|
static void adjust_foreign_grouping_path_cost(PlannerInfo *root,
|
|
|
|
List *pathkeys,
|
|
|
|
double retrieved_rows,
|
|
|
|
double width,
|
2019-04-02 13:30:45 +02:00
|
|
|
double limit_tuples,
|
2019-04-02 12:20:30 +02:00
|
|
|
Cost *p_startup_cost,
|
|
|
|
Cost *p_run_cost);
|
2013-03-22 00:43:59 +01:00
|
|
|
static bool ec_member_matches_foreign(PlannerInfo *root, RelOptInfo *rel,
|
|
|
|
EquivalenceClass *ec, EquivalenceMember *em,
|
|
|
|
void *arg);
|
2013-02-21 11:26:23 +01:00
|
|
|
static void create_cursor(ForeignScanState *node);
|
|
|
|
static void fetch_more_data(ForeignScanState *node);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
static void close_cursor(PGconn *conn, unsigned int cursor_number,
|
|
|
|
PgFdwConnState *conn_state);
|
2018-04-06 17:29:43 +02:00
|
|
|
static PgFdwModifyState *create_foreign_modify(EState *estate,
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
RangeTblEntry *rte,
|
2018-04-06 17:29:43 +02:00
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
CmdType operation,
|
|
|
|
Plan *subplan,
|
|
|
|
char *query,
|
|
|
|
List *target_attrs,
|
2021-01-20 23:05:46 +01:00
|
|
|
int len,
|
2018-04-06 17:29:43 +02:00
|
|
|
bool has_returning,
|
|
|
|
List *retrieved_attrs);
|
2021-01-20 23:05:46 +01:00
|
|
|
static TupleTableSlot **execute_foreign_modify(EState *estate,
|
2019-01-17 06:37:33 +01:00
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
CmdType operation,
|
2021-01-20 23:05:46 +01:00
|
|
|
TupleTableSlot **slots,
|
|
|
|
TupleTableSlot **planSlots,
|
|
|
|
int *numSlots);
|
2013-03-10 19:14:53 +01:00
|
|
|
static void prepare_foreign_modify(PgFdwModifyState *fmstate);
|
|
|
|
static const char **convert_prep_stmt_params(PgFdwModifyState *fmstate,
|
|
|
|
ItemPointer tupleid,
|
2021-01-20 23:05:46 +01:00
|
|
|
TupleTableSlot **slots,
|
|
|
|
int numSlots);
|
2013-03-10 19:14:53 +01:00
|
|
|
static void store_returning_result(PgFdwModifyState *fmstate,
|
|
|
|
TupleTableSlot *slot, PGresult *res);
|
2018-04-06 17:29:43 +02:00
|
|
|
static void finish_foreign_modify(PgFdwModifyState *fmstate);
|
2021-01-20 23:05:46 +01:00
|
|
|
static void deallocate_query(PgFdwModifyState *fmstate);
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
static List *build_remote_returning(Index rtindex, Relation rel,
|
|
|
|
List *returningList);
|
|
|
|
static void rebuild_fdw_scan_tlist(ForeignScan *fscan, List *tlist);
|
2016-03-18 18:48:58 +01:00
|
|
|
static void execute_dml_stmt(ForeignScanState *node);
|
|
|
|
static TupleTableSlot *get_returning_data(ForeignScanState *node);
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
static void init_returning_filter(PgFdwDirectModifyState *dmstate,
|
|
|
|
List *fdw_scan_tlist,
|
|
|
|
Index rtindex);
|
|
|
|
static TupleTableSlot *apply_returning_filter(PgFdwDirectModifyState *dmstate,
|
2020-10-14 09:58:38 +02:00
|
|
|
ResultRelInfo *resultRelInfo,
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
TupleTableSlot *slot,
|
|
|
|
EState *estate);
|
2016-03-18 18:48:58 +01:00
|
|
|
static void prepare_query_params(PlanState *node,
|
|
|
|
List *fdw_exprs,
|
|
|
|
int numParams,
|
|
|
|
FmgrInfo **param_flinfo,
|
|
|
|
List **param_exprs,
|
|
|
|
const char ***param_values);
|
|
|
|
static void process_query_params(ExprContext *econtext,
|
|
|
|
FmgrInfo *param_flinfo,
|
|
|
|
List *param_exprs,
|
|
|
|
const char **param_values);
|
2013-02-21 11:26:23 +01:00
|
|
|
static int postgresAcquireSampleRowsFunc(Relation relation, int elevel,
|
|
|
|
HeapTuple *rows, int targrows,
|
|
|
|
double *totalrows,
|
|
|
|
double *totaldeadrows);
|
|
|
|
static void analyze_row_processor(PGresult *res, int row,
|
|
|
|
PgFdwAnalyzeState *astate);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
static void produce_tuple_asynchronously(AsyncRequest *areq, bool fetch);
|
|
|
|
static void fetch_more_data_begin(AsyncRequest *areq);
|
2013-02-21 11:26:23 +01:00
|
|
|
static HeapTuple make_tuple_from_result_row(PGresult *res,
|
|
|
|
int row,
|
|
|
|
Relation rel,
|
|
|
|
AttInMetadata *attinmeta,
|
2013-03-22 05:31:11 +01:00
|
|
|
List *retrieved_attrs,
|
2016-02-09 20:00:50 +01:00
|
|
|
ForeignScanState *fsstate,
|
2013-02-21 11:26:23 +01:00
|
|
|
MemoryContext temp_context);
|
|
|
|
static void conversion_error_callback(void *arg);
|
2016-02-09 20:00:50 +01:00
|
|
|
static bool foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel,
|
|
|
|
JoinType jointype, RelOptInfo *outerrel, RelOptInfo *innerrel,
|
|
|
|
JoinPathExtraData *extra);
|
2018-04-02 16:51:50 +02:00
|
|
|
static bool foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel,
|
|
|
|
Node *havingQual);
|
2016-03-09 16:51:49 +01:00
|
|
|
static List *get_useful_pathkeys_for_relation(PlannerInfo *root,
|
|
|
|
RelOptInfo *rel);
|
|
|
|
static List *get_useful_ecs_for_relation(PlannerInfo *root, RelOptInfo *rel);
|
|
|
|
static void add_paths_with_pathkeys_for_rel(PlannerInfo *root, RelOptInfo *rel,
|
|
|
|
Path *epq_path);
|
2016-10-21 15:54:29 +02:00
|
|
|
static void add_foreign_grouping_paths(PlannerInfo *root,
|
|
|
|
RelOptInfo *input_rel,
|
2018-04-02 16:51:50 +02:00
|
|
|
RelOptInfo *grouped_rel,
|
|
|
|
GroupPathExtraData *extra);
|
2019-04-02 12:20:30 +02:00
|
|
|
static void add_foreign_ordered_paths(PlannerInfo *root,
|
|
|
|
RelOptInfo *input_rel,
|
|
|
|
RelOptInfo *ordered_rel);
|
2019-04-02 13:30:45 +02:00
|
|
|
static void add_foreign_final_paths(PlannerInfo *root,
|
|
|
|
RelOptInfo *input_rel,
|
|
|
|
RelOptInfo *final_rel,
|
|
|
|
FinalPathExtraData *extra);
|
2017-04-25 04:50:07 +02:00
|
|
|
static void apply_server_options(PgFdwRelationInfo *fpinfo);
|
|
|
|
static void apply_table_options(PgFdwRelationInfo *fpinfo);
|
|
|
|
static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
|
|
|
|
const PgFdwRelationInfo *fpinfo_o,
|
|
|
|
const PgFdwRelationInfo *fpinfo_i);
|
2021-01-20 23:05:46 +01:00
|
|
|
static int get_batch_size_option(Relation rel);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Foreign-data wrapper handler function: return a struct with pointers
|
|
|
|
* to my callback routines.
|
|
|
|
*/
|
|
|
|
Datum
|
|
|
|
postgres_fdw_handler(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
FdwRoutine *routine = makeNode(FdwRoutine);
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/* Functions for scanning foreign tables */
|
2013-02-21 11:26:23 +01:00
|
|
|
routine->GetForeignRelSize = postgresGetForeignRelSize;
|
|
|
|
routine->GetForeignPaths = postgresGetForeignPaths;
|
|
|
|
routine->GetForeignPlan = postgresGetForeignPlan;
|
|
|
|
routine->BeginForeignScan = postgresBeginForeignScan;
|
|
|
|
routine->IterateForeignScan = postgresIterateForeignScan;
|
|
|
|
routine->ReScanForeignScan = postgresReScanForeignScan;
|
|
|
|
routine->EndForeignScan = postgresEndForeignScan;
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/* Functions for updating foreign tables */
|
|
|
|
routine->AddForeignUpdateTargets = postgresAddForeignUpdateTargets;
|
|
|
|
routine->PlanForeignModify = postgresPlanForeignModify;
|
|
|
|
routine->BeginForeignModify = postgresBeginForeignModify;
|
|
|
|
routine->ExecForeignInsert = postgresExecForeignInsert;
|
2021-01-20 23:05:46 +01:00
|
|
|
routine->ExecForeignBatchInsert = postgresExecForeignBatchInsert;
|
|
|
|
routine->GetForeignModifyBatchSize = postgresGetForeignModifyBatchSize;
|
2013-03-10 19:14:53 +01:00
|
|
|
routine->ExecForeignUpdate = postgresExecForeignUpdate;
|
|
|
|
routine->ExecForeignDelete = postgresExecForeignDelete;
|
|
|
|
routine->EndForeignModify = postgresEndForeignModify;
|
2018-04-07 01:16:11 +02:00
|
|
|
routine->BeginForeignInsert = postgresBeginForeignInsert;
|
|
|
|
routine->EndForeignInsert = postgresEndForeignInsert;
|
2013-06-12 23:52:54 +02:00
|
|
|
routine->IsForeignRelUpdatable = postgresIsForeignRelUpdatable;
|
2016-03-18 18:48:58 +01:00
|
|
|
routine->PlanDirectModify = postgresPlanDirectModify;
|
|
|
|
routine->BeginDirectModify = postgresBeginDirectModify;
|
|
|
|
routine->IterateDirectModify = postgresIterateDirectModify;
|
|
|
|
routine->EndDirectModify = postgresEndDirectModify;
|
2013-03-10 19:14:53 +01:00
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/* Function for EvalPlanQual rechecks */
|
|
|
|
routine->RecheckForeignScan = postgresRecheckForeignScan;
|
2013-03-10 19:14:53 +01:00
|
|
|
/* Support functions for EXPLAIN */
|
|
|
|
routine->ExplainForeignScan = postgresExplainForeignScan;
|
|
|
|
routine->ExplainForeignModify = postgresExplainForeignModify;
|
2016-03-18 18:48:58 +01:00
|
|
|
routine->ExplainDirectModify = postgresExplainDirectModify;
|
2013-03-10 19:14:53 +01:00
|
|
|
|
Allow TRUNCATE command to truncate foreign tables.
This commit introduces new foreign data wrapper API for TRUNCATE.
It extends TRUNCATE command so that it accepts foreign tables as
the targets to truncate and invokes that API. Also it extends postgres_fdw
so that it can issue TRUNCATE command to foreign servers, by adding
new routine for that TRUNCATE API.
The information about options specified in TRUNCATE command, e.g.,
ONLY, CACADE, etc is passed to FDW via API. The list of foreign tables to
truncate is also passed to FDW. FDW truncates the foreign data sources
that the passed foreign tables specify, based on those information.
For example, postgres_fdw constructs TRUNCATE command using them
and issues it to the foreign server.
For performance, TRUNCATE command invokes the FDW routine for
TRUNCATE once per foreign server that foreign tables to truncate belong to.
Author: Kazutaka Onishi, Kohei KaiGai, slightly modified by Fujii Masao
Reviewed-by: Bharath Rupireddy, Michael Paquier, Zhihong Yu, Alvaro Herrera, Stephen Frost, Ashutosh Bapat, Amit Langote, Daniel Gustafsson, Ibrar Ahmed, Fujii Masao
Discussion: https://postgr.es/m/CAOP8fzb_gkReLput7OvOK+8NHgw-RKqNv59vem7=524krQTcWA@mail.gmail.com
Discussion: https://postgr.es/m/CAJuF6cMWDDqU-vn_knZgma+2GMaout68YUgn1uyDnexRhqqM5Q@mail.gmail.com
2021-04-08 13:56:08 +02:00
|
|
|
/* Support function for TRUNCATE */
|
|
|
|
routine->ExecForeignTruncate = postgresExecForeignTruncate;
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/* Support functions for ANALYZE */
|
2013-02-21 11:26:23 +01:00
|
|
|
routine->AnalyzeForeignTable = postgresAnalyzeForeignTable;
|
|
|
|
|
2014-07-10 21:01:31 +02:00
|
|
|
/* Support functions for IMPORT FOREIGN SCHEMA */
|
|
|
|
routine->ImportForeignSchema = postgresImportForeignSchema;
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/* Support functions for join push-down */
|
|
|
|
routine->GetForeignJoinPaths = postgresGetForeignJoinPaths;
|
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
/* Support functions for upper relation push-down */
|
|
|
|
routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/* Support functions for asynchronous execution */
|
|
|
|
routine->IsForeignPathAsyncCapable = postgresIsForeignPathAsyncCapable;
|
|
|
|
routine->ForeignAsyncRequest = postgresForeignAsyncRequest;
|
|
|
|
routine->ForeignAsyncConfigureWait = postgresForeignAsyncConfigureWait;
|
|
|
|
routine->ForeignAsyncNotify = postgresForeignAsyncNotify;
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
PG_RETURN_POINTER(routine);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresGetForeignRelSize
|
|
|
|
* Estimate # of rows and width of the result of the scan
|
|
|
|
*
|
2013-03-22 00:43:59 +01:00
|
|
|
* We should consider the effect of all baserestrictinfo clauses here, but
|
|
|
|
* not any join clauses.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresGetForeignRelSize(PlannerInfo *root,
|
|
|
|
RelOptInfo *baserel,
|
|
|
|
Oid foreigntableid)
|
|
|
|
{
|
|
|
|
PgFdwRelationInfo *fpinfo;
|
2013-03-10 19:14:53 +01:00
|
|
|
ListCell *lc;
|
2016-02-09 20:00:50 +01:00
|
|
|
RangeTblEntry *rte = planner_rt_fetch(baserel->relid, root);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We use PgFdwRelationInfo to pass various information to subsequent
|
|
|
|
* functions.
|
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
fpinfo = (PgFdwRelationInfo *) palloc0(sizeof(PgFdwRelationInfo));
|
2013-03-22 00:43:59 +01:00
|
|
|
baserel->fdw_private = (void *) fpinfo;
|
|
|
|
|
2016-08-24 22:29:10 +02:00
|
|
|
/* Base foreign tables need to be pushed down always. */
|
2016-02-09 20:00:50 +01:00
|
|
|
fpinfo->pushdown_safe = true;
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/* Look up foreign-table catalog info. */
|
|
|
|
fpinfo->table = GetForeignTable(foreigntableid);
|
|
|
|
fpinfo->server = GetForeignServer(fpinfo->table->serverid);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
2021-03-30 06:00:00 +02:00
|
|
|
* Extract user-settable option values. Note that per-table settings of
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
* use_remote_estimate, fetch_size and async_capable override per-server
|
|
|
|
* settings of them, respectively.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
fpinfo->use_remote_estimate = false;
|
|
|
|
fpinfo->fdw_startup_cost = DEFAULT_FDW_STARTUP_COST;
|
|
|
|
fpinfo->fdw_tuple_cost = DEFAULT_FDW_TUPLE_COST;
|
Allow postgres_fdw to ship extension funcs/operators for remote execution.
The user can whitelist specified extension(s) in the foreign server's
options, whereupon we will treat immutable functions and operators of those
extensions as candidates to be sent for remote execution.
Whitelisting an extension in this way basically promises that the extension
exists on the remote server and behaves compatibly with the local instance.
We have no way to prove that formally, so we have to rely on the user to
get it right. But this seems like something that people can usually get
right in practice.
We might in future allow functions and operators to be whitelisted
individually, but extension granularity is a very convenient special case,
so it got done first.
The patch as-committed lacks any regression tests, which is unfortunate,
but introducing dependencies on other extensions for testing purposes
would break "make installcheck" scenarios, which is worse. I have some
ideas about klugy ways around that, but it seems like material for a
separate patch. For the moment, leave the problem open.
Paul Ramsey, hacked up a bit more by me
2015-11-04 00:42:02 +01:00
|
|
|
fpinfo->shippable_extensions = NIL;
|
2016-02-03 15:01:59 +01:00
|
|
|
fpinfo->fetch_size = 100;
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
fpinfo->async_capable = false;
|
2013-03-22 00:43:59 +01:00
|
|
|
|
2017-04-25 04:50:07 +02:00
|
|
|
apply_server_options(fpinfo);
|
|
|
|
apply_table_options(fpinfo);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* If the table or the server is configured to use remote estimates,
|
|
|
|
* identify which user to do remote access as during planning. This
|
|
|
|
* should match what ExecCheckRTEPerms() does. If we fail due to lack of
|
|
|
|
* permissions, the query would have failed at runtime anyway.
|
|
|
|
*/
|
|
|
|
if (fpinfo->use_remote_estimate)
|
|
|
|
{
|
|
|
|
Oid userid = rte->checkAsUser ? rte->checkAsUser : GetUserId();
|
|
|
|
|
|
|
|
fpinfo->user = GetUserMapping(userid, fpinfo->server->serverid);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
fpinfo->user = NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Identify which baserestrictinfo clauses can be sent to the remote
|
|
|
|
* server and which can't.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2014-03-07 22:35:58 +01:00
|
|
|
classifyConditions(root, baserel, baserel->baserestrictinfo,
|
2013-03-22 00:43:59 +01:00
|
|
|
&fpinfo->remote_conds, &fpinfo->local_conds);
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Identify which attributes will need to be retrieved from the remote
|
2013-03-22 00:43:59 +01:00
|
|
|
* server. These include all attrs needed for joins or final output, plus
|
|
|
|
* all attrs used in the local_conds. (Note: if we end up using a
|
|
|
|
* parameterized scan, it's possible that some of the join clauses will be
|
|
|
|
* sent to the remote and thus we wouldn't really need to retrieve the
|
|
|
|
* columns used in them. Doesn't seem worth detecting that case though.)
|
2013-03-10 19:14:53 +01:00
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
fpinfo->attrs_used = NULL;
|
2016-03-14 21:59:59 +01:00
|
|
|
pull_varattnos((Node *) baserel->reltarget->exprs, baserel->relid,
|
2013-03-22 00:43:59 +01:00
|
|
|
&fpinfo->attrs_used);
|
|
|
|
foreach(lc, fpinfo->local_conds)
|
2013-03-10 19:14:53 +01:00
|
|
|
{
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc);
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
pull_varattnos((Node *) rinfo->clause, baserel->relid,
|
2013-03-22 00:43:59 +01:00
|
|
|
&fpinfo->attrs_used);
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* Compute the selectivity and cost of the local_conds, so we don't have
|
|
|
|
* to do it over again for each path. The best we can do for these
|
|
|
|
* conditions is to estimate selectivity on the basis of local statistics.
|
2013-03-10 19:14:53 +01:00
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
fpinfo->local_conds_sel = clauselist_selectivity(root,
|
|
|
|
fpinfo->local_conds,
|
|
|
|
baserel->relid,
|
|
|
|
JOIN_INNER,
|
|
|
|
NULL);
|
|
|
|
|
|
|
|
cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2016-03-09 16:51:49 +01:00
|
|
|
/*
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
* Set # of retrieved rows and cached relation costs to some negative
|
|
|
|
* value, so that we can detect when they are set to some sensible values,
|
|
|
|
* during one (usually the first) of the calls to estimate_path_cost_size.
|
2016-03-09 16:51:49 +01:00
|
|
|
*/
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
fpinfo->retrieved_rows = -1;
|
2016-03-09 16:51:49 +01:00
|
|
|
fpinfo->rel_startup_cost = -1;
|
|
|
|
fpinfo->rel_total_cost = -1;
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
2013-02-23 18:20:48 +01:00
|
|
|
* If the table or the server is configured to use remote estimates,
|
2013-03-22 00:43:59 +01:00
|
|
|
* connect to the foreign server and execute EXPLAIN to estimate the
|
|
|
|
* number of rows selected by the restriction clauses, as well as the
|
|
|
|
* average row width. Otherwise, estimate using whatever statistics we
|
|
|
|
* have locally, in a way similar to ordinary tables.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
if (fpinfo->use_remote_estimate)
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* Get cost/size estimates with help of remote server. Save the
|
|
|
|
* values in fpinfo so we don't need to do it again to generate the
|
|
|
|
* basic foreign path.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2019-04-02 12:20:30 +02:00
|
|
|
estimate_path_cost_size(root, baserel, NIL, NIL, NULL,
|
2013-03-22 00:43:59 +01:00
|
|
|
&fpinfo->rows, &fpinfo->width,
|
|
|
|
&fpinfo->startup_cost, &fpinfo->total_cost);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/* Report estimated baserel size to planner. */
|
|
|
|
baserel->rows = fpinfo->rows;
|
2016-03-14 21:59:59 +01:00
|
|
|
baserel->reltarget->width = fpinfo->width;
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
Redefine pg_class.reltuples to be -1 before the first VACUUM or ANALYZE.
Historically, we've considered the state with relpages and reltuples
both zero as indicating that we do not know the table's tuple density.
This is problematic because it's impossible to distinguish "never yet
vacuumed" from "vacuumed and seen to be empty". In particular, a user
cannot use VACUUM or ANALYZE to override the planner's normal heuristic
that an empty table should not be believed to be empty because it is
probably about to get populated. That heuristic is a good safety
measure, so I don't care to abandon it, but there should be a way to
override it if the table is indeed intended to stay empty.
Hence, represent the initial state of ignorance by setting reltuples
to -1 (relpages is still set to zero), and apply the minimum-ten-pages
heuristic only when reltuples is still -1. If the table is empty,
VACUUM or ANALYZE (but not CREATE INDEX) will override that to
reltuples = relpages = 0, and then we'll plan on that basis.
This requires a bunch of fiddly little changes, but we can get rid of
some ugly kluges that were formerly needed to maintain the old definition.
One notable point is that FDWs' GetForeignRelSize methods will see
baserel->tuples = -1 when no ANALYZE has been done on the foreign table.
That seems like a net improvement, since those methods were formerly
also in the dark about what baserel->tuples = 0 really meant. Still,
it is an API change.
I bumped catversion because code predating this change would get confused
by seeing reltuples = -1.
Discussion: https://postgr.es/m/F02298E0-6EF4-49A1-BCB6-C484794D9ACC@thebuild.com
2020-08-30 18:21:51 +02:00
|
|
|
* If the foreign table has never been ANALYZEd, it will have
|
|
|
|
* reltuples < 0, meaning "unknown". We can't do much if we're not
|
2013-02-21 11:26:23 +01:00
|
|
|
* allowed to consult the remote server, but we can use a hack similar
|
|
|
|
* to plancat.c's treatment of empty relations: use a minimum size
|
|
|
|
* estimate of 10 pages, and divide by the column-datatype-based width
|
|
|
|
* estimate to get the corresponding number of tuples.
|
|
|
|
*/
|
Redefine pg_class.reltuples to be -1 before the first VACUUM or ANALYZE.
Historically, we've considered the state with relpages and reltuples
both zero as indicating that we do not know the table's tuple density.
This is problematic because it's impossible to distinguish "never yet
vacuumed" from "vacuumed and seen to be empty". In particular, a user
cannot use VACUUM or ANALYZE to override the planner's normal heuristic
that an empty table should not be believed to be empty because it is
probably about to get populated. That heuristic is a good safety
measure, so I don't care to abandon it, but there should be a way to
override it if the table is indeed intended to stay empty.
Hence, represent the initial state of ignorance by setting reltuples
to -1 (relpages is still set to zero), and apply the minimum-ten-pages
heuristic only when reltuples is still -1. If the table is empty,
VACUUM or ANALYZE (but not CREATE INDEX) will override that to
reltuples = relpages = 0, and then we'll plan on that basis.
This requires a bunch of fiddly little changes, but we can get rid of
some ugly kluges that were formerly needed to maintain the old definition.
One notable point is that FDWs' GetForeignRelSize methods will see
baserel->tuples = -1 when no ANALYZE has been done on the foreign table.
That seems like a net improvement, since those methods were formerly
also in the dark about what baserel->tuples = 0 really meant. Still,
it is an API change.
I bumped catversion because code predating this change would get confused
by seeing reltuples = -1.
Discussion: https://postgr.es/m/F02298E0-6EF4-49A1-BCB6-C484794D9ACC@thebuild.com
2020-08-30 18:21:51 +02:00
|
|
|
if (baserel->tuples < 0)
|
2013-02-22 16:56:06 +01:00
|
|
|
{
|
|
|
|
baserel->pages = 10;
|
2013-02-21 11:26:23 +01:00
|
|
|
baserel->tuples =
|
2016-03-14 21:59:59 +01:00
|
|
|
(10 * BLCKSZ) / (baserel->reltarget->width +
|
Add an explicit representation of the output targetlist to Paths.
Up to now, there's been an assumption that all Paths for a given relation
compute the same output column set (targetlist). However, there are good
reasons to remove that assumption. For example, an indexscan on an
expression index might be able to return the value of an expensive function
"for free". While we have the ability to generate such a plan today in
simple cases, we don't have a way to model that it's cheaper than a plan
that computes the function from scratch, nor a way to create such a plan
in join cases (where the function computation would normally happen at
the topmost join node). Also, we need this so that we can have Paths
representing post-scan/join steps, where the targetlist may well change
from one step to the next. Therefore, invent a "struct PathTarget"
representing the columns we expect a plan step to emit. It's convenient
to include the output tuple width and tlist evaluation cost in this struct,
and there will likely be additional fields in future.
While Path nodes that actually do have custom outputs will need their own
PathTargets, it will still be true that most Paths for a given relation
will compute the same tlist. To reduce the overhead added by this patch,
keep a "default PathTarget" in RelOptInfo, and allow Paths that compute
that column set to just point to their parent RelOptInfo's reltarget.
(In the patch as committed, actually every Path is like that, since we
do not yet have any cases of custom PathTargets.)
I took this opportunity to provide some more-honest costing of
PlaceHolderVar evaluation. Up to now, the assumption that "scan/join
reltargetlists have cost zero" was applied not only to Vars, where it's
reasonable, but also PlaceHolderVars where it isn't. Now, we add the eval
cost of a PlaceHolderVar's expression to the first plan level where it can
be computed, by including it in the PathTarget cost field and adding that
to the cost estimates for Paths. This isn't perfect yet but it's much
better than before, and there is a way forward to improve it more. This
costing change affects the join order chosen for a couple of the regression
tests, changing expected row ordering.
2016-02-19 02:01:49 +01:00
|
|
|
MAXALIGN(SizeofHeapTupleHeader));
|
2013-02-22 16:56:06 +01:00
|
|
|
}
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/* Estimate baserel size as best we can with local statistics. */
|
2013-02-21 11:26:23 +01:00
|
|
|
set_baserel_size_estimates(root, baserel);
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/* Fill in basically-bogus cost estimates for use later. */
|
2019-04-02 12:20:30 +02:00
|
|
|
estimate_path_cost_size(root, baserel, NIL, NIL, NULL,
|
2013-03-22 00:43:59 +01:00
|
|
|
&fpinfo->rows, &fpinfo->width,
|
|
|
|
&fpinfo->startup_cost, &fpinfo->total_cost);
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
/*
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
* fpinfo->relation_name gets the numeric rangetable index of the foreign
|
|
|
|
* table RTE. (If this query gets EXPLAIN'd, we'll convert that to a
|
|
|
|
* human-readable string at that time.)
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
fpinfo->relation_name = psprintf("%u", baserel->relid);
|
2017-03-16 18:34:59 +01:00
|
|
|
|
|
|
|
/* No outer and inner relations. */
|
|
|
|
fpinfo->make_outerrel_subquery = false;
|
|
|
|
fpinfo->make_innerrel_subquery = false;
|
|
|
|
fpinfo->lower_subquery_rels = NULL;
|
|
|
|
/* Set the relation index. */
|
|
|
|
fpinfo->relation_index = baserel->relid;
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
|
2015-12-22 19:46:40 +01:00
|
|
|
/*
|
|
|
|
* get_useful_ecs_for_relation
|
|
|
|
* Determine which EquivalenceClasses might be involved in useful
|
|
|
|
* orderings of this relation.
|
|
|
|
*
|
|
|
|
* This function is in some respects a mirror image of the core function
|
|
|
|
* pathkeys_useful_for_merging: for a regular table, we know what indexes
|
|
|
|
* we have and want to test whether any of them are useful. For a foreign
|
|
|
|
* table, we don't know what indexes are present on the remote side but
|
|
|
|
* want to speculate about which ones we'd like to use if they existed.
|
2016-05-16 17:28:28 +02:00
|
|
|
*
|
|
|
|
* This function returns a list of potentially-useful equivalence classes,
|
|
|
|
* but it does not guarantee that an EquivalenceMember exists which contains
|
|
|
|
* Vars only from the given relation. For example, given ft1 JOIN t1 ON
|
|
|
|
* ft1.x + t1.x = 0, this function will say that the equivalence class
|
|
|
|
* containing ft1.x + t1.x is potentially useful. Supposing ft1 is remote and
|
|
|
|
* t1 is local (or on a different server), it will turn out that no useful
|
|
|
|
* ORDER BY clause can be generated. It's not our job to figure that out
|
|
|
|
* here; we're only interested in identifying relevant ECs.
|
2015-12-22 19:46:40 +01:00
|
|
|
*/
|
|
|
|
static List *
|
|
|
|
get_useful_ecs_for_relation(PlannerInfo *root, RelOptInfo *rel)
|
|
|
|
{
|
|
|
|
List *useful_eclass_list = NIL;
|
|
|
|
ListCell *lc;
|
|
|
|
Relids relids;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* First, consider whether any active EC is potentially useful for a merge
|
|
|
|
* join against this relation.
|
|
|
|
*/
|
|
|
|
if (rel->has_eclass_joins)
|
|
|
|
{
|
|
|
|
foreach(lc, root->eq_classes)
|
|
|
|
{
|
|
|
|
EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc);
|
|
|
|
|
|
|
|
if (eclass_useful_for_merging(root, cur_ec, rel))
|
|
|
|
useful_eclass_list = lappend(useful_eclass_list, cur_ec);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Next, consider whether there are any non-EC derivable join clauses that
|
|
|
|
* are merge-joinable. If the joininfo list is empty, we can exit
|
|
|
|
* quickly.
|
|
|
|
*/
|
|
|
|
if (rel->joininfo == NIL)
|
|
|
|
return useful_eclass_list;
|
|
|
|
|
|
|
|
/* If this is a child rel, we must use the topmost parent rel to search. */
|
Abstract logic to allow for multiple kinds of child rels.
Currently, the only type of child relation is an "other member rel",
which is the child of a baserel, but in the future joins and even
upper relations may have child rels. To facilitate that, introduce
macros that test to test for particular RelOptKind values, and use
them in various places where they help to clarify the sense of a test.
(For example, a test may allow RELOPT_OTHER_MEMBER_REL either because
it intends to allow child rels, or because it intends to allow simple
rels.)
Also, remove find_childrel_top_parent, which will not work for a
child rel that is not a baserel. Instead, add a new RelOptInfo
member top_parent_relids to track the same kind of information in a
more generic manner.
Ashutosh Bapat, slightly tweaked by me. Review and testing of the
patch set from which this was taken by Rajkumar Raghuwanshi and Rafia
Sabih.
Discussion: http://postgr.es/m/CA+TgmoagTnF2yqR3PT2rv=om=wJiZ4-A+ATwdnriTGku1CLYxA@mail.gmail.com
2017-04-04 04:41:31 +02:00
|
|
|
if (IS_OTHER_REL(rel))
|
|
|
|
{
|
|
|
|
Assert(!bms_is_empty(rel->top_parent_relids));
|
|
|
|
relids = rel->top_parent_relids;
|
|
|
|
}
|
2015-12-22 19:46:40 +01:00
|
|
|
else
|
|
|
|
relids = rel->relids;
|
|
|
|
|
|
|
|
/* Check each join clause in turn. */
|
|
|
|
foreach(lc, rel->joininfo)
|
|
|
|
{
|
|
|
|
RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc);
|
|
|
|
|
|
|
|
/* Consider only mergejoinable clauses */
|
|
|
|
if (restrictinfo->mergeopfamilies == NIL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Make sure we've got canonical ECs. */
|
|
|
|
update_mergeclause_eclasses(root, restrictinfo);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* restrictinfo->mergeopfamilies != NIL is sufficient to guarantee
|
|
|
|
* that left_ec and right_ec will be initialized, per comments in
|
2016-05-16 17:28:28 +02:00
|
|
|
* distribute_qual_to_rels.
|
|
|
|
*
|
|
|
|
* We want to identify which side of this merge-joinable clause
|
|
|
|
* contains columns from the relation produced by this RelOptInfo. We
|
|
|
|
* test for overlap, not containment, because there could be extra
|
|
|
|
* relations on either side. For example, suppose we've got something
|
|
|
|
* like ((A JOIN B ON A.x = B.x) JOIN C ON A.y = C.y) LEFT JOIN D ON
|
|
|
|
* A.y = D.y. The input rel might be the joinrel between A and B, and
|
|
|
|
* we'll consider the join clause A.y = D.y. relids contains a
|
|
|
|
* relation not involved in the join class (B) and the equivalence
|
|
|
|
* class for the left-hand side of the clause contains a relation not
|
|
|
|
* involved in the input rel (C). Despite the fact that we have only
|
|
|
|
* overlap and not containment in either direction, A.y is potentially
|
|
|
|
* useful as a sort column.
|
|
|
|
*
|
|
|
|
* Note that it's even possible that relids overlaps neither side of
|
|
|
|
* the join clause. For example, consider A LEFT JOIN B ON A.x = B.x
|
|
|
|
* AND A.x = 1. The clause A.x = 1 will appear in B's joininfo list,
|
|
|
|
* but overlaps neither side of B. In that case, we just skip this
|
|
|
|
* join clause, since it doesn't suggest a useful sort order for this
|
|
|
|
* relation.
|
2015-12-22 19:46:40 +01:00
|
|
|
*/
|
2016-05-16 17:28:28 +02:00
|
|
|
if (bms_overlap(relids, restrictinfo->right_ec->ec_relids))
|
2015-12-22 19:46:40 +01:00
|
|
|
useful_eclass_list = list_append_unique_ptr(useful_eclass_list,
|
|
|
|
restrictinfo->right_ec);
|
2016-05-16 17:28:28 +02:00
|
|
|
else if (bms_overlap(relids, restrictinfo->left_ec->ec_relids))
|
2015-12-22 19:46:40 +01:00
|
|
|
useful_eclass_list = list_append_unique_ptr(useful_eclass_list,
|
|
|
|
restrictinfo->left_ec);
|
|
|
|
}
|
|
|
|
|
|
|
|
return useful_eclass_list;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* get_useful_pathkeys_for_relation
|
|
|
|
* Determine which orderings of a relation might be useful.
|
|
|
|
*
|
|
|
|
* Getting data in sorted order can be useful either because the requested
|
|
|
|
* order matches the final output ordering for the overall query we're
|
|
|
|
* planning, or because it enables an efficient merge join. Here, we try
|
|
|
|
* to figure out which pathkeys to consider.
|
|
|
|
*/
|
|
|
|
static List *
|
|
|
|
get_useful_pathkeys_for_relation(PlannerInfo *root, RelOptInfo *rel)
|
|
|
|
{
|
|
|
|
List *useful_pathkeys_list = NIL;
|
|
|
|
List *useful_eclass_list;
|
|
|
|
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private;
|
|
|
|
EquivalenceClass *query_ec = NULL;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Pushing the query_pathkeys to the remote server is always worth
|
|
|
|
* considering, because it might let us avoid a local sort.
|
|
|
|
*/
|
2019-04-02 12:20:30 +02:00
|
|
|
fpinfo->qp_is_pushdown_safe = false;
|
2015-12-22 19:46:40 +01:00
|
|
|
if (root->query_pathkeys)
|
|
|
|
{
|
|
|
|
bool query_pathkeys_ok = true;
|
|
|
|
|
|
|
|
foreach(lc, root->query_pathkeys)
|
|
|
|
{
|
|
|
|
PathKey *pathkey = (PathKey *) lfirst(lc);
|
|
|
|
EquivalenceClass *pathkey_ec = pathkey->pk_eclass;
|
|
|
|
Expr *em_expr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The planner and executor don't have any clever strategy for
|
|
|
|
* taking data sorted by a prefix of the query's pathkeys and
|
|
|
|
* getting it to be sorted by all of those pathkeys. We'll just
|
|
|
|
* end up resorting the entire data set. So, unless we can push
|
|
|
|
* down all of the query pathkeys, forget it.
|
|
|
|
*
|
|
|
|
* is_foreign_expr would detect volatile expressions as well, but
|
|
|
|
* checking ec_has_volatile here saves some cycles.
|
|
|
|
*/
|
|
|
|
if (pathkey_ec->ec_has_volatile ||
|
|
|
|
!(em_expr = find_em_expr_for_rel(pathkey_ec, rel)) ||
|
|
|
|
!is_foreign_expr(root, rel, em_expr))
|
|
|
|
{
|
|
|
|
query_pathkeys_ok = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (query_pathkeys_ok)
|
2019-04-02 12:20:30 +02:00
|
|
|
{
|
2015-12-22 19:46:40 +01:00
|
|
|
useful_pathkeys_list = list_make1(list_copy(root->query_pathkeys));
|
2019-04-02 12:20:30 +02:00
|
|
|
fpinfo->qp_is_pushdown_safe = true;
|
|
|
|
}
|
2015-12-22 19:46:40 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Even if we're not using remote estimates, having the remote side do the
|
|
|
|
* sort generally won't be any worse than doing it locally, and it might
|
|
|
|
* be much better if the remote side can generate data in the right order
|
|
|
|
* without needing a sort at all. However, what we're going to do next is
|
|
|
|
* try to generate pathkeys that seem promising for possible merge joins,
|
|
|
|
* and that's more speculative. A wrong choice might hurt quite a bit, so
|
|
|
|
* bail out if we can't use remote estimates.
|
|
|
|
*/
|
|
|
|
if (!fpinfo->use_remote_estimate)
|
|
|
|
return useful_pathkeys_list;
|
|
|
|
|
|
|
|
/* Get the list of interesting EquivalenceClasses. */
|
|
|
|
useful_eclass_list = get_useful_ecs_for_relation(root, rel);
|
|
|
|
|
|
|
|
/* Extract unique EC for query, if any, so we don't consider it again. */
|
|
|
|
if (list_length(root->query_pathkeys) == 1)
|
|
|
|
{
|
|
|
|
PathKey *query_pathkey = linitial(root->query_pathkeys);
|
|
|
|
|
|
|
|
query_ec = query_pathkey->pk_eclass;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* As a heuristic, the only pathkeys we consider here are those of length
|
|
|
|
* one. It's surely possible to consider more, but since each one we
|
|
|
|
* choose to consider will generate a round-trip to the remote side, we
|
|
|
|
* need to be a bit cautious here. It would sure be nice to have a local
|
|
|
|
* cache of information about remote index definitions...
|
|
|
|
*/
|
|
|
|
foreach(lc, useful_eclass_list)
|
|
|
|
{
|
|
|
|
EquivalenceClass *cur_ec = lfirst(lc);
|
|
|
|
Expr *em_expr;
|
|
|
|
PathKey *pathkey;
|
|
|
|
|
|
|
|
/* If redundant with what we did above, skip it. */
|
|
|
|
if (cur_ec == query_ec)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* If no pushable expression for this rel, skip it. */
|
|
|
|
em_expr = find_em_expr_for_rel(cur_ec, rel);
|
|
|
|
if (em_expr == NULL || !is_foreign_expr(root, rel, em_expr))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Looks like we can generate a pathkey, so let's do it. */
|
|
|
|
pathkey = make_canonical_pathkey(root, cur_ec,
|
|
|
|
linitial_oid(cur_ec->ec_opfamilies),
|
|
|
|
BTLessStrategyNumber,
|
|
|
|
false);
|
|
|
|
useful_pathkeys_list = lappend(useful_pathkeys_list,
|
|
|
|
list_make1(pathkey));
|
|
|
|
}
|
|
|
|
|
|
|
|
return useful_pathkeys_list;
|
|
|
|
}
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* postgresGetForeignPaths
|
|
|
|
* Create possible scan paths for a scan on the foreign table
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresGetForeignPaths(PlannerInfo *root,
|
|
|
|
RelOptInfo *baserel,
|
|
|
|
Oid foreigntableid)
|
|
|
|
{
|
|
|
|
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) baserel->fdw_private;
|
|
|
|
ForeignPath *path;
|
2014-03-07 22:35:58 +01:00
|
|
|
List *ppi_list;
|
2013-03-22 00:43:59 +01:00
|
|
|
ListCell *lc;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* Create simplest ForeignScan path node and add it to baserel. This path
|
|
|
|
* corresponds to SeqScan path of regular tables (though depending on what
|
|
|
|
* baserestrict conditions we were able to send to remote, there might
|
|
|
|
* actually be an indexscan happening there). We already did all the work
|
|
|
|
* to estimate cost and size of this path.
|
Split create_foreignscan_path() into three functions.
Up to now postgres_fdw has been using create_foreignscan_path() to
generate not only base-relation paths, but also paths for foreign joins
and foreign upperrels. This is wrong, because create_foreignscan_path()
calls get_baserel_parampathinfo() which will only do the right thing for
baserels. It accidentally fails to fail for unparameterized paths, which
are the only ones postgres_fdw (thought it) was handling, but we really
need different APIs for the baserel and join cases.
In HEAD, the best thing to do seems to be to split up the baserel,
joinrel, and upperrel cases into three functions so that they can
have different APIs. I haven't actually given create_foreign_join_path
a different API in this commit: we should spend a bit of time thinking
about just what we want to do there, since perhaps FDWs would want to
do something different from the build-up-a-join-pairwise approach that
get_joinrel_parampathinfo expects. In the meantime, since postgres_fdw
isn't prepared to generate parameterized joins anyway, just give it a
defense against trying to plan joins with lateral refs.
In addition (and this is what triggered this whole mess) fix bug #15613
from Srinivasan S A, by teaching file_fdw and postgres_fdw that plain
baserel foreign paths still have outer refs if the relation has
lateral_relids. Add some assertions in relnode.c to catch future
occurrences of the same error --- in particular, to catch other FDWs
doing that, but also as backstop against core-code mistakes like the
one fixed by commit bdd9a99aa.
Bug #15613 also needs to be fixed in the back branches, but the
appropriate fix will look quite a bit different there, since we don't
want to assume that existing FDWs get the word right away.
Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org
2019-02-07 18:59:47 +01:00
|
|
|
*
|
|
|
|
* Although this path uses no join clauses, it could still have required
|
|
|
|
* parameterization due to LATERAL refs in its tlist.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
path = create_foreignscan_path(root, baserel,
|
2016-03-14 22:31:28 +01:00
|
|
|
NULL, /* default pathtarget */
|
2013-03-22 00:43:59 +01:00
|
|
|
fpinfo->rows,
|
|
|
|
fpinfo->startup_cost,
|
|
|
|
fpinfo->total_cost,
|
|
|
|
NIL, /* no pathkeys */
|
Split create_foreignscan_path() into three functions.
Up to now postgres_fdw has been using create_foreignscan_path() to
generate not only base-relation paths, but also paths for foreign joins
and foreign upperrels. This is wrong, because create_foreignscan_path()
calls get_baserel_parampathinfo() which will only do the right thing for
baserels. It accidentally fails to fail for unparameterized paths, which
are the only ones postgres_fdw (thought it) was handling, but we really
need different APIs for the baserel and join cases.
In HEAD, the best thing to do seems to be to split up the baserel,
joinrel, and upperrel cases into three functions so that they can
have different APIs. I haven't actually given create_foreign_join_path
a different API in this commit: we should spend a bit of time thinking
about just what we want to do there, since perhaps FDWs would want to
do something different from the build-up-a-join-pairwise approach that
get_joinrel_parampathinfo expects. In the meantime, since postgres_fdw
isn't prepared to generate parameterized joins anyway, just give it a
defense against trying to plan joins with lateral refs.
In addition (and this is what triggered this whole mess) fix bug #15613
from Srinivasan S A, by teaching file_fdw and postgres_fdw that plain
baserel foreign paths still have outer refs if the relation has
lateral_relids. Add some assertions in relnode.c to catch future
occurrences of the same error --- in particular, to catch other FDWs
doing that, but also as backstop against core-code mistakes like the
one fixed by commit bdd9a99aa.
Bug #15613 also needs to be fixed in the back branches, but the
appropriate fix will look quite a bit different there, since we don't
want to assume that existing FDWs get the word right away.
Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org
2019-02-07 18:59:47 +01:00
|
|
|
baserel->lateral_relids,
|
Allow foreign and custom joins to handle EvalPlanQual rechecks.
Commit e7cb7ee14555cc9c5773e2c102efd6371f6f2005 provided basic
infrastructure for allowing a foreign data wrapper or custom scan
provider to replace a join of one or more tables with a scan.
However, this infrastructure failed to take into account the need
for possible EvalPlanQual rechecks, and ExecScanFetch would fail
an assertion (or just overwrite memory) if such a check was attempted
for a plan containing a pushed-down join. To fix, adjust the EPQ
machinery to skip some processing steps when scanrelid == 0, making
those the responsibility of scan's recheck method, which also has
the responsibility in this case of correctly populating the relevant
slot.
To allow foreign scans to gain control in the right place to make
use of this new facility, add a new, optional RecheckForeignScan
method. Also, allow a foreign scan to have a child plan, which can
be used to correctly populate the slot (or perhaps for something
else, but this is the only use currently envisioned).
KaiGai Kohei, reviewed by Robert Haas, Etsuro Fujita, and Kyotaro
Horiguchi.
2015-12-08 18:31:03 +01:00
|
|
|
NULL, /* no extra plan */
|
2013-03-22 00:43:59 +01:00
|
|
|
NIL); /* no fdw_private list */
|
|
|
|
add_path(baserel, (Path *) path);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2016-03-09 16:51:49 +01:00
|
|
|
/* Add paths with pathkeys */
|
|
|
|
add_paths_with_pathkeys_for_rel(root, baserel, NULL);
|
2015-11-03 18:46:06 +01:00
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* If we're not using remote estimates, stop here. We have no way to
|
|
|
|
* estimate whether any join clauses would be worth sending across, so
|
|
|
|
* don't bother building parameterized paths.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
if (!fpinfo->use_remote_estimate)
|
|
|
|
return;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
2014-03-07 22:35:58 +01:00
|
|
|
* Thumb through all join clauses for the rel to identify which outer
|
|
|
|
* relations could supply one or more safe-to-send-to-remote join clauses.
|
|
|
|
* We'll build a parameterized path for each such outer relation.
|
|
|
|
*
|
|
|
|
* It's convenient to manage this by representing each candidate outer
|
|
|
|
* relation by the ParamPathInfo node for it. We can then use the
|
|
|
|
* ppi_clauses list in the ParamPathInfo node directly as a list of the
|
|
|
|
* interesting join clauses for that rel. This takes care of the
|
|
|
|
* possibility that there are multiple safe join clauses for such a rel,
|
|
|
|
* and also ensures that we account for unsafe join clauses that we'll
|
|
|
|
* still have to enforce locally (since the parameterized-path machinery
|
|
|
|
* insists that we handle all movable clauses).
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2014-03-07 22:35:58 +01:00
|
|
|
ppi_list = NIL;
|
2013-03-22 00:43:59 +01:00
|
|
|
foreach(lc, baserel->joininfo)
|
|
|
|
{
|
|
|
|
RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc);
|
2014-03-07 22:35:58 +01:00
|
|
|
Relids required_outer;
|
|
|
|
ParamPathInfo *param_info;
|
2013-03-22 00:43:59 +01:00
|
|
|
|
|
|
|
/* Check if clause can be moved to this rel */
|
2013-08-18 02:22:37 +02:00
|
|
|
if (!join_clause_is_movable_to(rinfo, baserel))
|
2013-03-22 00:43:59 +01:00
|
|
|
continue;
|
|
|
|
|
|
|
|
/* See if it is safe to send to remote */
|
|
|
|
if (!is_foreign_expr(root, baserel, rinfo->clause))
|
|
|
|
continue;
|
|
|
|
|
2014-03-07 22:35:58 +01:00
|
|
|
/* Calculate required outer rels for the resulting path */
|
2013-03-22 00:43:59 +01:00
|
|
|
required_outer = bms_union(rinfo->clause_relids,
|
|
|
|
baserel->lateral_relids);
|
|
|
|
/* We do not want the foreign rel itself listed in required_outer */
|
|
|
|
required_outer = bms_del_member(required_outer, baserel->relid);
|
2014-03-07 22:35:58 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* required_outer probably can't be empty here, but if it were, we
|
|
|
|
* couldn't make a parameterized path.
|
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
if (bms_is_empty(required_outer))
|
2014-03-07 22:35:58 +01:00
|
|
|
continue;
|
2013-03-22 00:43:59 +01:00
|
|
|
|
2014-03-07 22:35:58 +01:00
|
|
|
/* Get the ParamPathInfo */
|
|
|
|
param_info = get_baserel_parampathinfo(root, baserel,
|
|
|
|
required_outer);
|
|
|
|
Assert(param_info != NULL);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Add it to list unless we already have it. Testing pointer equality
|
|
|
|
* is OK since get_baserel_parampathinfo won't make duplicates.
|
|
|
|
*/
|
|
|
|
ppi_list = list_append_unique_ptr(ppi_list, param_info);
|
2013-03-22 00:43:59 +01:00
|
|
|
}
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* The above scan examined only "generic" join clauses, not those that
|
|
|
|
* were absorbed into EquivalenceClauses. See if we can make anything out
|
|
|
|
* of EquivalenceClauses.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
if (baserel->has_eclass_joins)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We repeatedly scan the eclass list looking for column references
|
|
|
|
* (or expressions) belonging to the foreign rel. Each time we find
|
|
|
|
* one, we generate a list of equivalence joinclauses for it, and then
|
2014-03-07 22:35:58 +01:00
|
|
|
* see if any are safe to send to the remote. Repeat till there are
|
|
|
|
* no more candidate EC members.
|
2013-03-22 00:43:59 +01:00
|
|
|
*/
|
|
|
|
ec_member_foreign_arg arg;
|
|
|
|
|
|
|
|
arg.already_used = NIL;
|
|
|
|
for (;;)
|
|
|
|
{
|
|
|
|
List *clauses;
|
|
|
|
|
|
|
|
/* Make clauses, skipping any that join to lateral_referencers */
|
|
|
|
arg.current = NULL;
|
|
|
|
clauses = generate_implied_equalities_for_column(root,
|
|
|
|
baserel,
|
|
|
|
ec_member_matches_foreign,
|
|
|
|
(void *) &arg,
|
2013-08-18 02:22:37 +02:00
|
|
|
baserel->lateral_referencers);
|
2013-03-22 00:43:59 +01:00
|
|
|
|
|
|
|
/* Done if there are no more expressions in the foreign rel */
|
|
|
|
if (arg.current == NULL)
|
|
|
|
{
|
|
|
|
Assert(clauses == NIL);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Scan the extracted join clauses */
|
|
|
|
foreach(lc, clauses)
|
|
|
|
{
|
|
|
|
RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc);
|
2014-03-07 22:35:58 +01:00
|
|
|
Relids required_outer;
|
|
|
|
ParamPathInfo *param_info;
|
2013-03-22 00:43:59 +01:00
|
|
|
|
|
|
|
/* Check if clause can be moved to this rel */
|
2013-08-18 02:22:37 +02:00
|
|
|
if (!join_clause_is_movable_to(rinfo, baserel))
|
2013-03-22 00:43:59 +01:00
|
|
|
continue;
|
|
|
|
|
|
|
|
/* See if it is safe to send to remote */
|
|
|
|
if (!is_foreign_expr(root, baserel, rinfo->clause))
|
|
|
|
continue;
|
|
|
|
|
2014-03-07 22:35:58 +01:00
|
|
|
/* Calculate required outer rels for the resulting path */
|
2013-03-22 00:43:59 +01:00
|
|
|
required_outer = bms_union(rinfo->clause_relids,
|
|
|
|
baserel->lateral_relids);
|
|
|
|
required_outer = bms_del_member(required_outer, baserel->relid);
|
|
|
|
if (bms_is_empty(required_outer))
|
2014-03-07 22:35:58 +01:00
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Get the ParamPathInfo */
|
|
|
|
param_info = get_baserel_parampathinfo(root, baserel,
|
|
|
|
required_outer);
|
|
|
|
Assert(param_info != NULL);
|
|
|
|
|
|
|
|
/* Add it to list unless we already have it */
|
|
|
|
ppi_list = list_append_unique_ptr(ppi_list, param_info);
|
2013-03-22 00:43:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Try again, now ignoring the expression we found this time */
|
|
|
|
arg.already_used = lappend(arg.already_used, arg.current);
|
|
|
|
}
|
|
|
|
}
|
2014-03-07 22:35:58 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Now build a path for each useful outer relation.
|
|
|
|
*/
|
|
|
|
foreach(lc, ppi_list)
|
|
|
|
{
|
|
|
|
ParamPathInfo *param_info = (ParamPathInfo *) lfirst(lc);
|
|
|
|
double rows;
|
|
|
|
int width;
|
|
|
|
Cost startup_cost;
|
|
|
|
Cost total_cost;
|
|
|
|
|
|
|
|
/* Get a cost estimate from the remote */
|
|
|
|
estimate_path_cost_size(root, baserel,
|
2019-04-02 12:20:30 +02:00
|
|
|
param_info->ppi_clauses, NIL, NULL,
|
2014-03-07 22:35:58 +01:00
|
|
|
&rows, &width,
|
|
|
|
&startup_cost, &total_cost);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ppi_rows currently won't get looked at by anything, but still we
|
|
|
|
* may as well ensure that it matches our idea of the rowcount.
|
|
|
|
*/
|
|
|
|
param_info->ppi_rows = rows;
|
|
|
|
|
|
|
|
/* Make the path */
|
|
|
|
path = create_foreignscan_path(root, baserel,
|
2016-03-14 22:31:28 +01:00
|
|
|
NULL, /* default pathtarget */
|
2014-03-07 22:35:58 +01:00
|
|
|
rows,
|
|
|
|
startup_cost,
|
|
|
|
total_cost,
|
|
|
|
NIL, /* no pathkeys */
|
|
|
|
param_info->ppi_req_outer,
|
Allow foreign and custom joins to handle EvalPlanQual rechecks.
Commit e7cb7ee14555cc9c5773e2c102efd6371f6f2005 provided basic
infrastructure for allowing a foreign data wrapper or custom scan
provider to replace a join of one or more tables with a scan.
However, this infrastructure failed to take into account the need
for possible EvalPlanQual rechecks, and ExecScanFetch would fail
an assertion (or just overwrite memory) if such a check was attempted
for a plan containing a pushed-down join. To fix, adjust the EPQ
machinery to skip some processing steps when scanrelid == 0, making
those the responsibility of scan's recheck method, which also has
the responsibility in this case of correctly populating the relevant
slot.
To allow foreign scans to gain control in the right place to make
use of this new facility, add a new, optional RecheckForeignScan
method. Also, allow a foreign scan to have a child plan, which can
be used to correctly populate the slot (or perhaps for something
else, but this is the only use currently envisioned).
KaiGai Kohei, reviewed by Robert Haas, Etsuro Fujita, and Kyotaro
Horiguchi.
2015-12-08 18:31:03 +01:00
|
|
|
NULL,
|
2014-03-07 22:35:58 +01:00
|
|
|
NIL); /* no fdw_private list */
|
|
|
|
add_path(baserel, (Path *) path);
|
|
|
|
}
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresGetForeignPlan
|
|
|
|
* Create ForeignScan plan node which implements selected best path
|
|
|
|
*/
|
|
|
|
static ForeignScan *
|
|
|
|
postgresGetForeignPlan(PlannerInfo *root,
|
2016-02-09 20:00:50 +01:00
|
|
|
RelOptInfo *foreignrel,
|
2013-02-21 11:26:23 +01:00
|
|
|
Oid foreigntableid,
|
|
|
|
ForeignPath *best_path,
|
|
|
|
List *tlist,
|
Allow foreign and custom joins to handle EvalPlanQual rechecks.
Commit e7cb7ee14555cc9c5773e2c102efd6371f6f2005 provided basic
infrastructure for allowing a foreign data wrapper or custom scan
provider to replace a join of one or more tables with a scan.
However, this infrastructure failed to take into account the need
for possible EvalPlanQual rechecks, and ExecScanFetch would fail
an assertion (or just overwrite memory) if such a check was attempted
for a plan containing a pushed-down join. To fix, adjust the EPQ
machinery to skip some processing steps when scanrelid == 0, making
those the responsibility of scan's recheck method, which also has
the responsibility in this case of correctly populating the relevant
slot.
To allow foreign scans to gain control in the right place to make
use of this new facility, add a new, optional RecheckForeignScan
method. Also, allow a foreign scan to have a child plan, which can
be used to correctly populate the slot (or perhaps for something
else, but this is the only use currently envisioned).
KaiGai Kohei, reviewed by Robert Haas, Etsuro Fujita, and Kyotaro
Horiguchi.
2015-12-08 18:31:03 +01:00
|
|
|
List *scan_clauses,
|
|
|
|
Plan *outer_plan)
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
2016-02-09 20:00:50 +01:00
|
|
|
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) foreignrel->fdw_private;
|
|
|
|
Index scan_relid;
|
2013-03-22 00:43:59 +01:00
|
|
|
List *fdw_private;
|
2015-10-15 19:00:40 +02:00
|
|
|
List *remote_exprs = NIL;
|
2013-02-21 11:26:23 +01:00
|
|
|
List *local_exprs = NIL;
|
2013-03-22 00:43:59 +01:00
|
|
|
List *params_list = NIL;
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
List *fdw_scan_tlist = NIL;
|
|
|
|
List *fdw_recheck_quals = NIL;
|
2013-03-22 05:31:11 +01:00
|
|
|
List *retrieved_attrs;
|
2013-03-22 00:43:59 +01:00
|
|
|
StringInfoData sql;
|
2019-04-02 12:20:30 +02:00
|
|
|
bool has_final_sort = false;
|
2019-04-02 13:30:45 +02:00
|
|
|
bool has_limit = false;
|
2013-02-21 11:26:23 +01:00
|
|
|
ListCell *lc;
|
2016-02-09 20:00:50 +01:00
|
|
|
|
2019-04-02 12:20:30 +02:00
|
|
|
/*
|
|
|
|
* Get FDW private data created by postgresGetForeignUpperPaths(), if any.
|
|
|
|
*/
|
|
|
|
if (best_path->fdw_private)
|
2019-04-02 13:30:45 +02:00
|
|
|
{
|
2019-04-02 12:20:30 +02:00
|
|
|
has_final_sort = intVal(list_nth(best_path->fdw_private,
|
|
|
|
FdwPathPrivateHasFinalSort));
|
2019-04-02 13:30:45 +02:00
|
|
|
has_limit = intVal(list_nth(best_path->fdw_private,
|
|
|
|
FdwPathPrivateHasLimit));
|
|
|
|
}
|
2019-04-02 12:20:30 +02:00
|
|
|
|
Abstract logic to allow for multiple kinds of child rels.
Currently, the only type of child relation is an "other member rel",
which is the child of a baserel, but in the future joins and even
upper relations may have child rels. To facilitate that, introduce
macros that test to test for particular RelOptKind values, and use
them in various places where they help to clarify the sense of a test.
(For example, a test may allow RELOPT_OTHER_MEMBER_REL either because
it intends to allow child rels, or because it intends to allow simple
rels.)
Also, remove find_childrel_top_parent, which will not work for a
child rel that is not a baserel. Instead, add a new RelOptInfo
member top_parent_relids to track the same kind of information in a
more generic manner.
Ashutosh Bapat, slightly tweaked by me. Review and testing of the
patch set from which this was taken by Rajkumar Raghuwanshi and Rafia
Sabih.
Discussion: http://postgr.es/m/CA+TgmoagTnF2yqR3PT2rv=om=wJiZ4-A+ATwdnriTGku1CLYxA@mail.gmail.com
2017-04-04 04:41:31 +02:00
|
|
|
if (IS_SIMPLE_REL(foreignrel))
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* For base relations, set scan_relid as the relid of the relation.
|
|
|
|
*/
|
2016-02-09 20:00:50 +01:00
|
|
|
scan_relid = foreignrel->relid;
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* In a base-relation scan, we must apply the given scan_clauses.
|
|
|
|
*
|
|
|
|
* Separate the scan_clauses into those that can be executed remotely
|
|
|
|
* and those that can't. baserestrictinfo clauses that were
|
|
|
|
* previously determined to be safe or unsafe by classifyConditions
|
|
|
|
* are found in fpinfo->remote_conds and fpinfo->local_conds. Anything
|
|
|
|
* else in the scan_clauses list will be a join clause, which we have
|
|
|
|
* to check for remote-safety.
|
|
|
|
*
|
|
|
|
* Note: the join clauses we see here should be the exact same ones
|
|
|
|
* previously examined by postgresGetForeignPaths. Possibly it'd be
|
|
|
|
* worth passing forward the classification work done then, rather
|
|
|
|
* than repeating it here.
|
|
|
|
*
|
|
|
|
* This code must match "extract_actual_clauses(scan_clauses, false)"
|
|
|
|
* except for the additional decision about remote versus local
|
|
|
|
* execution.
|
|
|
|
*/
|
|
|
|
foreach(lc, scan_clauses)
|
|
|
|
{
|
|
|
|
RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc);
|
|
|
|
|
|
|
|
/* Ignore any pseudoconstants, they're dealt with elsewhere */
|
|
|
|
if (rinfo->pseudoconstant)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (list_member_ptr(fpinfo->remote_conds, rinfo))
|
|
|
|
remote_exprs = lappend(remote_exprs, rinfo->clause);
|
|
|
|
else if (list_member_ptr(fpinfo->local_conds, rinfo))
|
|
|
|
local_exprs = lappend(local_exprs, rinfo->clause);
|
|
|
|
else if (is_foreign_expr(root, foreignrel, rinfo->clause))
|
|
|
|
remote_exprs = lappend(remote_exprs, rinfo->clause);
|
|
|
|
else
|
|
|
|
local_exprs = lappend(local_exprs, rinfo->clause);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For a base-relation scan, we have to support EPQ recheck, which
|
|
|
|
* should recheck all the remote quals.
|
|
|
|
*/
|
|
|
|
fdw_recheck_quals = remote_exprs;
|
|
|
|
}
|
2016-02-09 20:00:50 +01:00
|
|
|
else
|
|
|
|
{
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
/*
|
|
|
|
* Join relation or upper relation - set scan_relid to 0.
|
|
|
|
*/
|
2016-02-09 20:00:50 +01:00
|
|
|
scan_relid = 0;
|
|
|
|
|
|
|
|
/*
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
* For a join rel, baserestrictinfo is NIL and we are not considering
|
|
|
|
* parameterization right now, so there should be no scan_clauses for
|
|
|
|
* a joinrel or an upper rel either.
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
|
|
|
Assert(!scan_clauses);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
/*
|
|
|
|
* Instead we get the conditions to apply from the fdw_private
|
|
|
|
* structure.
|
|
|
|
*/
|
|
|
|
remote_exprs = extract_actual_clauses(fpinfo->remote_conds, false);
|
|
|
|
local_exprs = extract_actual_clauses(fpinfo->local_conds, false);
|
2013-03-22 00:43:59 +01:00
|
|
|
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
/*
|
|
|
|
* We leave fdw_recheck_quals empty in this case, since we never need
|
|
|
|
* to apply EPQ recheck clauses. In the case of a joinrel, EPQ
|
|
|
|
* recheck is handled elsewhere --- see postgresGetForeignJoinPaths().
|
|
|
|
* If we're planning an upperrel (ie, remote grouping or aggregation)
|
|
|
|
* then there's no EPQ to do because SELECT FOR UPDATE wouldn't be
|
|
|
|
* allowed, and indeed we *can't* put the remote clauses into
|
|
|
|
* fdw_recheck_quals because the unaggregated Vars won't be available
|
|
|
|
* locally.
|
|
|
|
*/
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
/* Build the list of columns to be fetched from the foreign server. */
|
|
|
|
fdw_scan_tlist = build_tlist_to_deparse(foreignrel);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure that the outer plan produces a tuple whose descriptor
|
2018-12-12 22:08:30 +01:00
|
|
|
* matches our scan tuple slot. Also, remove the local conditions
|
|
|
|
* from outer plan's quals, lest they be evaluated twice, once by the
|
|
|
|
* local plan and once by the scan.
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
|
|
|
if (outer_plan)
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
/*
|
|
|
|
* Right now, we only consider grouping and aggregation beyond
|
|
|
|
* joins. Queries involving aggregates or grouping do not require
|
|
|
|
* EPQ mechanism, hence should not have an outer plan here.
|
|
|
|
*/
|
Abstract logic to allow for multiple kinds of child rels.
Currently, the only type of child relation is an "other member rel",
which is the child of a baserel, but in the future joins and even
upper relations may have child rels. To facilitate that, introduce
macros that test to test for particular RelOptKind values, and use
them in various places where they help to clarify the sense of a test.
(For example, a test may allow RELOPT_OTHER_MEMBER_REL either because
it intends to allow child rels, or because it intends to allow simple
rels.)
Also, remove find_childrel_top_parent, which will not work for a
child rel that is not a baserel. Instead, add a new RelOptInfo
member top_parent_relids to track the same kind of information in a
more generic manner.
Ashutosh Bapat, slightly tweaked by me. Review and testing of the
patch set from which this was taken by Rajkumar Raghuwanshi and Rafia
Sabih.
Discussion: http://postgr.es/m/CA+TgmoagTnF2yqR3PT2rv=om=wJiZ4-A+ATwdnriTGku1CLYxA@mail.gmail.com
2017-04-04 04:41:31 +02:00
|
|
|
Assert(!IS_UPPER_REL(foreignrel));
|
2016-10-21 15:54:29 +02:00
|
|
|
|
2018-12-12 22:08:30 +01:00
|
|
|
/*
|
|
|
|
* First, update the plan's qual list if possible. In some cases
|
|
|
|
* the quals might be enforced below the topmost plan level, in
|
|
|
|
* which case we'll fail to remove them; it's not worth working
|
|
|
|
* harder than this.
|
|
|
|
*/
|
2016-02-09 20:00:50 +01:00
|
|
|
foreach(lc, local_exprs)
|
|
|
|
{
|
|
|
|
Node *qual = lfirst(lc);
|
|
|
|
|
|
|
|
outer_plan->qual = list_delete(outer_plan->qual, qual);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For an inner join the local conditions of foreign scan plan
|
2018-12-12 22:08:30 +01:00
|
|
|
* can be part of the joinquals as well. (They might also be
|
|
|
|
* in the mergequals or hashquals, but we can't touch those
|
|
|
|
* without breaking the plan.)
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
2018-12-12 22:08:30 +01:00
|
|
|
if (IsA(outer_plan, NestLoop) ||
|
|
|
|
IsA(outer_plan, MergeJoin) ||
|
|
|
|
IsA(outer_plan, HashJoin))
|
|
|
|
{
|
|
|
|
Join *join_plan = (Join *) outer_plan;
|
|
|
|
|
|
|
|
if (join_plan->jointype == JOIN_INNER)
|
|
|
|
join_plan->joinqual = list_delete(join_plan->joinqual,
|
|
|
|
qual);
|
|
|
|
}
|
2016-02-09 20:00:50 +01:00
|
|
|
}
|
2018-12-12 22:08:30 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Now fix the subplan's tlist --- this might result in inserting
|
|
|
|
* a Result node atop the plan tree.
|
|
|
|
*/
|
|
|
|
outer_plan = change_plan_targetlist(outer_plan, fdw_scan_tlist,
|
|
|
|
best_path->path.parallel_safe);
|
2016-02-09 20:00:50 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/*
|
|
|
|
* Build the query string to be sent for execution, and identify
|
|
|
|
* expressions to be sent as parameters.
|
|
|
|
*/
|
|
|
|
initStringInfo(&sql);
|
2016-02-09 20:00:50 +01:00
|
|
|
deparseSelectStmtForRel(&sql, root, foreignrel, fdw_scan_tlist,
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
remote_exprs, best_path->path.pathkeys,
|
2019-04-02 13:30:45 +02:00
|
|
|
has_final_sort, has_limit, false,
|
2019-04-02 12:20:30 +02:00
|
|
|
&retrieved_attrs, ¶ms_list);
|
2016-02-05 04:30:08 +01:00
|
|
|
|
2017-04-11 19:53:13 +02:00
|
|
|
/* Remember remote_exprs for possible use by postgresPlanDirectModify */
|
|
|
|
fpinfo->final_remote_exprs = remote_exprs;
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/*
|
|
|
|
* Build the fdw_private list that will be available to the executor.
|
2016-02-09 20:00:50 +01:00
|
|
|
* Items in the list must match order in enum FdwScanPrivateIndex.
|
2013-03-22 00:43:59 +01:00
|
|
|
*/
|
2017-04-11 19:53:13 +02:00
|
|
|
fdw_private = list_make3(makeString(sql.data),
|
2016-02-03 15:01:59 +01:00
|
|
|
retrieved_attrs,
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
makeInteger(fpinfo->fetch_size));
|
Abstract logic to allow for multiple kinds of child rels.
Currently, the only type of child relation is an "other member rel",
which is the child of a baserel, but in the future joins and even
upper relations may have child rels. To facilitate that, introduce
macros that test to test for particular RelOptKind values, and use
them in various places where they help to clarify the sense of a test.
(For example, a test may allow RELOPT_OTHER_MEMBER_REL either because
it intends to allow child rels, or because it intends to allow simple
rels.)
Also, remove find_childrel_top_parent, which will not work for a
child rel that is not a baserel. Instead, add a new RelOptInfo
member top_parent_relids to track the same kind of information in a
more generic manner.
Ashutosh Bapat, slightly tweaked by me. Review and testing of the
patch set from which this was taken by Rajkumar Raghuwanshi and Rafia
Sabih.
Discussion: http://postgr.es/m/CA+TgmoagTnF2yqR3PT2rv=om=wJiZ4-A+ATwdnriTGku1CLYxA@mail.gmail.com
2017-04-04 04:41:31 +02:00
|
|
|
if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel))
|
2016-02-09 20:00:50 +01:00
|
|
|
fdw_private = lappend(fdw_private,
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
makeString(fpinfo->relation_name));
|
2013-03-22 00:43:59 +01:00
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
2016-02-09 20:00:50 +01:00
|
|
|
* Create the ForeignScan node for the given relation.
|
2013-02-21 11:26:23 +01:00
|
|
|
*
|
2013-03-22 00:43:59 +01:00
|
|
|
* Note that the remote parameter expressions are stored in the fdw_exprs
|
|
|
|
* field of the finished plan node; we can't keep them in private state
|
|
|
|
* because then they wouldn't be subject to later planner processing.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
|
|
|
return make_foreignscan(tlist,
|
|
|
|
local_exprs,
|
|
|
|
scan_relid,
|
2013-03-22 00:43:59 +01:00
|
|
|
params_list,
|
Code review for foreign/custom join pushdown patch.
Commit e7cb7ee14555cc9c5773e2c102efd6371f6f2005 included some design
decisions that seem pretty questionable to me, and there was quite a lot
of stuff not to like about the documentation and comments. Clean up
as follows:
* Consider foreign joins only between foreign tables on the same server,
rather than between any two foreign tables with the same underlying FDW
handler function. In most if not all cases, the FDW would simply have had
to apply the same-server restriction itself (far more expensively, both for
lack of caching and because it would be repeated for each combination of
input sub-joins), or else risk nasty bugs. Anyone who's really intent on
doing something outside this restriction can always use the
set_join_pathlist_hook.
* Rename fdw_ps_tlist/custom_ps_tlist to fdw_scan_tlist/custom_scan_tlist
to better reflect what they're for, and allow these custom scan tlists
to be used even for base relations.
* Change make_foreignscan() API to include passing the fdw_scan_tlist
value, since the FDW is required to set that. Backwards compatibility
doesn't seem like an adequate reason to expect FDWs to set it in some
ad-hoc extra step, and anyway existing FDWs can just pass NIL.
* Change the API of path-generating subroutines of add_paths_to_joinrel,
and in particular that of GetForeignJoinPaths and set_join_pathlist_hook,
so that various less-used parameters are passed in a struct rather than
as separate parameter-list entries. The objective here is to reduce the
probability that future additions to those parameter lists will result in
source-level API breaks for users of these hooks. It's possible that this
is even a small win for the core code, since most CPU architectures can't
pass more than half a dozen parameters efficiently anyway. I kept root,
joinrel, outerrel, innerrel, and jointype as separate parameters to reduce
code churn in joinpath.c --- in particular, putting jointype into the
struct would have been problematic because of the subroutines' habit of
changing their local copies of that variable.
* Avoid ad-hocery in ExecAssignScanProjectionInfo. It was probably all
right for it to know about IndexOnlyScan, but if the list is to grow
we should refactor the knowledge out to the callers.
* Restore nodeForeignscan.c's previous use of the relcache to avoid
extra GetFdwRoutine lookups for base-relation scans.
* Lots of cleanup of documentation and missed comments. Re-order some
code additions into more logical places.
2015-05-10 20:36:30 +02:00
|
|
|
fdw_private,
|
2016-02-09 20:00:50 +01:00
|
|
|
fdw_scan_tlist,
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
fdw_recheck_quals,
|
Allow foreign and custom joins to handle EvalPlanQual rechecks.
Commit e7cb7ee14555cc9c5773e2c102efd6371f6f2005 provided basic
infrastructure for allowing a foreign data wrapper or custom scan
provider to replace a join of one or more tables with a scan.
However, this infrastructure failed to take into account the need
for possible EvalPlanQual rechecks, and ExecScanFetch would fail
an assertion (or just overwrite memory) if such a check was attempted
for a plan containing a pushed-down join. To fix, adjust the EPQ
machinery to skip some processing steps when scanrelid == 0, making
those the responsibility of scan's recheck method, which also has
the responsibility in this case of correctly populating the relevant
slot.
To allow foreign scans to gain control in the right place to make
use of this new facility, add a new, optional RecheckForeignScan
method. Also, allow a foreign scan to have a child plan, which can
be used to correctly populate the slot (or perhaps for something
else, but this is the only use currently envisioned).
KaiGai Kohei, reviewed by Robert Haas, Etsuro Fujita, and Kyotaro
Horiguchi.
2015-12-08 18:31:03 +01:00
|
|
|
outer_plan);
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
|
2021-06-05 02:07:08 +02:00
|
|
|
/*
|
|
|
|
* Construct a tuple descriptor for the scan tuples handled by a foreign join.
|
|
|
|
*/
|
|
|
|
static TupleDesc
|
|
|
|
get_tupdesc_for_join_scan_tuples(ForeignScanState *node)
|
|
|
|
{
|
|
|
|
ForeignScan *fsplan = (ForeignScan *) node->ss.ps.plan;
|
|
|
|
EState *estate = node->ss.ps.state;
|
|
|
|
TupleDesc tupdesc;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The core code has already set up a scan tuple slot based on
|
|
|
|
* fsplan->fdw_scan_tlist, and this slot's tupdesc is mostly good enough,
|
|
|
|
* but there's one case where it isn't. If we have any whole-row row
|
|
|
|
* identifier Vars, they may have vartype RECORD, and we need to replace
|
|
|
|
* that with the associated table's actual composite type. This ensures
|
|
|
|
* that when we read those ROW() expression values from the remote server,
|
|
|
|
* we can convert them to a composite type the local server knows.
|
|
|
|
*/
|
|
|
|
tupdesc = CreateTupleDescCopy(node->ss.ss_ScanTupleSlot->tts_tupleDescriptor);
|
|
|
|
for (int i = 0; i < tupdesc->natts; i++)
|
|
|
|
{
|
|
|
|
Form_pg_attribute att = TupleDescAttr(tupdesc, i);
|
|
|
|
Var *var;
|
|
|
|
RangeTblEntry *rte;
|
|
|
|
Oid reltype;
|
|
|
|
|
|
|
|
/* Nothing to do if it's not a generic RECORD attribute */
|
|
|
|
if (att->atttypid != RECORDOID || att->atttypmod >= 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we can't identify the referenced table, do nothing. This'll
|
|
|
|
* likely lead to failure later, but perhaps we can muddle through.
|
|
|
|
*/
|
|
|
|
var = (Var *) list_nth_node(TargetEntry, fsplan->fdw_scan_tlist,
|
|
|
|
i)->expr;
|
|
|
|
if (!IsA(var, Var) || var->varattno != 0)
|
|
|
|
continue;
|
|
|
|
rte = list_nth(estate->es_range_table, var->varno - 1);
|
|
|
|
if (rte->rtekind != RTE_RELATION)
|
|
|
|
continue;
|
|
|
|
reltype = get_rel_type_id(rte->relid);
|
|
|
|
if (!OidIsValid(reltype))
|
|
|
|
continue;
|
|
|
|
att->atttypid = reltype;
|
|
|
|
/* shouldn't need to change anything else */
|
|
|
|
}
|
|
|
|
return tupdesc;
|
|
|
|
}
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* postgresBeginForeignScan
|
|
|
|
* Initiate an executor scan of a foreign PostgreSQL table.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresBeginForeignScan(ForeignScanState *node, int eflags)
|
|
|
|
{
|
|
|
|
ForeignScan *fsplan = (ForeignScan *) node->ss.ps.plan;
|
|
|
|
EState *estate = node->ss.ps.state;
|
2013-03-10 19:14:53 +01:00
|
|
|
PgFdwScanState *fsstate;
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
RangeTblEntry *rte;
|
|
|
|
Oid userid;
|
|
|
|
ForeignTable *table;
|
2013-02-21 11:26:23 +01:00
|
|
|
UserMapping *user;
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
int rtindex;
|
2013-02-21 11:26:23 +01:00
|
|
|
int numParams;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Do nothing in EXPLAIN (no ANALYZE) case. node->fdw_state stays NULL.
|
|
|
|
*/
|
|
|
|
if (eflags & EXEC_FLAG_EXPLAIN_ONLY)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We'll save private state in node->fdw_state.
|
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate = (PgFdwScanState *) palloc0(sizeof(PgFdwScanState));
|
|
|
|
node->fdw_state = (void *) fsstate;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
* Identify which user to do the remote access as. This should match what
|
2016-10-21 15:54:29 +02:00
|
|
|
* ExecCheckRTEPerms() does. In case of a join or aggregate, use the
|
|
|
|
* lowest-numbered member RTE as a representative; we would get the same
|
|
|
|
* result from any.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2016-02-09 20:00:50 +01:00
|
|
|
if (fsplan->scan.scanrelid > 0)
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
rtindex = fsplan->scan.scanrelid;
|
2016-02-09 20:00:50 +01:00
|
|
|
else
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
rtindex = bms_next_member(fsplan->fs_relids, -1);
|
2018-10-04 21:48:17 +02:00
|
|
|
rte = exec_rt_fetch(rtindex, estate);
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
userid = rte->checkAsUser ? rte->checkAsUser : GetUserId();
|
2016-02-09 20:00:50 +01:00
|
|
|
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
/* Get info about foreign table. */
|
|
|
|
table = GetForeignTable(rte->relid);
|
|
|
|
user = GetUserMapping(userid, table->serverid);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get connection to the foreign server. Connection manager will
|
|
|
|
* establish new connection if necessary.
|
|
|
|
*/
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
fsstate->conn = GetConnection(user, false, &fsstate->conn_state);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Assign a unique ID for my cursor */
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->cursor_number = GetCursorNumber(fsstate->conn);
|
|
|
|
fsstate->cursor_exists = false;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Get private info created by planner functions. */
|
2013-03-22 05:31:11 +01:00
|
|
|
fsstate->query = strVal(list_nth(fsplan->fdw_private,
|
|
|
|
FdwScanPrivateSelectSql));
|
|
|
|
fsstate->retrieved_attrs = (List *) list_nth(fsplan->fdw_private,
|
|
|
|
FdwScanPrivateRetrievedAttrs);
|
2016-02-03 15:01:59 +01:00
|
|
|
fsstate->fetch_size = intVal(list_nth(fsplan->fdw_private,
|
|
|
|
FdwScanPrivateFetchSize));
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Create contexts for batches of tuples and per-tuple temp workspace. */
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->batch_cxt = AllocSetContextCreate(estate->es_query_cxt,
|
2013-02-21 11:26:23 +01:00
|
|
|
"postgres_fdw tuple data",
|
Add macros to make AllocSetContextCreate() calls simpler and safer.
I found that half a dozen (nearly 5%) of our AllocSetContextCreate calls
had typos in the context-sizing parameters. While none of these led to
especially significant problems, they did create minor inefficiencies,
and it's now clear that expecting people to copy-and-paste those calls
accurately is not a great idea. Let's reduce the risk of future errors
by introducing single macros that encapsulate the common use-cases.
Three such macros are enough to cover all but two special-purpose contexts;
those two calls can be left as-is, I think.
While this patch doesn't in itself improve matters for third-party
extensions, it doesn't break anything for them either, and they can
gradually adopt the simplified notation over time.
In passing, change TopMemoryContext to use the default allocation
parameters. Formerly it could only be extended 8K at a time. That was
probably reasonable when this code was written; but nowadays we create
many more contexts than we did then, so that it's not unusual to have a
couple hundred K in TopMemoryContext, even without considering various
dubious code that sticks other things there. There seems no good reason
not to let it use growing blocks like most other contexts.
Back-patch to 9.6, mostly because that's still close enough to HEAD that
it's easy to do so, and keeping the branches in sync can be expected to
avoid some future back-patching pain. The bugs fixed by these changes
don't seem to be significant enough to justify fixing them further back.
Discussion: <21072.1472321324@sss.pgh.pa.us>
2016-08-27 23:50:38 +02:00
|
|
|
ALLOCSET_DEFAULT_SIZES);
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->temp_cxt = AllocSetContextCreate(estate->es_query_cxt,
|
2013-02-21 11:26:23 +01:00
|
|
|
"postgres_fdw temporary data",
|
Add macros to make AllocSetContextCreate() calls simpler and safer.
I found that half a dozen (nearly 5%) of our AllocSetContextCreate calls
had typos in the context-sizing parameters. While none of these led to
especially significant problems, they did create minor inefficiencies,
and it's now clear that expecting people to copy-and-paste those calls
accurately is not a great idea. Let's reduce the risk of future errors
by introducing single macros that encapsulate the common use-cases.
Three such macros are enough to cover all but two special-purpose contexts;
those two calls can be left as-is, I think.
While this patch doesn't in itself improve matters for third-party
extensions, it doesn't break anything for them either, and they can
gradually adopt the simplified notation over time.
In passing, change TopMemoryContext to use the default allocation
parameters. Formerly it could only be extended 8K at a time. That was
probably reasonable when this code was written; but nowadays we create
many more contexts than we did then, so that it's not unusual to have a
couple hundred K in TopMemoryContext, even without considering various
dubious code that sticks other things there. There seems no good reason
not to let it use growing blocks like most other contexts.
Back-patch to 9.6, mostly because that's still close enough to HEAD that
it's easy to do so, and keeping the branches in sync can be expected to
avoid some future back-patching pain. The bugs fixed by these changes
don't seem to be significant enough to justify fixing them further back.
Discussion: <21072.1472321324@sss.pgh.pa.us>
2016-08-27 23:50:38 +02:00
|
|
|
ALLOCSET_SMALL_SIZES);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
|
|
|
* Get info we'll need for converting data fetched from the foreign server
|
|
|
|
* into local representation and error reporting during that process.
|
|
|
|
*/
|
|
|
|
if (fsplan->scan.scanrelid > 0)
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
{
|
|
|
|
fsstate->rel = node->ss.ss_currentRelation;
|
2016-02-09 20:00:50 +01:00
|
|
|
fsstate->tupdesc = RelationGetDescr(fsstate->rel);
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
}
|
2016-02-09 20:00:50 +01:00
|
|
|
else
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
{
|
|
|
|
fsstate->rel = NULL;
|
2021-06-05 02:07:08 +02:00
|
|
|
fsstate->tupdesc = get_tupdesc_for_join_scan_tuples(node);
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
}
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
fsstate->attinmeta = TupleDescGetAttInMetadata(fsstate->tupdesc);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/*
|
2016-03-18 18:48:58 +01:00
|
|
|
* Prepare for processing of parameters used in remote query, if any.
|
2013-03-22 00:43:59 +01:00
|
|
|
*/
|
2016-03-18 18:48:58 +01:00
|
|
|
numParams = list_length(fsplan->fdw_exprs);
|
|
|
|
fsstate->numParams = numParams;
|
2013-02-21 11:26:23 +01:00
|
|
|
if (numParams > 0)
|
2016-03-18 18:48:58 +01:00
|
|
|
prepare_query_params((PlanState *) node,
|
|
|
|
fsplan->fdw_exprs,
|
|
|
|
numParams,
|
|
|
|
&fsstate->param_flinfo,
|
|
|
|
&fsstate->param_exprs,
|
|
|
|
&fsstate->param_values);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
|
|
|
|
/* Set the async-capable flag */
|
2021-05-12 07:00:00 +02:00
|
|
|
fsstate->async_capable = node->ss.ps.async_capable;
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresIterateForeignScan
|
|
|
|
* Retrieve next row from the result set, or clear tuple slot to indicate
|
|
|
|
* EOF.
|
|
|
|
*/
|
|
|
|
static TupleTableSlot *
|
|
|
|
postgresIterateForeignScan(ForeignScanState *node)
|
|
|
|
{
|
2013-03-10 19:14:53 +01:00
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
2013-02-21 11:26:23 +01:00
|
|
|
TupleTableSlot *slot = node->ss.ss_ScanTupleSlot;
|
|
|
|
|
|
|
|
/*
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
* In sync mode, if this is the first call after Begin or ReScan, we need
|
|
|
|
* to create the cursor on the remote side. In async mode, we would have
|
|
|
|
* already created the cursor before we get here, even if this is the
|
|
|
|
* first call after Begin or ReScan.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
if (!fsstate->cursor_exists)
|
2013-02-21 11:26:23 +01:00
|
|
|
create_cursor(node);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get some more tuples, if we've run out.
|
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
if (fsstate->next_tuple >= fsstate->num_tuples)
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/* In async mode, just clear tuple slot. */
|
|
|
|
if (fsstate->async_capable)
|
|
|
|
return ExecClearTuple(slot);
|
2013-02-21 11:26:23 +01:00
|
|
|
/* No point in another fetch if we already detected EOF, though. */
|
2013-03-10 19:14:53 +01:00
|
|
|
if (!fsstate->eof_reached)
|
2013-02-21 11:26:23 +01:00
|
|
|
fetch_more_data(node);
|
|
|
|
/* If we didn't get any tuples, must be end of data. */
|
2013-03-10 19:14:53 +01:00
|
|
|
if (fsstate->next_tuple >= fsstate->num_tuples)
|
2013-02-21 11:26:23 +01:00
|
|
|
return ExecClearTuple(slot);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return the next tuple.
|
|
|
|
*/
|
2018-09-26 01:27:48 +02:00
|
|
|
ExecStoreHeapTuple(fsstate->tuples[fsstate->next_tuple++],
|
|
|
|
slot,
|
|
|
|
false);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
return slot;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresReScanForeignScan
|
|
|
|
* Restart the scan.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresReScanForeignScan(ForeignScanState *node)
|
|
|
|
{
|
2013-03-10 19:14:53 +01:00
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
2013-02-21 11:26:23 +01:00
|
|
|
char sql[64];
|
|
|
|
PGresult *res;
|
|
|
|
|
|
|
|
/* If we haven't created the cursor yet, nothing to do. */
|
2013-03-10 19:14:53 +01:00
|
|
|
if (!fsstate->cursor_exists)
|
2013-02-21 11:26:23 +01:00
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If any internal parameters affecting this node have changed, we'd
|
|
|
|
* better destroy and recreate the cursor. Otherwise, rewinding it should
|
|
|
|
* be good enough. If we've only fetched zero or one batch, we needn't
|
|
|
|
* even rewind the cursor, just rescan what we have.
|
|
|
|
*/
|
|
|
|
if (node->ss.ps.chgParam != NULL)
|
|
|
|
{
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->cursor_exists = false;
|
2013-02-21 11:26:23 +01:00
|
|
|
snprintf(sql, sizeof(sql), "CLOSE c%u",
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->cursor_number);
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
2013-03-10 19:14:53 +01:00
|
|
|
else if (fsstate->fetch_ct_2 > 1)
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
|
|
|
snprintf(sql, sizeof(sql), "MOVE BACKWARD ALL IN c%u",
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->cursor_number);
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* Easy: just rescan what we already have in memory, if anything */
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->next_tuple = 0;
|
2013-02-21 11:26:23 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't use a PG_TRY block here, so be careful not to throw error
|
|
|
|
* without releasing the PGresult.
|
|
|
|
*/
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(fsstate->conn, sql, fsstate->conn_state);
|
2013-02-21 11:26:23 +01:00
|
|
|
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
2014-02-04 03:30:02 +01:00
|
|
|
pgfdw_report_error(ERROR, res, fsstate->conn, true, sql);
|
2013-02-21 11:26:23 +01:00
|
|
|
PQclear(res);
|
|
|
|
|
|
|
|
/* Now force a fresh FETCH. */
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->tuples = NULL;
|
|
|
|
fsstate->num_tuples = 0;
|
|
|
|
fsstate->next_tuple = 0;
|
|
|
|
fsstate->fetch_ct_2 = 0;
|
|
|
|
fsstate->eof_reached = false;
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresEndForeignScan
|
|
|
|
* Finish scanning foreign table and dispose objects used for this scan
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresEndForeignScan(ForeignScanState *node)
|
|
|
|
{
|
2013-03-10 19:14:53 +01:00
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/* if fsstate is NULL, we are in EXPLAIN; nothing to do */
|
|
|
|
if (fsstate == NULL)
|
2013-02-21 11:26:23 +01:00
|
|
|
return;
|
|
|
|
|
|
|
|
/* Close the cursor if open, to prevent accumulation of cursors */
|
2013-03-10 19:14:53 +01:00
|
|
|
if (fsstate->cursor_exists)
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
close_cursor(fsstate->conn, fsstate->cursor_number,
|
|
|
|
fsstate->conn_state);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Release remote connection */
|
2013-03-10 19:14:53 +01:00
|
|
|
ReleaseConnection(fsstate->conn);
|
|
|
|
fsstate->conn = NULL;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* MemoryContexts will be deleted automatically. */
|
|
|
|
}
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/*
|
|
|
|
* postgresAddForeignUpdateTargets
|
|
|
|
* Add resjunk column(s) needed for update/delete on a foreign table
|
|
|
|
*/
|
|
|
|
static void
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
postgresAddForeignUpdateTargets(PlannerInfo *root,
|
|
|
|
Index rtindex,
|
2013-03-10 19:14:53 +01:00
|
|
|
RangeTblEntry *target_rte,
|
|
|
|
Relation target_relation)
|
|
|
|
{
|
|
|
|
Var *var;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* In postgres_fdw, what we need is the ctid, same as for a regular table.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* Make a Var representing the desired value */
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
var = makeVar(rtindex,
|
2013-03-10 19:14:53 +01:00
|
|
|
SelfItemPointerAttributeNumber,
|
|
|
|
TIDOID,
|
|
|
|
-1,
|
|
|
|
InvalidOid,
|
|
|
|
0);
|
|
|
|
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
/* Register it as a row-identity column needed by this target rel */
|
|
|
|
add_row_identity_var(root, var, rtindex, "ctid");
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresPlanForeignModify
|
|
|
|
* Plan an insert/update/delete operation on a foreign table
|
|
|
|
*/
|
|
|
|
static List *
|
|
|
|
postgresPlanForeignModify(PlannerInfo *root,
|
|
|
|
ModifyTable *plan,
|
|
|
|
Index resultRelation,
|
|
|
|
int subplan_index)
|
|
|
|
{
|
|
|
|
CmdType operation = plan->operation;
|
2013-03-12 23:58:13 +01:00
|
|
|
RangeTblEntry *rte = planner_rt_fetch(resultRelation, root);
|
|
|
|
Relation rel;
|
2013-03-10 19:14:53 +01:00
|
|
|
StringInfoData sql;
|
|
|
|
List *targetAttrs = NIL;
|
Fix WITH CHECK OPTION on views referencing postgres_fdw tables.
If a view references a foreign table, and the foreign table has a
BEFORE INSERT trigger, then it's possible for a tuple inserted or
updated through the view to be changed such that it violates the
view's WITH CHECK OPTION constraint.
Before this commit, postgres_fdw handled this case inconsistently. A
RETURNING clause on the INSERT or UPDATE statement targeting the view
would cause the finally-inserted tuple to be read back, and the WITH
CHECK OPTION violation would throw an error. But without a RETURNING
clause, postgres_fdw would not read the final tuple back, and WITH
CHECK OPTION would not throw an error for the violation (or may throw
an error when there is no real violation). AFTER ROW triggers on the
foreign table had a similar effect as a RETURNING clause on the INSERT
or UPDATE statement.
To fix, this commit retrieves the attributes needed to enforce the
WITH CHECK OPTION constraint along with the attributes needed for the
RETURNING clause (if any) from the remote side. Thus, the WITH CHECK
OPTION constraint is always evaluated against the final tuple after
any triggers on the remote side.
This fix may be considered inconsistent with CHECK constraints
declared on foreign tables, which are not enforced locally at all
(because the constraint is on a remote object). The discussion
concluded that this difference is reasonable, because the WITH CHECK
OPTION is a constraint on the local view (not any remote object);
therefore it only makes sense to enforce its WITH CHECK OPTION
constraint locally.
Author: Etsuro Fujita
Reviewed-by: Arthur Zakirov, Stephen Frost
Discussion: https://www.postgresql.org/message-id/7eb58fab-fd3b-781b-ac33-f7cfec96021f%40lab.ntt.co.jp
2018-07-08 09:14:51 +02:00
|
|
|
List *withCheckOptionList = NIL;
|
2013-03-10 19:14:53 +01:00
|
|
|
List *returningList = NIL;
|
2013-03-22 05:31:11 +01:00
|
|
|
List *retrieved_attrs = NIL;
|
Add support for INSERT ... ON CONFLICT DO NOTHING/UPDATE.
The newly added ON CONFLICT clause allows to specify an alternative to
raising a unique or exclusion constraint violation error when inserting.
ON CONFLICT refers to constraints that can either be specified using a
inference clause (by specifying the columns of a unique constraint) or
by naming a unique or exclusion constraint. DO NOTHING avoids the
constraint violation, without touching the pre-existing row. DO UPDATE
SET ... [WHERE ...] updates the pre-existing tuple, and has access to
both the tuple proposed for insertion and the existing tuple; the
optional WHERE clause can be used to prevent an update from being
executed. The UPDATE SET and WHERE clauses have access to the tuple
proposed for insertion using the "magic" EXCLUDED alias, and to the
pre-existing tuple using the table name or its alias.
This feature is often referred to as upsert.
This is implemented using a new infrastructure called "speculative
insertion". It is an optimistic variant of regular insertion that first
does a pre-check for existing tuples and then attempts an insert. If a
violating tuple was inserted concurrently, the speculatively inserted
tuple is deleted and a new attempt is made. If the pre-check finds a
matching tuple the alternative DO NOTHING or DO UPDATE action is taken.
If the insertion succeeds without detecting a conflict, the tuple is
deemed inserted.
To handle the possible ambiguity between the excluded alias and a table
named excluded, and for convenience with long relation names, INSERT
INTO now can alias its target table.
Bumps catversion as stored rules change.
Author: Peter Geoghegan, with significant contributions from Heikki
Linnakangas and Andres Freund. Testing infrastructure by Jeff Janes.
Reviewed-By: Heikki Linnakangas, Andres Freund, Robert Haas, Simon Riggs,
Dean Rasheed, Stephen Frost and many others.
2015-05-08 05:31:36 +02:00
|
|
|
bool doNothing = false;
|
2021-01-20 23:05:46 +01:00
|
|
|
int values_end_len = -1;
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
initStringInfo(&sql);
|
|
|
|
|
|
|
|
/*
|
2013-03-12 23:58:13 +01:00
|
|
|
* Core code already has some lock on each rel being planned, so we can
|
|
|
|
* use NoLock here.
|
2013-03-10 19:14:53 +01:00
|
|
|
*/
|
2019-01-21 19:32:19 +01:00
|
|
|
rel = table_open(rte->relid, NoLock);
|
2013-03-12 23:58:13 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* In an INSERT, we transmit all columns that are defined in the foreign
|
2019-06-13 10:59:09 +02:00
|
|
|
* table. In an UPDATE, if there are BEFORE ROW UPDATE triggers on the
|
|
|
|
* foreign table, we transmit all columns like INSERT; else we transmit
|
|
|
|
* only columns that were explicitly targets of the UPDATE, so as to avoid
|
|
|
|
* unnecessary data transmission. (We can't do that for INSERT since we
|
|
|
|
* would miss sending default values for columns not listed in the source
|
|
|
|
* statement, and for UPDATE if there are BEFORE ROW UPDATE triggers since
|
|
|
|
* those triggers might change values for non-target columns, in which
|
|
|
|
* case we would miss sending changed values for those columns.)
|
2013-03-12 23:58:13 +01:00
|
|
|
*/
|
2019-06-13 10:59:09 +02:00
|
|
|
if (operation == CMD_INSERT ||
|
|
|
|
(operation == CMD_UPDATE &&
|
|
|
|
rel->trigdesc &&
|
|
|
|
rel->trigdesc->trig_update_before_row))
|
2013-03-12 23:58:13 +01:00
|
|
|
{
|
|
|
|
TupleDesc tupdesc = RelationGetDescr(rel);
|
|
|
|
int attnum;
|
|
|
|
|
|
|
|
for (attnum = 1; attnum <= tupdesc->natts; attnum++)
|
|
|
|
{
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1);
|
2013-03-12 23:58:13 +01:00
|
|
|
|
|
|
|
if (!attr->attisdropped)
|
|
|
|
targetAttrs = lappend_int(targetAttrs, attnum);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else if (operation == CMD_UPDATE)
|
2013-03-10 19:14:53 +01:00
|
|
|
{
|
2014-11-28 19:37:25 +01:00
|
|
|
int col;
|
2019-03-30 08:13:09 +01:00
|
|
|
Bitmapset *allUpdatedCols = bms_union(rte->updatedCols, rte->extraUpdatedCols);
|
2013-03-10 19:14:53 +01:00
|
|
|
|
2014-11-28 19:37:25 +01:00
|
|
|
col = -1;
|
2019-03-30 08:13:09 +01:00
|
|
|
while ((col = bms_next_member(allUpdatedCols, col)) >= 0)
|
2013-03-10 19:14:53 +01:00
|
|
|
{
|
2014-11-28 19:37:25 +01:00
|
|
|
/* bit numbers are offset by FirstLowInvalidHeapAttributeNumber */
|
|
|
|
AttrNumber attno = col + FirstLowInvalidHeapAttributeNumber;
|
|
|
|
|
|
|
|
if (attno <= InvalidAttrNumber) /* shouldn't happen */
|
2013-03-10 19:14:53 +01:00
|
|
|
elog(ERROR, "system-column update is not supported");
|
2014-11-28 19:37:25 +01:00
|
|
|
targetAttrs = lappend_int(targetAttrs, attno);
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Fix WITH CHECK OPTION on views referencing postgres_fdw tables.
If a view references a foreign table, and the foreign table has a
BEFORE INSERT trigger, then it's possible for a tuple inserted or
updated through the view to be changed such that it violates the
view's WITH CHECK OPTION constraint.
Before this commit, postgres_fdw handled this case inconsistently. A
RETURNING clause on the INSERT or UPDATE statement targeting the view
would cause the finally-inserted tuple to be read back, and the WITH
CHECK OPTION violation would throw an error. But without a RETURNING
clause, postgres_fdw would not read the final tuple back, and WITH
CHECK OPTION would not throw an error for the violation (or may throw
an error when there is no real violation). AFTER ROW triggers on the
foreign table had a similar effect as a RETURNING clause on the INSERT
or UPDATE statement.
To fix, this commit retrieves the attributes needed to enforce the
WITH CHECK OPTION constraint along with the attributes needed for the
RETURNING clause (if any) from the remote side. Thus, the WITH CHECK
OPTION constraint is always evaluated against the final tuple after
any triggers on the remote side.
This fix may be considered inconsistent with CHECK constraints
declared on foreign tables, which are not enforced locally at all
(because the constraint is on a remote object). The discussion
concluded that this difference is reasonable, because the WITH CHECK
OPTION is a constraint on the local view (not any remote object);
therefore it only makes sense to enforce its WITH CHECK OPTION
constraint locally.
Author: Etsuro Fujita
Reviewed-by: Arthur Zakirov, Stephen Frost
Discussion: https://www.postgresql.org/message-id/7eb58fab-fd3b-781b-ac33-f7cfec96021f%40lab.ntt.co.jp
2018-07-08 09:14:51 +02:00
|
|
|
/*
|
|
|
|
* Extract the relevant WITH CHECK OPTION list if any.
|
|
|
|
*/
|
|
|
|
if (plan->withCheckOptionLists)
|
|
|
|
withCheckOptionList = (List *) list_nth(plan->withCheckOptionLists,
|
|
|
|
subplan_index);
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/*
|
|
|
|
* Extract the relevant RETURNING list if any.
|
|
|
|
*/
|
|
|
|
if (plan->returningLists)
|
|
|
|
returningList = (List *) list_nth(plan->returningLists, subplan_index);
|
|
|
|
|
Add support for INSERT ... ON CONFLICT DO NOTHING/UPDATE.
The newly added ON CONFLICT clause allows to specify an alternative to
raising a unique or exclusion constraint violation error when inserting.
ON CONFLICT refers to constraints that can either be specified using a
inference clause (by specifying the columns of a unique constraint) or
by naming a unique or exclusion constraint. DO NOTHING avoids the
constraint violation, without touching the pre-existing row. DO UPDATE
SET ... [WHERE ...] updates the pre-existing tuple, and has access to
both the tuple proposed for insertion and the existing tuple; the
optional WHERE clause can be used to prevent an update from being
executed. The UPDATE SET and WHERE clauses have access to the tuple
proposed for insertion using the "magic" EXCLUDED alias, and to the
pre-existing tuple using the table name or its alias.
This feature is often referred to as upsert.
This is implemented using a new infrastructure called "speculative
insertion". It is an optimistic variant of regular insertion that first
does a pre-check for existing tuples and then attempts an insert. If a
violating tuple was inserted concurrently, the speculatively inserted
tuple is deleted and a new attempt is made. If the pre-check finds a
matching tuple the alternative DO NOTHING or DO UPDATE action is taken.
If the insertion succeeds without detecting a conflict, the tuple is
deemed inserted.
To handle the possible ambiguity between the excluded alias and a table
named excluded, and for convenience with long relation names, INSERT
INTO now can alias its target table.
Bumps catversion as stored rules change.
Author: Peter Geoghegan, with significant contributions from Heikki
Linnakangas and Andres Freund. Testing infrastructure by Jeff Janes.
Reviewed-By: Heikki Linnakangas, Andres Freund, Robert Haas, Simon Riggs,
Dean Rasheed, Stephen Frost and many others.
2015-05-08 05:31:36 +02:00
|
|
|
/*
|
|
|
|
* ON CONFLICT DO UPDATE and DO NOTHING case with inference specification
|
|
|
|
* should have already been rejected in the optimizer, as presently there
|
|
|
|
* is no way to recognize an arbiter index on a foreign table. Only DO
|
|
|
|
* NOTHING is supported without an inference specification.
|
|
|
|
*/
|
|
|
|
if (plan->onConflictAction == ONCONFLICT_NOTHING)
|
|
|
|
doNothing = true;
|
|
|
|
else if (plan->onConflictAction != ONCONFLICT_NONE)
|
|
|
|
elog(ERROR, "unexpected ON CONFLICT specification: %d",
|
|
|
|
(int) plan->onConflictAction);
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/*
|
|
|
|
* Construct the SQL command string.
|
|
|
|
*/
|
|
|
|
switch (operation)
|
|
|
|
{
|
|
|
|
case CMD_INSERT:
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
deparseInsertSql(&sql, rte, resultRelation, rel,
|
Fix WITH CHECK OPTION on views referencing postgres_fdw tables.
If a view references a foreign table, and the foreign table has a
BEFORE INSERT trigger, then it's possible for a tuple inserted or
updated through the view to be changed such that it violates the
view's WITH CHECK OPTION constraint.
Before this commit, postgres_fdw handled this case inconsistently. A
RETURNING clause on the INSERT or UPDATE statement targeting the view
would cause the finally-inserted tuple to be read back, and the WITH
CHECK OPTION violation would throw an error. But without a RETURNING
clause, postgres_fdw would not read the final tuple back, and WITH
CHECK OPTION would not throw an error for the violation (or may throw
an error when there is no real violation). AFTER ROW triggers on the
foreign table had a similar effect as a RETURNING clause on the INSERT
or UPDATE statement.
To fix, this commit retrieves the attributes needed to enforce the
WITH CHECK OPTION constraint along with the attributes needed for the
RETURNING clause (if any) from the remote side. Thus, the WITH CHECK
OPTION constraint is always evaluated against the final tuple after
any triggers on the remote side.
This fix may be considered inconsistent with CHECK constraints
declared on foreign tables, which are not enforced locally at all
(because the constraint is on a remote object). The discussion
concluded that this difference is reasonable, because the WITH CHECK
OPTION is a constraint on the local view (not any remote object);
therefore it only makes sense to enforce its WITH CHECK OPTION
constraint locally.
Author: Etsuro Fujita
Reviewed-by: Arthur Zakirov, Stephen Frost
Discussion: https://www.postgresql.org/message-id/7eb58fab-fd3b-781b-ac33-f7cfec96021f%40lab.ntt.co.jp
2018-07-08 09:14:51 +02:00
|
|
|
targetAttrs, doNothing,
|
|
|
|
withCheckOptionList, returningList,
|
2021-01-20 23:05:46 +01:00
|
|
|
&retrieved_attrs, &values_end_len);
|
2013-03-10 19:14:53 +01:00
|
|
|
break;
|
|
|
|
case CMD_UPDATE:
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
deparseUpdateSql(&sql, rte, resultRelation, rel,
|
Fix WITH CHECK OPTION on views referencing postgres_fdw tables.
If a view references a foreign table, and the foreign table has a
BEFORE INSERT trigger, then it's possible for a tuple inserted or
updated through the view to be changed such that it violates the
view's WITH CHECK OPTION constraint.
Before this commit, postgres_fdw handled this case inconsistently. A
RETURNING clause on the INSERT or UPDATE statement targeting the view
would cause the finally-inserted tuple to be read back, and the WITH
CHECK OPTION violation would throw an error. But without a RETURNING
clause, postgres_fdw would not read the final tuple back, and WITH
CHECK OPTION would not throw an error for the violation (or may throw
an error when there is no real violation). AFTER ROW triggers on the
foreign table had a similar effect as a RETURNING clause on the INSERT
or UPDATE statement.
To fix, this commit retrieves the attributes needed to enforce the
WITH CHECK OPTION constraint along with the attributes needed for the
RETURNING clause (if any) from the remote side. Thus, the WITH CHECK
OPTION constraint is always evaluated against the final tuple after
any triggers on the remote side.
This fix may be considered inconsistent with CHECK constraints
declared on foreign tables, which are not enforced locally at all
(because the constraint is on a remote object). The discussion
concluded that this difference is reasonable, because the WITH CHECK
OPTION is a constraint on the local view (not any remote object);
therefore it only makes sense to enforce its WITH CHECK OPTION
constraint locally.
Author: Etsuro Fujita
Reviewed-by: Arthur Zakirov, Stephen Frost
Discussion: https://www.postgresql.org/message-id/7eb58fab-fd3b-781b-ac33-f7cfec96021f%40lab.ntt.co.jp
2018-07-08 09:14:51 +02:00
|
|
|
targetAttrs,
|
|
|
|
withCheckOptionList, returningList,
|
2013-03-22 05:31:11 +01:00
|
|
|
&retrieved_attrs);
|
2013-03-10 19:14:53 +01:00
|
|
|
break;
|
|
|
|
case CMD_DELETE:
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
deparseDeleteSql(&sql, rte, resultRelation, rel,
|
2013-03-22 05:31:11 +01:00
|
|
|
returningList,
|
|
|
|
&retrieved_attrs);
|
2013-03-10 19:14:53 +01:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
elog(ERROR, "unexpected operation: %d", (int) operation);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, NoLock);
|
2013-03-12 23:58:13 +01:00
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/*
|
|
|
|
* Build the fdw_private list that will be available to the executor.
|
|
|
|
* Items in the list must match enum FdwModifyPrivateIndex, above.
|
|
|
|
*/
|
2021-01-20 23:05:46 +01:00
|
|
|
return list_make5(makeString(sql.data),
|
2013-03-10 19:14:53 +01:00
|
|
|
targetAttrs,
|
2021-01-20 23:05:46 +01:00
|
|
|
makeInteger(values_end_len),
|
2014-03-23 07:16:34 +01:00
|
|
|
makeInteger((retrieved_attrs != NIL)),
|
2013-03-22 05:31:11 +01:00
|
|
|
retrieved_attrs);
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresBeginForeignModify
|
|
|
|
* Begin an insert/update/delete operation on a foreign table
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresBeginForeignModify(ModifyTableState *mtstate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
List *fdw_private,
|
|
|
|
int subplan_index,
|
|
|
|
int eflags)
|
|
|
|
{
|
|
|
|
PgFdwModifyState *fmstate;
|
2018-04-06 17:29:43 +02:00
|
|
|
char *query;
|
|
|
|
List *target_attrs;
|
|
|
|
bool has_returning;
|
2021-01-20 23:05:46 +01:00
|
|
|
int values_end_len;
|
2018-04-06 17:29:43 +02:00
|
|
|
List *retrieved_attrs;
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
RangeTblEntry *rte;
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Do nothing in EXPLAIN (no ANALYZE) case. resultRelInfo->ri_FdwState
|
|
|
|
* stays NULL.
|
|
|
|
*/
|
|
|
|
if (eflags & EXEC_FLAG_EXPLAIN_ONLY)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Deconstruct fdw_private data. */
|
2018-04-06 17:29:43 +02:00
|
|
|
query = strVal(list_nth(fdw_private,
|
|
|
|
FdwModifyPrivateUpdateSql));
|
|
|
|
target_attrs = (List *) list_nth(fdw_private,
|
|
|
|
FdwModifyPrivateTargetAttnums);
|
2021-01-20 23:05:46 +01:00
|
|
|
values_end_len = intVal(list_nth(fdw_private,
|
|
|
|
FdwModifyPrivateLen));
|
2018-04-06 17:29:43 +02:00
|
|
|
has_returning = intVal(list_nth(fdw_private,
|
|
|
|
FdwModifyPrivateHasReturning));
|
|
|
|
retrieved_attrs = (List *) list_nth(fdw_private,
|
|
|
|
FdwModifyPrivateRetrievedAttrs);
|
|
|
|
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
/* Find RTE. */
|
2018-10-04 21:48:17 +02:00
|
|
|
rte = exec_rt_fetch(resultRelInfo->ri_RangeTableIndex,
|
|
|
|
mtstate->ps.state);
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
|
2018-04-06 17:29:43 +02:00
|
|
|
/* Construct an execution state. */
|
|
|
|
fmstate = create_foreign_modify(mtstate->ps.state,
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
rte,
|
2018-04-06 17:29:43 +02:00
|
|
|
resultRelInfo,
|
|
|
|
mtstate->operation,
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
outerPlanState(mtstate)->plan,
|
2018-04-06 17:29:43 +02:00
|
|
|
query,
|
|
|
|
target_attrs,
|
2021-01-20 23:05:46 +01:00
|
|
|
values_end_len,
|
2018-04-06 17:29:43 +02:00
|
|
|
has_returning,
|
|
|
|
retrieved_attrs);
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
resultRelInfo->ri_FdwState = fmstate;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresExecForeignInsert
|
|
|
|
* Insert one row into a foreign table
|
|
|
|
*/
|
|
|
|
static TupleTableSlot *
|
|
|
|
postgresExecForeignInsert(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
TupleTableSlot *slot,
|
|
|
|
TupleTableSlot *planSlot)
|
|
|
|
{
|
2019-04-24 11:31:50 +02:00
|
|
|
PgFdwModifyState *fmstate = (PgFdwModifyState *) resultRelInfo->ri_FdwState;
|
2021-01-20 23:05:46 +01:00
|
|
|
TupleTableSlot **rslot;
|
|
|
|
int numSlots = 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the fmstate has aux_fmstate set, use the aux_fmstate (see
|
|
|
|
* postgresBeginForeignInsert())
|
|
|
|
*/
|
|
|
|
if (fmstate->aux_fmstate)
|
|
|
|
resultRelInfo->ri_FdwState = fmstate->aux_fmstate;
|
|
|
|
rslot = execute_foreign_modify(estate, resultRelInfo, CMD_INSERT,
|
|
|
|
&slot, &planSlot, &numSlots);
|
|
|
|
/* Revert that change */
|
|
|
|
if (fmstate->aux_fmstate)
|
|
|
|
resultRelInfo->ri_FdwState = fmstate;
|
|
|
|
|
|
|
|
return rslot ? *rslot : NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresExecForeignBatchInsert
|
|
|
|
* Insert multiple rows into a foreign table
|
|
|
|
*/
|
|
|
|
static TupleTableSlot **
|
|
|
|
postgresExecForeignBatchInsert(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
TupleTableSlot **slots,
|
|
|
|
TupleTableSlot **planSlots,
|
|
|
|
int *numSlots)
|
|
|
|
{
|
|
|
|
PgFdwModifyState *fmstate = (PgFdwModifyState *) resultRelInfo->ri_FdwState;
|
|
|
|
TupleTableSlot **rslot;
|
2019-04-24 11:31:50 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the fmstate has aux_fmstate set, use the aux_fmstate (see
|
|
|
|
* postgresBeginForeignInsert())
|
|
|
|
*/
|
|
|
|
if (fmstate->aux_fmstate)
|
|
|
|
resultRelInfo->ri_FdwState = fmstate->aux_fmstate;
|
|
|
|
rslot = execute_foreign_modify(estate, resultRelInfo, CMD_INSERT,
|
2021-01-20 23:05:46 +01:00
|
|
|
slots, planSlots, numSlots);
|
2019-04-24 11:31:50 +02:00
|
|
|
/* Revert that change */
|
|
|
|
if (fmstate->aux_fmstate)
|
|
|
|
resultRelInfo->ri_FdwState = fmstate;
|
|
|
|
|
|
|
|
return rslot;
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
/*
|
|
|
|
* postgresGetForeignModifyBatchSize
|
|
|
|
* Determine the maximum number of tuples that can be inserted in bulk
|
|
|
|
*
|
|
|
|
* Returns the batch size specified for server or table. When batching is not
|
|
|
|
* allowed (e.g. for tables with AFTER ROW triggers or with RETURNING clause),
|
|
|
|
* returns 1.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
postgresGetForeignModifyBatchSize(ResultRelInfo *resultRelInfo)
|
|
|
|
{
|
|
|
|
int batch_size;
|
2021-02-18 00:02:00 +01:00
|
|
|
PgFdwModifyState *fmstate = resultRelInfo->ri_FdwState ?
|
|
|
|
(PgFdwModifyState *) resultRelInfo->ri_FdwState :
|
|
|
|
NULL;
|
2021-01-20 23:05:46 +01:00
|
|
|
|
|
|
|
/* should be called only once */
|
|
|
|
Assert(resultRelInfo->ri_BatchSize == 0);
|
|
|
|
|
2021-02-18 00:02:00 +01:00
|
|
|
/*
|
|
|
|
* Should never get called when the insert is being performed as part of a
|
|
|
|
* row movement operation.
|
|
|
|
*/
|
|
|
|
Assert(fmstate == NULL || fmstate->aux_fmstate == NULL);
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
/*
|
2021-06-08 20:22:18 +02:00
|
|
|
* In EXPLAIN without ANALYZE, ri_FdwState is NULL, so we have to lookup
|
2021-01-20 23:05:46 +01:00
|
|
|
* the option directly in server/table options. Otherwise just use the
|
|
|
|
* value we determined earlier.
|
|
|
|
*/
|
2021-02-18 00:02:00 +01:00
|
|
|
if (fmstate)
|
|
|
|
batch_size = fmstate->batch_size;
|
2021-01-20 23:05:46 +01:00
|
|
|
else
|
|
|
|
batch_size = get_batch_size_option(resultRelInfo->ri_RelationDesc);
|
|
|
|
|
|
|
|
/* Disable batching when we have to use RETURNING. */
|
|
|
|
if (resultRelInfo->ri_projectReturning != NULL ||
|
|
|
|
(resultRelInfo->ri_TrigDesc &&
|
|
|
|
resultRelInfo->ri_TrigDesc->trig_insert_after_row))
|
|
|
|
return 1;
|
|
|
|
|
2021-06-08 20:22:18 +02:00
|
|
|
/*
|
|
|
|
* Otherwise use the batch size specified for server/table. The number of
|
|
|
|
* parameters in a batch is limited to 65535 (uint16), so make sure we
|
|
|
|
* don't exceed this limit by using the maximum batch_size possible.
|
|
|
|
*/
|
|
|
|
if (fmstate && fmstate->p_nums > 0)
|
|
|
|
batch_size = Min(batch_size, PQ_QUERY_PARAM_MAX_LIMIT / fmstate->p_nums);
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
return batch_size;
|
|
|
|
}
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/*
|
|
|
|
* postgresExecForeignUpdate
|
|
|
|
* Update one row in a foreign table
|
|
|
|
*/
|
|
|
|
static TupleTableSlot *
|
|
|
|
postgresExecForeignUpdate(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
TupleTableSlot *slot,
|
|
|
|
TupleTableSlot *planSlot)
|
|
|
|
{
|
2021-01-20 23:05:46 +01:00
|
|
|
TupleTableSlot **rslot;
|
|
|
|
int numSlots = 1;
|
|
|
|
|
|
|
|
rslot = execute_foreign_modify(estate, resultRelInfo, CMD_UPDATE,
|
|
|
|
&slot, &planSlot, &numSlots);
|
|
|
|
|
|
|
|
return rslot ? rslot[0] : NULL;
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresExecForeignDelete
|
|
|
|
* Delete one row from a foreign table
|
|
|
|
*/
|
|
|
|
static TupleTableSlot *
|
|
|
|
postgresExecForeignDelete(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
TupleTableSlot *slot,
|
|
|
|
TupleTableSlot *planSlot)
|
|
|
|
{
|
2021-01-20 23:05:46 +01:00
|
|
|
TupleTableSlot **rslot;
|
|
|
|
int numSlots = 1;
|
|
|
|
|
|
|
|
rslot = execute_foreign_modify(estate, resultRelInfo, CMD_DELETE,
|
|
|
|
&slot, &planSlot, &numSlots);
|
|
|
|
|
|
|
|
return rslot ? rslot[0] : NULL;
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresEndForeignModify
|
|
|
|
* Finish an insert/update/delete operation on a foreign table
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresEndForeignModify(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo)
|
|
|
|
{
|
|
|
|
PgFdwModifyState *fmstate = (PgFdwModifyState *) resultRelInfo->ri_FdwState;
|
|
|
|
|
|
|
|
/* If fmstate is NULL, we are in EXPLAIN; nothing to do */
|
|
|
|
if (fmstate == NULL)
|
|
|
|
return;
|
|
|
|
|
2018-04-06 17:29:43 +02:00
|
|
|
/* Destroy the execution state */
|
|
|
|
finish_foreign_modify(fmstate);
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
/*
|
|
|
|
* postgresBeginForeignInsert
|
|
|
|
* Begin an insert operation on a foreign table
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresBeginForeignInsert(ModifyTableState *mtstate,
|
|
|
|
ResultRelInfo *resultRelInfo)
|
|
|
|
{
|
|
|
|
PgFdwModifyState *fmstate;
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
ModifyTable *plan = castNode(ModifyTable, mtstate->ps.plan);
|
|
|
|
EState *estate = mtstate->ps.state;
|
Fix permission checks on constraint violation errors on partitions.
If a cross-partition UPDATE violates a constraint on the target partition,
and the columns in the new partition are in different physical order than
in the parent, the error message can reveal columns that the user does not
have SELECT permission on. A similar bug was fixed earlier in commit
804b6b6db4.
The cause of the bug is that the callers of the
ExecBuildSlotValueDescription() function got confused when constructing
the list of modified columns. If the tuple was routed from a parent, we
converted the tuple to the parent's format, but the list of modified
columns was grabbed directly from the child's RTE entry.
ExecUpdateLockMode() had a similar issue. That lead to confusion on which
columns are key columns, leading to wrong tuple lock being taken on tables
referenced by foreign keys, when a row is updated with INSERT ON CONFLICT
UPDATE. A new isolation test is added for that corner case.
With this patch, the ri_RangeTableIndex field is no longer set for
partitions that don't have an entry in the range table. Previously, it was
set to the RTE entry of the parent relation, but that was confusing.
NOTE: This modifies the ResultRelInfo struct, replacing the
ri_PartitionRoot field with ri_RootResultRelInfo. That's a bit risky to
backpatch, because it breaks any extensions accessing the field. The
change that ri_RangeTableIndex is not set for partitions could potentially
break extensions, too. The ResultRelInfos are visible to FDWs at least,
and this patch required small changes to postgres_fdw. Nevertheless, this
seem like the least bad option. I don't think these fields widely used in
extensions; I don't think there are FDWs out there that uses the FDW
"direct update" API, other than postgres_fdw. If there is, you will get a
compilation error, so hopefully it is caught quickly.
Backpatch to 11, where support for both cross-partition UPDATEs, and unique
indexes on partitioned tables, were added.
Reviewed-by: Amit Langote
Security: CVE-2021-3393
2021-02-08 10:01:51 +01:00
|
|
|
Index resultRelation;
|
2018-04-07 01:16:11 +02:00
|
|
|
Relation rel = resultRelInfo->ri_RelationDesc;
|
|
|
|
RangeTblEntry *rte;
|
|
|
|
TupleDesc tupdesc = RelationGetDescr(rel);
|
|
|
|
int attnum;
|
2021-01-20 23:05:46 +01:00
|
|
|
int values_end_len;
|
2018-04-07 01:16:11 +02:00
|
|
|
StringInfoData sql;
|
|
|
|
List *targetAttrs = NIL;
|
|
|
|
List *retrieved_attrs = NIL;
|
|
|
|
bool doNothing = false;
|
|
|
|
|
2019-04-24 11:31:50 +02:00
|
|
|
/*
|
|
|
|
* If the foreign table we are about to insert routed rows into is also an
|
|
|
|
* UPDATE subplan result rel that will be updated later, proceeding with
|
|
|
|
* the INSERT will result in the later UPDATE incorrectly modifying those
|
|
|
|
* routed rows, so prevent the INSERT --- it would be nice if we could
|
|
|
|
* handle this case; but for now, throw an error for safety.
|
|
|
|
*/
|
|
|
|
if (plan && plan->operation == CMD_UPDATE &&
|
|
|
|
(resultRelInfo->ri_usesFdwDirectModify ||
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
resultRelInfo->ri_FdwState))
|
2019-04-24 11:31:50 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("cannot route tuples into foreign table to be updated \"%s\"",
|
|
|
|
RelationGetRelationName(rel))));
|
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
initStringInfo(&sql);
|
|
|
|
|
|
|
|
/* We transmit all columns that are defined in the foreign table. */
|
|
|
|
for (attnum = 1; attnum <= tupdesc->natts; attnum++)
|
|
|
|
{
|
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1);
|
|
|
|
|
|
|
|
if (!attr->attisdropped)
|
|
|
|
targetAttrs = lappend_int(targetAttrs, attnum);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Check if we add the ON CONFLICT clause to the remote query. */
|
|
|
|
if (plan)
|
|
|
|
{
|
2018-05-03 02:26:02 +02:00
|
|
|
OnConflictAction onConflictAction = plan->onConflictAction;
|
2018-04-07 01:16:11 +02:00
|
|
|
|
|
|
|
/* We only support DO NOTHING without an inference specification. */
|
|
|
|
if (onConflictAction == ONCONFLICT_NOTHING)
|
|
|
|
doNothing = true;
|
|
|
|
else if (onConflictAction != ONCONFLICT_NONE)
|
|
|
|
elog(ERROR, "unexpected ON CONFLICT specification: %d",
|
|
|
|
(int) onConflictAction);
|
|
|
|
}
|
|
|
|
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
/*
|
Fix permission checks on constraint violation errors on partitions.
If a cross-partition UPDATE violates a constraint on the target partition,
and the columns in the new partition are in different physical order than
in the parent, the error message can reveal columns that the user does not
have SELECT permission on. A similar bug was fixed earlier in commit
804b6b6db4.
The cause of the bug is that the callers of the
ExecBuildSlotValueDescription() function got confused when constructing
the list of modified columns. If the tuple was routed from a parent, we
converted the tuple to the parent's format, but the list of modified
columns was grabbed directly from the child's RTE entry.
ExecUpdateLockMode() had a similar issue. That lead to confusion on which
columns are key columns, leading to wrong tuple lock being taken on tables
referenced by foreign keys, when a row is updated with INSERT ON CONFLICT
UPDATE. A new isolation test is added for that corner case.
With this patch, the ri_RangeTableIndex field is no longer set for
partitions that don't have an entry in the range table. Previously, it was
set to the RTE entry of the parent relation, but that was confusing.
NOTE: This modifies the ResultRelInfo struct, replacing the
ri_PartitionRoot field with ri_RootResultRelInfo. That's a bit risky to
backpatch, because it breaks any extensions accessing the field. The
change that ri_RangeTableIndex is not set for partitions could potentially
break extensions, too. The ResultRelInfos are visible to FDWs at least,
and this patch required small changes to postgres_fdw. Nevertheless, this
seem like the least bad option. I don't think these fields widely used in
extensions; I don't think there are FDWs out there that uses the FDW
"direct update" API, other than postgres_fdw. If there is, you will get a
compilation error, so hopefully it is caught quickly.
Backpatch to 11, where support for both cross-partition UPDATEs, and unique
indexes on partitioned tables, were added.
Reviewed-by: Amit Langote
Security: CVE-2021-3393
2021-02-08 10:01:51 +01:00
|
|
|
* If the foreign table is a partition that doesn't have a corresponding
|
|
|
|
* RTE entry, we need to create a new RTE describing the foreign table for
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
* use by deparseInsertSql and create_foreign_modify() below, after first
|
|
|
|
* copying the parent's RTE and modifying some fields to describe the
|
|
|
|
* foreign partition to work on. However, if this is invoked by UPDATE,
|
|
|
|
* the existing RTE may already correspond to this partition if it is one
|
|
|
|
* of the UPDATE subplan target rels; in that case, we can just use the
|
|
|
|
* existing RTE as-is.
|
|
|
|
*/
|
Fix permission checks on constraint violation errors on partitions.
If a cross-partition UPDATE violates a constraint on the target partition,
and the columns in the new partition are in different physical order than
in the parent, the error message can reveal columns that the user does not
have SELECT permission on. A similar bug was fixed earlier in commit
804b6b6db4.
The cause of the bug is that the callers of the
ExecBuildSlotValueDescription() function got confused when constructing
the list of modified columns. If the tuple was routed from a parent, we
converted the tuple to the parent's format, but the list of modified
columns was grabbed directly from the child's RTE entry.
ExecUpdateLockMode() had a similar issue. That lead to confusion on which
columns are key columns, leading to wrong tuple lock being taken on tables
referenced by foreign keys, when a row is updated with INSERT ON CONFLICT
UPDATE. A new isolation test is added for that corner case.
With this patch, the ri_RangeTableIndex field is no longer set for
partitions that don't have an entry in the range table. Previously, it was
set to the RTE entry of the parent relation, but that was confusing.
NOTE: This modifies the ResultRelInfo struct, replacing the
ri_PartitionRoot field with ri_RootResultRelInfo. That's a bit risky to
backpatch, because it breaks any extensions accessing the field. The
change that ri_RangeTableIndex is not set for partitions could potentially
break extensions, too. The ResultRelInfos are visible to FDWs at least,
and this patch required small changes to postgres_fdw. Nevertheless, this
seem like the least bad option. I don't think these fields widely used in
extensions; I don't think there are FDWs out there that uses the FDW
"direct update" API, other than postgres_fdw. If there is, you will get a
compilation error, so hopefully it is caught quickly.
Backpatch to 11, where support for both cross-partition UPDATEs, and unique
indexes on partitioned tables, were added.
Reviewed-by: Amit Langote
Security: CVE-2021-3393
2021-02-08 10:01:51 +01:00
|
|
|
if (resultRelInfo->ri_RangeTableIndex == 0)
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
{
|
Fix permission checks on constraint violation errors on partitions.
If a cross-partition UPDATE violates a constraint on the target partition,
and the columns in the new partition are in different physical order than
in the parent, the error message can reveal columns that the user does not
have SELECT permission on. A similar bug was fixed earlier in commit
804b6b6db4.
The cause of the bug is that the callers of the
ExecBuildSlotValueDescription() function got confused when constructing
the list of modified columns. If the tuple was routed from a parent, we
converted the tuple to the parent's format, but the list of modified
columns was grabbed directly from the child's RTE entry.
ExecUpdateLockMode() had a similar issue. That lead to confusion on which
columns are key columns, leading to wrong tuple lock being taken on tables
referenced by foreign keys, when a row is updated with INSERT ON CONFLICT
UPDATE. A new isolation test is added for that corner case.
With this patch, the ri_RangeTableIndex field is no longer set for
partitions that don't have an entry in the range table. Previously, it was
set to the RTE entry of the parent relation, but that was confusing.
NOTE: This modifies the ResultRelInfo struct, replacing the
ri_PartitionRoot field with ri_RootResultRelInfo. That's a bit risky to
backpatch, because it breaks any extensions accessing the field. The
change that ri_RangeTableIndex is not set for partitions could potentially
break extensions, too. The ResultRelInfos are visible to FDWs at least,
and this patch required small changes to postgres_fdw. Nevertheless, this
seem like the least bad option. I don't think these fields widely used in
extensions; I don't think there are FDWs out there that uses the FDW
"direct update" API, other than postgres_fdw. If there is, you will get a
compilation error, so hopefully it is caught quickly.
Backpatch to 11, where support for both cross-partition UPDATEs, and unique
indexes on partitioned tables, were added.
Reviewed-by: Amit Langote
Security: CVE-2021-3393
2021-02-08 10:01:51 +01:00
|
|
|
ResultRelInfo *rootResultRelInfo = resultRelInfo->ri_RootResultRelInfo;
|
|
|
|
|
|
|
|
rte = exec_rt_fetch(rootResultRelInfo->ri_RangeTableIndex, estate);
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
rte = copyObject(rte);
|
|
|
|
rte->relid = RelationGetRelid(rel);
|
|
|
|
rte->relkind = RELKIND_FOREIGN_TABLE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For UPDATE, we must use the RT index of the first subplan target
|
|
|
|
* rel's RTE, because the core code would have built expressions for
|
|
|
|
* the partition, such as RETURNING, using that RT index as varno of
|
|
|
|
* Vars contained in those expressions.
|
|
|
|
*/
|
|
|
|
if (plan && plan->operation == CMD_UPDATE &&
|
Fix permission checks on constraint violation errors on partitions.
If a cross-partition UPDATE violates a constraint on the target partition,
and the columns in the new partition are in different physical order than
in the parent, the error message can reveal columns that the user does not
have SELECT permission on. A similar bug was fixed earlier in commit
804b6b6db4.
The cause of the bug is that the callers of the
ExecBuildSlotValueDescription() function got confused when constructing
the list of modified columns. If the tuple was routed from a parent, we
converted the tuple to the parent's format, but the list of modified
columns was grabbed directly from the child's RTE entry.
ExecUpdateLockMode() had a similar issue. That lead to confusion on which
columns are key columns, leading to wrong tuple lock being taken on tables
referenced by foreign keys, when a row is updated with INSERT ON CONFLICT
UPDATE. A new isolation test is added for that corner case.
With this patch, the ri_RangeTableIndex field is no longer set for
partitions that don't have an entry in the range table. Previously, it was
set to the RTE entry of the parent relation, but that was confusing.
NOTE: This modifies the ResultRelInfo struct, replacing the
ri_PartitionRoot field with ri_RootResultRelInfo. That's a bit risky to
backpatch, because it breaks any extensions accessing the field. The
change that ri_RangeTableIndex is not set for partitions could potentially
break extensions, too. The ResultRelInfos are visible to FDWs at least,
and this patch required small changes to postgres_fdw. Nevertheless, this
seem like the least bad option. I don't think these fields widely used in
extensions; I don't think there are FDWs out there that uses the FDW
"direct update" API, other than postgres_fdw. If there is, you will get a
compilation error, so hopefully it is caught quickly.
Backpatch to 11, where support for both cross-partition UPDATEs, and unique
indexes on partitioned tables, were added.
Reviewed-by: Amit Langote
Security: CVE-2021-3393
2021-02-08 10:01:51 +01:00
|
|
|
rootResultRelInfo->ri_RangeTableIndex == plan->rootRelation)
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
resultRelation = mtstate->resultRelInfo[0].ri_RangeTableIndex;
|
Fix permission checks on constraint violation errors on partitions.
If a cross-partition UPDATE violates a constraint on the target partition,
and the columns in the new partition are in different physical order than
in the parent, the error message can reveal columns that the user does not
have SELECT permission on. A similar bug was fixed earlier in commit
804b6b6db4.
The cause of the bug is that the callers of the
ExecBuildSlotValueDescription() function got confused when constructing
the list of modified columns. If the tuple was routed from a parent, we
converted the tuple to the parent's format, but the list of modified
columns was grabbed directly from the child's RTE entry.
ExecUpdateLockMode() had a similar issue. That lead to confusion on which
columns are key columns, leading to wrong tuple lock being taken on tables
referenced by foreign keys, when a row is updated with INSERT ON CONFLICT
UPDATE. A new isolation test is added for that corner case.
With this patch, the ri_RangeTableIndex field is no longer set for
partitions that don't have an entry in the range table. Previously, it was
set to the RTE entry of the parent relation, but that was confusing.
NOTE: This modifies the ResultRelInfo struct, replacing the
ri_PartitionRoot field with ri_RootResultRelInfo. That's a bit risky to
backpatch, because it breaks any extensions accessing the field. The
change that ri_RangeTableIndex is not set for partitions could potentially
break extensions, too. The ResultRelInfos are visible to FDWs at least,
and this patch required small changes to postgres_fdw. Nevertheless, this
seem like the least bad option. I don't think these fields widely used in
extensions; I don't think there are FDWs out there that uses the FDW
"direct update" API, other than postgres_fdw. If there is, you will get a
compilation error, so hopefully it is caught quickly.
Backpatch to 11, where support for both cross-partition UPDATEs, and unique
indexes on partitioned tables, were added.
Reviewed-by: Amit Langote
Security: CVE-2021-3393
2021-02-08 10:01:51 +01:00
|
|
|
else
|
|
|
|
resultRelation = rootResultRelInfo->ri_RangeTableIndex;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
resultRelation = resultRelInfo->ri_RangeTableIndex;
|
|
|
|
rte = exec_rt_fetch(resultRelation, estate);
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
}
|
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
/* Construct the SQL command string. */
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
deparseInsertSql(&sql, rte, resultRelation, rel, targetAttrs, doNothing,
|
Fix WITH CHECK OPTION on views referencing postgres_fdw tables.
If a view references a foreign table, and the foreign table has a
BEFORE INSERT trigger, then it's possible for a tuple inserted or
updated through the view to be changed such that it violates the
view's WITH CHECK OPTION constraint.
Before this commit, postgres_fdw handled this case inconsistently. A
RETURNING clause on the INSERT or UPDATE statement targeting the view
would cause the finally-inserted tuple to be read back, and the WITH
CHECK OPTION violation would throw an error. But without a RETURNING
clause, postgres_fdw would not read the final tuple back, and WITH
CHECK OPTION would not throw an error for the violation (or may throw
an error when there is no real violation). AFTER ROW triggers on the
foreign table had a similar effect as a RETURNING clause on the INSERT
or UPDATE statement.
To fix, this commit retrieves the attributes needed to enforce the
WITH CHECK OPTION constraint along with the attributes needed for the
RETURNING clause (if any) from the remote side. Thus, the WITH CHECK
OPTION constraint is always evaluated against the final tuple after
any triggers on the remote side.
This fix may be considered inconsistent with CHECK constraints
declared on foreign tables, which are not enforced locally at all
(because the constraint is on a remote object). The discussion
concluded that this difference is reasonable, because the WITH CHECK
OPTION is a constraint on the local view (not any remote object);
therefore it only makes sense to enforce its WITH CHECK OPTION
constraint locally.
Author: Etsuro Fujita
Reviewed-by: Arthur Zakirov, Stephen Frost
Discussion: https://www.postgresql.org/message-id/7eb58fab-fd3b-781b-ac33-f7cfec96021f%40lab.ntt.co.jp
2018-07-08 09:14:51 +02:00
|
|
|
resultRelInfo->ri_WithCheckOptions,
|
|
|
|
resultRelInfo->ri_returningList,
|
2021-01-20 23:05:46 +01:00
|
|
|
&retrieved_attrs, &values_end_len);
|
2018-04-07 01:16:11 +02:00
|
|
|
|
|
|
|
/* Construct an execution state. */
|
|
|
|
fmstate = create_foreign_modify(mtstate->ps.state,
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
rte,
|
2018-04-07 01:16:11 +02:00
|
|
|
resultRelInfo,
|
|
|
|
CMD_INSERT,
|
|
|
|
NULL,
|
|
|
|
sql.data,
|
|
|
|
targetAttrs,
|
2021-01-20 23:05:46 +01:00
|
|
|
values_end_len,
|
2018-04-07 01:16:11 +02:00
|
|
|
retrieved_attrs != NIL,
|
|
|
|
retrieved_attrs);
|
|
|
|
|
2019-04-24 11:31:50 +02:00
|
|
|
/*
|
|
|
|
* If the given resultRelInfo already has PgFdwModifyState set, it means
|
|
|
|
* the foreign table is an UPDATE subplan result rel; in which case, store
|
|
|
|
* the resulting state into the aux_fmstate of the PgFdwModifyState.
|
|
|
|
*/
|
|
|
|
if (resultRelInfo->ri_FdwState)
|
|
|
|
{
|
|
|
|
Assert(plan && plan->operation == CMD_UPDATE);
|
|
|
|
Assert(resultRelInfo->ri_usesFdwDirectModify == false);
|
|
|
|
((PgFdwModifyState *) resultRelInfo->ri_FdwState)->aux_fmstate = fmstate;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
resultRelInfo->ri_FdwState = fmstate;
|
2018-04-07 01:16:11 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresEndForeignInsert
|
|
|
|
* Finish an insert operation on a foreign table
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresEndForeignInsert(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo)
|
|
|
|
{
|
|
|
|
PgFdwModifyState *fmstate = (PgFdwModifyState *) resultRelInfo->ri_FdwState;
|
|
|
|
|
|
|
|
Assert(fmstate != NULL);
|
|
|
|
|
2019-04-24 11:31:50 +02:00
|
|
|
/*
|
|
|
|
* If the fmstate has aux_fmstate set, get the aux_fmstate (see
|
|
|
|
* postgresBeginForeignInsert())
|
|
|
|
*/
|
|
|
|
if (fmstate->aux_fmstate)
|
|
|
|
fmstate = fmstate->aux_fmstate;
|
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
/* Destroy the execution state */
|
|
|
|
finish_foreign_modify(fmstate);
|
|
|
|
}
|
|
|
|
|
2013-06-12 23:52:54 +02:00
|
|
|
/*
|
|
|
|
* postgresIsForeignRelUpdatable
|
|
|
|
* Determine whether a foreign table supports INSERT, UPDATE and/or
|
|
|
|
* DELETE.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
postgresIsForeignRelUpdatable(Relation rel)
|
|
|
|
{
|
|
|
|
bool updatable;
|
|
|
|
ForeignTable *table;
|
|
|
|
ForeignServer *server;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* By default, all postgres_fdw foreign tables are assumed updatable. This
|
|
|
|
* can be overridden by a per-server setting, which in turn can be
|
|
|
|
* overridden by a per-table setting.
|
|
|
|
*/
|
|
|
|
updatable = true;
|
|
|
|
|
|
|
|
table = GetForeignTable(RelationGetRelid(rel));
|
|
|
|
server = GetForeignServer(table->serverid);
|
|
|
|
|
|
|
|
foreach(lc, server->options)
|
|
|
|
{
|
|
|
|
DefElem *def = (DefElem *) lfirst(lc);
|
|
|
|
|
|
|
|
if (strcmp(def->defname, "updatable") == 0)
|
|
|
|
updatable = defGetBoolean(def);
|
|
|
|
}
|
|
|
|
foreach(lc, table->options)
|
|
|
|
{
|
|
|
|
DefElem *def = (DefElem *) lfirst(lc);
|
|
|
|
|
|
|
|
if (strcmp(def->defname, "updatable") == 0)
|
|
|
|
updatable = defGetBoolean(def);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Currently "updatable" means support for INSERT, UPDATE and DELETE.
|
|
|
|
*/
|
|
|
|
return updatable ?
|
|
|
|
(1 << CMD_INSERT) | (1 << CMD_UPDATE) | (1 << CMD_DELETE) : 0;
|
|
|
|
}
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
|
|
|
* postgresRecheckForeignScan
|
|
|
|
* Execute a local join execution plan for a foreign join
|
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
postgresRecheckForeignScan(ForeignScanState *node, TupleTableSlot *slot)
|
|
|
|
{
|
|
|
|
Index scanrelid = ((Scan *) node->ss.ps.plan)->scanrelid;
|
|
|
|
PlanState *outerPlan = outerPlanState(node);
|
|
|
|
TupleTableSlot *result;
|
|
|
|
|
|
|
|
/* For base foreign relations, it suffices to set fdw_recheck_quals */
|
|
|
|
if (scanrelid > 0)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
Assert(outerPlan != NULL);
|
|
|
|
|
|
|
|
/* Execute a local join execution plan */
|
|
|
|
result = ExecProcNode(outerPlan);
|
|
|
|
if (TupIsNull(result))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Store result in the given slot */
|
|
|
|
ExecCopySlot(slot, result);
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
/*
|
|
|
|
* find_modifytable_subplan
|
|
|
|
* Helper routine for postgresPlanDirectModify to find the
|
|
|
|
* ModifyTable subplan node that scans the specified RTI.
|
|
|
|
*
|
|
|
|
* Returns NULL if the subplan couldn't be identified. That's not a fatal
|
|
|
|
* error condition, we just abandon trying to do the update directly.
|
|
|
|
*/
|
|
|
|
static ForeignScan *
|
|
|
|
find_modifytable_subplan(PlannerInfo *root,
|
|
|
|
ModifyTable *plan,
|
|
|
|
Index rtindex,
|
|
|
|
int subplan_index)
|
|
|
|
{
|
|
|
|
Plan *subplan = outerPlan(plan);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The cases we support are (1) the desired ForeignScan is the immediate
|
|
|
|
* child of ModifyTable, or (2) it is the subplan_index'th child of an
|
|
|
|
* Append node that is the immediate child of ModifyTable. There is no
|
|
|
|
* point in looking further down, as that would mean that local joins are
|
|
|
|
* involved, so we can't do the update directly.
|
|
|
|
*
|
|
|
|
* There could be a Result atop the Append too, acting to compute the
|
|
|
|
* UPDATE targetlist values. We ignore that here; the tlist will be
|
|
|
|
* checked by our caller.
|
|
|
|
*
|
|
|
|
* In principle we could examine all the children of the Append, but it's
|
|
|
|
* currently unlikely that the core planner would generate such a plan
|
|
|
|
* with the children out-of-order. Moreover, such a search risks costing
|
|
|
|
* O(N^2) time when there are a lot of children.
|
|
|
|
*/
|
|
|
|
if (IsA(subplan, Append))
|
|
|
|
{
|
|
|
|
Append *appendplan = (Append *) subplan;
|
|
|
|
|
|
|
|
if (subplan_index < list_length(appendplan->appendplans))
|
|
|
|
subplan = (Plan *) list_nth(appendplan->appendplans, subplan_index);
|
|
|
|
}
|
|
|
|
else if (IsA(subplan, Result) && IsA(outerPlan(subplan), Append))
|
|
|
|
{
|
|
|
|
Append *appendplan = (Append *) outerPlan(subplan);
|
|
|
|
|
|
|
|
if (subplan_index < list_length(appendplan->appendplans))
|
|
|
|
subplan = (Plan *) list_nth(appendplan->appendplans, subplan_index);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Now, have we got a ForeignScan on the desired rel? */
|
|
|
|
if (IsA(subplan, ForeignScan))
|
|
|
|
{
|
|
|
|
ForeignScan *fscan = (ForeignScan *) subplan;
|
|
|
|
|
|
|
|
if (bms_is_member(rtindex, fscan->fs_relids))
|
|
|
|
return fscan;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* postgresPlanDirectModify
|
|
|
|
* Consider a direct foreign table modification
|
|
|
|
*
|
2016-03-21 16:59:49 +01:00
|
|
|
* Decide whether it is safe to modify a foreign table directly, and if so,
|
2016-03-18 18:48:58 +01:00
|
|
|
* rewrite subplan accordingly.
|
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
postgresPlanDirectModify(PlannerInfo *root,
|
|
|
|
ModifyTable *plan,
|
|
|
|
Index resultRelation,
|
|
|
|
int subplan_index)
|
|
|
|
{
|
|
|
|
CmdType operation = plan->operation;
|
2017-04-11 19:53:13 +02:00
|
|
|
RelOptInfo *foreignrel;
|
|
|
|
RangeTblEntry *rte;
|
|
|
|
PgFdwRelationInfo *fpinfo;
|
2016-03-18 18:48:58 +01:00
|
|
|
Relation rel;
|
|
|
|
StringInfoData sql;
|
|
|
|
ForeignScan *fscan;
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
List *processed_tlist = NIL;
|
2016-03-18 18:48:58 +01:00
|
|
|
List *targetAttrs = NIL;
|
2017-04-11 19:53:13 +02:00
|
|
|
List *remote_exprs;
|
2016-03-18 18:48:58 +01:00
|
|
|
List *params_list = NIL;
|
|
|
|
List *returningList = NIL;
|
|
|
|
List *retrieved_attrs = NIL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Decide whether it is safe to modify a foreign table directly.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The table modification must be an UPDATE or DELETE.
|
|
|
|
*/
|
|
|
|
if (operation != CMD_UPDATE && operation != CMD_DELETE)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
* Try to locate the ForeignScan subplan that's scanning resultRelation.
|
2016-03-18 18:48:58 +01:00
|
|
|
*/
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
fscan = find_modifytable_subplan(root, plan, resultRelation, subplan_index);
|
|
|
|
if (!fscan)
|
2016-03-18 18:48:58 +01:00
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It's unsafe to modify a foreign table directly if there are any quals
|
|
|
|
* that should be evaluated locally.
|
|
|
|
*/
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
if (fscan->scan.plan.qual != NIL)
|
2016-03-18 18:48:58 +01:00
|
|
|
return false;
|
|
|
|
|
2017-04-11 19:53:13 +02:00
|
|
|
/* Safe to fetch data about the target foreign rel */
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
if (fscan->scan.scanrelid == 0)
|
|
|
|
{
|
|
|
|
foreignrel = find_join_rel(root, fscan->fs_relids);
|
|
|
|
/* We should have a rel for this foreign join. */
|
|
|
|
Assert(foreignrel);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
foreignrel = root->simple_rel_array[resultRelation];
|
2017-04-11 19:53:13 +02:00
|
|
|
rte = root->simple_rte_array[resultRelation];
|
|
|
|
fpinfo = (PgFdwRelationInfo *) foreignrel->fdw_private;
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* It's unsafe to update a foreign table directly, if any expressions to
|
|
|
|
* assign to the target columns are unsafe to evaluate remotely.
|
|
|
|
*/
|
|
|
|
if (operation == CMD_UPDATE)
|
|
|
|
{
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
ListCell *lc,
|
|
|
|
*lc2;
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/*
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
* The expressions of concern are the first N columns of the processed
|
|
|
|
* targetlist, where N is the length of the rel's update_colnos.
|
2016-03-18 18:48:58 +01:00
|
|
|
*/
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
get_translated_update_targetlist(root, resultRelation,
|
|
|
|
&processed_tlist, &targetAttrs);
|
|
|
|
forboth(lc, processed_tlist, lc2, targetAttrs)
|
2016-03-18 18:48:58 +01:00
|
|
|
{
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
TargetEntry *tle = lfirst_node(TargetEntry, lc);
|
|
|
|
AttrNumber attno = lfirst_int(lc2);
|
|
|
|
|
|
|
|
/* update's new-value expressions shouldn't be resjunk */
|
|
|
|
Assert(!tle->resjunk);
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
if (attno <= InvalidAttrNumber) /* shouldn't happen */
|
|
|
|
elog(ERROR, "system-column update is not supported");
|
|
|
|
|
2017-04-11 19:53:13 +02:00
|
|
|
if (!is_foreign_expr(root, foreignrel, (Expr *) tle->expr))
|
2016-03-18 18:48:58 +01:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ok, rewrite subplan so as to modify the foreign table directly.
|
|
|
|
*/
|
|
|
|
initStringInfo(&sql);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Core code already has some lock on each rel being planned, so we can
|
|
|
|
* use NoLock here.
|
|
|
|
*/
|
2019-01-21 19:32:19 +01:00
|
|
|
rel = table_open(rte->relid, NoLock);
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/*
|
2017-04-11 19:53:13 +02:00
|
|
|
* Recall the qual clauses that must be evaluated remotely. (These are
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
* bare clauses not RestrictInfos, but deparse.c's appendConditions()
|
|
|
|
* doesn't care.)
|
2016-03-18 18:48:58 +01:00
|
|
|
*/
|
2017-04-11 19:53:13 +02:00
|
|
|
remote_exprs = fpinfo->final_remote_exprs;
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract the relevant RETURNING list if any.
|
|
|
|
*/
|
|
|
|
if (plan->returningLists)
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
{
|
2016-03-18 18:48:58 +01:00
|
|
|
returningList = (List *) list_nth(plan->returningLists, subplan_index);
|
|
|
|
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
/*
|
|
|
|
* When performing an UPDATE/DELETE .. RETURNING on a join directly,
|
|
|
|
* we fetch from the foreign server any Vars specified in RETURNING
|
|
|
|
* that refer not only to the target relation but to non-target
|
|
|
|
* relations. So we'll deparse them into the RETURNING clause of the
|
|
|
|
* remote query; use a targetlist consisting of them instead, which
|
|
|
|
* will be adjusted to be new fdw_scan_tlist of the foreign-scan plan
|
|
|
|
* node below.
|
|
|
|
*/
|
|
|
|
if (fscan->scan.scanrelid == 0)
|
|
|
|
returningList = build_remote_returning(resultRelation, rel,
|
|
|
|
returningList);
|
|
|
|
}
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* Construct the SQL command string.
|
|
|
|
*/
|
|
|
|
switch (operation)
|
|
|
|
{
|
|
|
|
case CMD_UPDATE:
|
|
|
|
deparseDirectUpdateSql(&sql, root, resultRelation, rel,
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
foreignrel,
|
Rework planning and execution of UPDATE and DELETE.
This patch makes two closely related sets of changes:
1. For UPDATE, the subplan of the ModifyTable node now only delivers
the new values of the changed columns (i.e., the expressions computed
in the query's SET clause) plus row identity information such as CTID.
ModifyTable must re-fetch the original tuple to merge in the old
values of any unchanged columns. The core advantage of this is that
the changed columns are uniform across all tables of an inherited or
partitioned target relation, whereas the other columns might not be.
A secondary advantage, when the UPDATE involves joins, is that less
data needs to pass through the plan tree. The disadvantage of course
is an extra fetch of each tuple to be updated. However, that seems to
be very nearly free in context; even worst-case tests don't show it to
add more than a couple percent to the total query cost. At some point
it might be interesting to combine the re-fetch with the tuple access
that ModifyTable must do anyway to mark the old tuple dead; but that
would require a good deal of refactoring and it seems it wouldn't buy
all that much, so this patch doesn't attempt it.
2. For inherited UPDATE/DELETE, instead of generating a separate
subplan for each target relation, we now generate a single subplan
that is just exactly like a SELECT's plan, then stick ModifyTable
on top of that. To let ModifyTable know which target relation a
given incoming row refers to, a tableoid junk column is added to
the row identity information. This gets rid of the horrid hack
that was inheritance_planner(), eliminating O(N^2) planning cost
and memory consumption in cases where there were many unprunable
target relations.
Point 2 of course requires point 1, so that there is a uniform
definition of the non-junk columns to be returned by the subplan.
We can't insist on uniform definition of the row identity junk
columns however, if we want to keep the ability to have both
plain and foreign tables in a partitioning hierarchy. Since
it wouldn't scale very far to have every child table have its
own row identity column, this patch includes provisions to merge
similar row identity columns into one column of the subplan result.
In particular, we can merge the whole-row Vars typically used as
row identity by FDWs into one column by pretending they are type
RECORD. (It's still okay for the actual composite Datums to be
labeled with the table's rowtype OID, though.)
There is more that can be done to file down residual inefficiencies
in this patch, but it seems to be committable now.
FDW authors should note several API changes:
* The argument list for AddForeignUpdateTargets() has changed, and so
has the method it must use for adding junk columns to the query. Call
add_row_identity_var() instead of manipulating the parse tree directly.
You might want to reconsider exactly what you're adding, too.
* PlanDirectModify() must now work a little harder to find the
ForeignScan plan node; if the foreign table is part of a partitioning
hierarchy then the ForeignScan might not be the direct child of
ModifyTable. See postgres_fdw for sample code.
* To check whether a relation is a target relation, it's no
longer sufficient to compare its relid to root->parse->resultRelation.
Instead, check it against all_result_relids or leaf_result_relids,
as appropriate.
Amit Langote and Tom Lane
Discussion: https://postgr.es/m/CA+HiwqHpHdqdDn48yCEhynnniahH78rwcrv1rEX65-fsZGBOLQ@mail.gmail.com
2021-03-31 17:52:34 +02:00
|
|
|
processed_tlist,
|
2016-03-18 18:48:58 +01:00
|
|
|
targetAttrs,
|
2017-04-11 19:53:13 +02:00
|
|
|
remote_exprs, ¶ms_list,
|
2016-03-18 18:48:58 +01:00
|
|
|
returningList, &retrieved_attrs);
|
|
|
|
break;
|
|
|
|
case CMD_DELETE:
|
|
|
|
deparseDirectDeleteSql(&sql, root, resultRelation, rel,
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
foreignrel,
|
2017-04-11 19:53:13 +02:00
|
|
|
remote_exprs, ¶ms_list,
|
2016-03-18 18:48:58 +01:00
|
|
|
returningList, &retrieved_attrs);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
elog(ERROR, "unexpected operation: %d", (int) operation);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2020-10-14 09:58:38 +02:00
|
|
|
* Update the operation and target relation info.
|
2016-03-18 18:48:58 +01:00
|
|
|
*/
|
|
|
|
fscan->operation = operation;
|
2020-10-14 09:58:38 +02:00
|
|
|
fscan->resultRelation = resultRelation;
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Update the fdw_exprs list that will be available to the executor.
|
|
|
|
*/
|
|
|
|
fscan->fdw_exprs = params_list;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Update the fdw_private list that will be available to the executor.
|
|
|
|
* Items in the list must match enum FdwDirectModifyPrivateIndex, above.
|
|
|
|
*/
|
|
|
|
fscan->fdw_private = list_make4(makeString(sql.data),
|
|
|
|
makeInteger((retrieved_attrs != NIL)),
|
|
|
|
retrieved_attrs,
|
|
|
|
makeInteger(plan->canSetTag));
|
|
|
|
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
/*
|
|
|
|
* Update the foreign-join-related fields.
|
|
|
|
*/
|
|
|
|
if (fscan->scan.scanrelid == 0)
|
|
|
|
{
|
|
|
|
/* No need for the outer subplan. */
|
|
|
|
fscan->scan.plan.lefttree = NULL;
|
|
|
|
|
|
|
|
/* Build new fdw_scan_tlist if UPDATE/DELETE .. RETURNING. */
|
|
|
|
if (returningList)
|
|
|
|
rebuild_fdw_scan_tlist(fscan, returningList);
|
|
|
|
}
|
|
|
|
|
2021-05-13 13:00:00 +02:00
|
|
|
/*
|
|
|
|
* Finally, unset the async-capable flag if it is set, as we currently
|
|
|
|
* don't support asynchronous execution of direct modifications.
|
|
|
|
*/
|
|
|
|
if (fscan->scan.plan.async_capable)
|
|
|
|
fscan->scan.plan.async_capable = false;
|
|
|
|
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, NoLock);
|
2016-03-18 18:48:58 +01:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresBeginDirectModify
|
|
|
|
* Prepare a direct foreign table modification
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresBeginDirectModify(ForeignScanState *node, int eflags)
|
|
|
|
{
|
|
|
|
ForeignScan *fsplan = (ForeignScan *) node->ss.ps.plan;
|
|
|
|
EState *estate = node->ss.ps.state;
|
|
|
|
PgFdwDirectModifyState *dmstate;
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
Index rtindex;
|
2016-03-18 18:48:58 +01:00
|
|
|
RangeTblEntry *rte;
|
|
|
|
Oid userid;
|
|
|
|
ForeignTable *table;
|
|
|
|
UserMapping *user;
|
|
|
|
int numParams;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Do nothing in EXPLAIN (no ANALYZE) case. node->fdw_state stays NULL.
|
|
|
|
*/
|
|
|
|
if (eflags & EXEC_FLAG_EXPLAIN_ONLY)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We'll save private state in node->fdw_state.
|
|
|
|
*/
|
|
|
|
dmstate = (PgFdwDirectModifyState *) palloc0(sizeof(PgFdwDirectModifyState));
|
|
|
|
node->fdw_state = (void *) dmstate;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Identify which user to do the remote access as. This should match what
|
|
|
|
* ExecCheckRTEPerms() does.
|
|
|
|
*/
|
2020-10-14 09:58:38 +02:00
|
|
|
rtindex = node->resultRelInfo->ri_RangeTableIndex;
|
2018-10-04 21:48:17 +02:00
|
|
|
rte = exec_rt_fetch(rtindex, estate);
|
2016-03-18 18:48:58 +01:00
|
|
|
userid = rte->checkAsUser ? rte->checkAsUser : GetUserId();
|
|
|
|
|
|
|
|
/* Get info about foreign table. */
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
if (fsplan->scan.scanrelid == 0)
|
|
|
|
dmstate->rel = ExecOpenScanRelation(estate, rtindex, eflags);
|
|
|
|
else
|
|
|
|
dmstate->rel = node->ss.ss_currentRelation;
|
2016-03-18 18:48:58 +01:00
|
|
|
table = GetForeignTable(RelationGetRelid(dmstate->rel));
|
|
|
|
user = GetUserMapping(userid, table->serverid);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get connection to the foreign server. Connection manager will
|
|
|
|
* establish new connection if necessary.
|
|
|
|
*/
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
dmstate->conn = GetConnection(user, false, &dmstate->conn_state);
|
2016-03-18 18:48:58 +01:00
|
|
|
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
/* Update the foreign-join-related fields. */
|
|
|
|
if (fsplan->scan.scanrelid == 0)
|
|
|
|
{
|
|
|
|
/* Save info about foreign table. */
|
|
|
|
dmstate->resultRel = dmstate->rel;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set dmstate->rel to NULL to teach get_returning_data() and
|
|
|
|
* make_tuple_from_result_row() that columns fetched from the remote
|
|
|
|
* server are described by fdw_scan_tlist of the foreign-scan plan
|
|
|
|
* node, not the tuple descriptor for the target relation.
|
|
|
|
*/
|
|
|
|
dmstate->rel = NULL;
|
|
|
|
}
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/* Initialize state variable */
|
|
|
|
dmstate->num_tuples = -1; /* -1 means not set yet */
|
|
|
|
|
|
|
|
/* Get private info created by planner functions. */
|
|
|
|
dmstate->query = strVal(list_nth(fsplan->fdw_private,
|
|
|
|
FdwDirectModifyPrivateUpdateSql));
|
|
|
|
dmstate->has_returning = intVal(list_nth(fsplan->fdw_private,
|
|
|
|
FdwDirectModifyPrivateHasReturning));
|
|
|
|
dmstate->retrieved_attrs = (List *) list_nth(fsplan->fdw_private,
|
|
|
|
FdwDirectModifyPrivateRetrievedAttrs);
|
|
|
|
dmstate->set_processed = intVal(list_nth(fsplan->fdw_private,
|
|
|
|
FdwDirectModifyPrivateSetProcessed));
|
|
|
|
|
|
|
|
/* Create context for per-tuple temp workspace. */
|
|
|
|
dmstate->temp_cxt = AllocSetContextCreate(estate->es_query_cxt,
|
|
|
|
"postgres_fdw temporary data",
|
Add macros to make AllocSetContextCreate() calls simpler and safer.
I found that half a dozen (nearly 5%) of our AllocSetContextCreate calls
had typos in the context-sizing parameters. While none of these led to
especially significant problems, they did create minor inefficiencies,
and it's now clear that expecting people to copy-and-paste those calls
accurately is not a great idea. Let's reduce the risk of future errors
by introducing single macros that encapsulate the common use-cases.
Three such macros are enough to cover all but two special-purpose contexts;
those two calls can be left as-is, I think.
While this patch doesn't in itself improve matters for third-party
extensions, it doesn't break anything for them either, and they can
gradually adopt the simplified notation over time.
In passing, change TopMemoryContext to use the default allocation
parameters. Formerly it could only be extended 8K at a time. That was
probably reasonable when this code was written; but nowadays we create
many more contexts than we did then, so that it's not unusual to have a
couple hundred K in TopMemoryContext, even without considering various
dubious code that sticks other things there. There seems no good reason
not to let it use growing blocks like most other contexts.
Back-patch to 9.6, mostly because that's still close enough to HEAD that
it's easy to do so, and keeping the branches in sync can be expected to
avoid some future back-patching pain. The bugs fixed by these changes
don't seem to be significant enough to justify fixing them further back.
Discussion: <21072.1472321324@sss.pgh.pa.us>
2016-08-27 23:50:38 +02:00
|
|
|
ALLOCSET_SMALL_SIZES);
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/* Prepare for input conversion of RETURNING results. */
|
|
|
|
if (dmstate->has_returning)
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
{
|
|
|
|
TupleDesc tupdesc;
|
|
|
|
|
|
|
|
if (fsplan->scan.scanrelid == 0)
|
2021-06-05 02:07:08 +02:00
|
|
|
tupdesc = get_tupdesc_for_join_scan_tuples(node);
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
else
|
|
|
|
tupdesc = RelationGetDescr(dmstate->rel);
|
|
|
|
|
|
|
|
dmstate->attinmeta = TupleDescGetAttInMetadata(tupdesc);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When performing an UPDATE/DELETE .. RETURNING on a join directly,
|
|
|
|
* initialize a filter to extract an updated/deleted tuple from a scan
|
|
|
|
* tuple.
|
|
|
|
*/
|
|
|
|
if (fsplan->scan.scanrelid == 0)
|
|
|
|
init_returning_filter(dmstate, fsplan->fdw_scan_tlist, rtindex);
|
|
|
|
}
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Prepare for processing of parameters used in remote query, if any.
|
|
|
|
*/
|
|
|
|
numParams = list_length(fsplan->fdw_exprs);
|
|
|
|
dmstate->numParams = numParams;
|
|
|
|
if (numParams > 0)
|
|
|
|
prepare_query_params((PlanState *) node,
|
|
|
|
fsplan->fdw_exprs,
|
|
|
|
numParams,
|
|
|
|
&dmstate->param_flinfo,
|
|
|
|
&dmstate->param_exprs,
|
|
|
|
&dmstate->param_values);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresIterateDirectModify
|
|
|
|
* Execute a direct foreign table modification
|
|
|
|
*/
|
|
|
|
static TupleTableSlot *
|
|
|
|
postgresIterateDirectModify(ForeignScanState *node)
|
|
|
|
{
|
|
|
|
PgFdwDirectModifyState *dmstate = (PgFdwDirectModifyState *) node->fdw_state;
|
|
|
|
EState *estate = node->ss.ps.state;
|
2020-10-14 09:58:38 +02:00
|
|
|
ResultRelInfo *resultRelInfo = node->resultRelInfo;
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is the first call after Begin, execute the statement.
|
|
|
|
*/
|
|
|
|
if (dmstate->num_tuples == -1)
|
|
|
|
execute_dml_stmt(node);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the local query doesn't specify RETURNING, just clear tuple slot.
|
|
|
|
*/
|
|
|
|
if (!resultRelInfo->ri_projectReturning)
|
|
|
|
{
|
|
|
|
TupleTableSlot *slot = node->ss.ss_ScanTupleSlot;
|
|
|
|
Instrumentation *instr = node->ss.ps.instrument;
|
|
|
|
|
|
|
|
Assert(!dmstate->has_returning);
|
|
|
|
|
|
|
|
/* Increment the command es_processed count if necessary. */
|
|
|
|
if (dmstate->set_processed)
|
|
|
|
estate->es_processed += dmstate->num_tuples;
|
|
|
|
|
|
|
|
/* Increment the tuple count for EXPLAIN ANALYZE if necessary. */
|
|
|
|
if (instr)
|
|
|
|
instr->tuplecount += dmstate->num_tuples;
|
|
|
|
|
|
|
|
return ExecClearTuple(slot);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the next RETURNING tuple.
|
|
|
|
*/
|
|
|
|
return get_returning_data(node);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresEndDirectModify
|
|
|
|
* Finish a direct foreign table modification
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresEndDirectModify(ForeignScanState *node)
|
|
|
|
{
|
|
|
|
PgFdwDirectModifyState *dmstate = (PgFdwDirectModifyState *) node->fdw_state;
|
|
|
|
|
|
|
|
/* if dmstate is NULL, we are in EXPLAIN; nothing to do */
|
|
|
|
if (dmstate == NULL)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Release PGresult */
|
|
|
|
if (dmstate->result)
|
|
|
|
PQclear(dmstate->result);
|
|
|
|
|
|
|
|
/* Release remote connection */
|
|
|
|
ReleaseConnection(dmstate->conn);
|
|
|
|
dmstate->conn = NULL;
|
|
|
|
|
|
|
|
/* MemoryContext will be deleted automatically. */
|
|
|
|
}
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/*
|
|
|
|
* postgresExplainForeignScan
|
|
|
|
* Produce extra output for EXPLAIN of a ForeignScan on a foreign table
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresExplainForeignScan(ForeignScanState *node, ExplainState *es)
|
|
|
|
{
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
ForeignScan *plan = castNode(ForeignScan, node->ss.ps.plan);
|
|
|
|
List *fdw_private = plan->fdw_private;
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
/*
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
* Identify foreign scans that are really joins or upper relations. The
|
|
|
|
* input looks something like "(1) LEFT JOIN (2)", and we must replace the
|
|
|
|
* digit string(s), which are RT indexes, with the correct relation names.
|
|
|
|
* We do that here, not when the plan is created, because we can't know
|
|
|
|
* what aliases ruleutils.c will assign at plan creation time.
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
|
|
|
if (list_length(fdw_private) > FdwScanPrivateRelations)
|
|
|
|
{
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
StringInfo relations;
|
|
|
|
char *rawrelations;
|
|
|
|
char *ptr;
|
|
|
|
int minrti,
|
|
|
|
rtoffset;
|
|
|
|
|
|
|
|
rawrelations = strVal(list_nth(fdw_private, FdwScanPrivateRelations));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A difficulty with using a string representation of RT indexes is
|
|
|
|
* that setrefs.c won't update the string when flattening the
|
|
|
|
* rangetable. To find out what rtoffset was applied, identify the
|
|
|
|
* minimum RT index appearing in the string and compare it to the
|
|
|
|
* minimum member of plan->fs_relids. (We expect all the relids in
|
|
|
|
* the join will have been offset by the same amount; the Asserts
|
|
|
|
* below should catch it if that ever changes.)
|
|
|
|
*/
|
|
|
|
minrti = INT_MAX;
|
|
|
|
ptr = rawrelations;
|
|
|
|
while (*ptr)
|
|
|
|
{
|
|
|
|
if (isdigit((unsigned char) *ptr))
|
|
|
|
{
|
|
|
|
int rti = strtol(ptr, &ptr, 10);
|
|
|
|
|
|
|
|
if (rti < minrti)
|
|
|
|
minrti = rti;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
rtoffset = bms_next_member(plan->fs_relids, -1) - minrti;
|
|
|
|
|
|
|
|
/* Now we can translate the string */
|
|
|
|
relations = makeStringInfo();
|
|
|
|
ptr = rawrelations;
|
|
|
|
while (*ptr)
|
|
|
|
{
|
|
|
|
if (isdigit((unsigned char) *ptr))
|
|
|
|
{
|
|
|
|
int rti = strtol(ptr, &ptr, 10);
|
|
|
|
RangeTblEntry *rte;
|
|
|
|
char *relname;
|
|
|
|
char *refname;
|
|
|
|
|
|
|
|
rti += rtoffset;
|
|
|
|
Assert(bms_is_member(rti, plan->fs_relids));
|
|
|
|
rte = rt_fetch(rti, es->rtable);
|
|
|
|
Assert(rte->rtekind == RTE_RELATION);
|
|
|
|
/* This logic should agree with explain.c's ExplainTargetRel */
|
|
|
|
relname = get_rel_name(rte->relid);
|
2019-12-03 18:25:56 +01:00
|
|
|
if (es->verbose)
|
|
|
|
{
|
|
|
|
char *namespace;
|
|
|
|
|
|
|
|
namespace = get_namespace_name(get_rel_namespace(rte->relid));
|
|
|
|
appendStringInfo(relations, "%s.%s",
|
|
|
|
quote_identifier(namespace),
|
|
|
|
quote_identifier(relname));
|
|
|
|
}
|
|
|
|
else
|
2020-10-15 09:35:17 +02:00
|
|
|
appendStringInfoString(relations,
|
|
|
|
quote_identifier(relname));
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
refname = (char *) list_nth(es->rtable_names, rti - 1);
|
|
|
|
if (refname == NULL)
|
|
|
|
refname = rte->eref->aliasname;
|
|
|
|
if (strcmp(refname, relname) != 0)
|
|
|
|
appendStringInfo(relations, " %s",
|
|
|
|
quote_identifier(refname));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
appendStringInfoChar(relations, *ptr++);
|
|
|
|
}
|
|
|
|
ExplainPropertyText("Relations", relations->data, es);
|
2016-02-09 20:00:50 +01:00
|
|
|
}
|
2013-03-10 19:14:53 +01:00
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
|
|
|
* Add remote query, when VERBOSE option is specified.
|
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
if (es->verbose)
|
|
|
|
{
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
char *sql;
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
sql = strVal(list_nth(fdw_private, FdwScanPrivateSelectSql));
|
|
|
|
ExplainPropertyText("Remote SQL", sql, es);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresExplainForeignModify
|
|
|
|
* Produce extra output for EXPLAIN of a ModifyTable on a foreign table
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresExplainForeignModify(ModifyTableState *mtstate,
|
|
|
|
ResultRelInfo *rinfo,
|
|
|
|
List *fdw_private,
|
|
|
|
int subplan_index,
|
|
|
|
ExplainState *es)
|
|
|
|
{
|
|
|
|
if (es->verbose)
|
|
|
|
{
|
|
|
|
char *sql = strVal(list_nth(fdw_private,
|
|
|
|
FdwModifyPrivateUpdateSql));
|
|
|
|
|
|
|
|
ExplainPropertyText("Remote SQL", sql, es);
|
2021-01-20 23:05:46 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* For INSERT we should always have batch size >= 1, but UPDATE and
|
|
|
|
* DELETE don't support batching so don't show the property.
|
|
|
|
*/
|
|
|
|
if (rinfo->ri_BatchSize > 0)
|
|
|
|
ExplainPropertyInteger("Batch Size", NULL, rinfo->ri_BatchSize, es);
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* postgresExplainDirectModify
|
|
|
|
* Produce extra output for EXPLAIN of a ForeignScan that modifies a
|
|
|
|
* foreign table directly
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresExplainDirectModify(ForeignScanState *node, ExplainState *es)
|
|
|
|
{
|
|
|
|
List *fdw_private;
|
|
|
|
char *sql;
|
|
|
|
|
|
|
|
if (es->verbose)
|
|
|
|
{
|
|
|
|
fdw_private = ((ForeignScan *) node->ss.ps.plan)->fdw_private;
|
|
|
|
sql = strVal(list_nth(fdw_private, FdwDirectModifyPrivateUpdateSql));
|
|
|
|
ExplainPropertyText("Remote SQL", sql, es);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Allow TRUNCATE command to truncate foreign tables.
This commit introduces new foreign data wrapper API for TRUNCATE.
It extends TRUNCATE command so that it accepts foreign tables as
the targets to truncate and invokes that API. Also it extends postgres_fdw
so that it can issue TRUNCATE command to foreign servers, by adding
new routine for that TRUNCATE API.
The information about options specified in TRUNCATE command, e.g.,
ONLY, CACADE, etc is passed to FDW via API. The list of foreign tables to
truncate is also passed to FDW. FDW truncates the foreign data sources
that the passed foreign tables specify, based on those information.
For example, postgres_fdw constructs TRUNCATE command using them
and issues it to the foreign server.
For performance, TRUNCATE command invokes the FDW routine for
TRUNCATE once per foreign server that foreign tables to truncate belong to.
Author: Kazutaka Onishi, Kohei KaiGai, slightly modified by Fujii Masao
Reviewed-by: Bharath Rupireddy, Michael Paquier, Zhihong Yu, Alvaro Herrera, Stephen Frost, Ashutosh Bapat, Amit Langote, Daniel Gustafsson, Ibrar Ahmed, Fujii Masao
Discussion: https://postgr.es/m/CAOP8fzb_gkReLput7OvOK+8NHgw-RKqNv59vem7=524krQTcWA@mail.gmail.com
Discussion: https://postgr.es/m/CAJuF6cMWDDqU-vn_knZgma+2GMaout68YUgn1uyDnexRhqqM5Q@mail.gmail.com
2021-04-08 13:56:08 +02:00
|
|
|
/*
|
|
|
|
* postgresExecForeignTruncate
|
|
|
|
* Truncate one or more foreign tables
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresExecForeignTruncate(List *rels,
|
|
|
|
DropBehavior behavior,
|
|
|
|
bool restart_seqs)
|
|
|
|
{
|
|
|
|
Oid serverid = InvalidOid;
|
|
|
|
UserMapping *user = NULL;
|
|
|
|
PGconn *conn = NULL;
|
|
|
|
StringInfoData sql;
|
|
|
|
ListCell *lc;
|
|
|
|
bool server_truncatable = true;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* By default, all postgres_fdw foreign tables are assumed truncatable.
|
|
|
|
* This can be overridden by a per-server setting, which in turn can be
|
|
|
|
* overridden by a per-table setting.
|
|
|
|
*/
|
|
|
|
foreach(lc, rels)
|
|
|
|
{
|
|
|
|
ForeignServer *server = NULL;
|
|
|
|
Relation rel = lfirst(lc);
|
|
|
|
ForeignTable *table = GetForeignTable(RelationGetRelid(rel));
|
|
|
|
ListCell *cell;
|
|
|
|
bool truncatable;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* First time through, determine whether the foreign server allows
|
|
|
|
* truncates. Since all specified foreign tables are assumed to belong
|
|
|
|
* to the same foreign server, this result can be used for other
|
|
|
|
* foreign tables.
|
|
|
|
*/
|
|
|
|
if (!OidIsValid(serverid))
|
|
|
|
{
|
|
|
|
serverid = table->serverid;
|
|
|
|
server = GetForeignServer(serverid);
|
|
|
|
|
|
|
|
foreach(cell, server->options)
|
|
|
|
{
|
|
|
|
DefElem *defel = (DefElem *) lfirst(cell);
|
|
|
|
|
|
|
|
if (strcmp(defel->defname, "truncatable") == 0)
|
|
|
|
{
|
|
|
|
server_truncatable = defGetBoolean(defel);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Confirm that all specified foreign tables belong to the same
|
|
|
|
* foreign server.
|
|
|
|
*/
|
|
|
|
Assert(table->serverid == serverid);
|
|
|
|
|
|
|
|
/* Determine whether this foreign table allows truncations */
|
|
|
|
truncatable = server_truncatable;
|
|
|
|
foreach(cell, table->options)
|
|
|
|
{
|
|
|
|
DefElem *defel = (DefElem *) lfirst(cell);
|
|
|
|
|
|
|
|
if (strcmp(defel->defname, "truncatable") == 0)
|
|
|
|
{
|
|
|
|
truncatable = defGetBoolean(defel);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!truncatable)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
|
|
|
|
errmsg("foreign table \"%s\" does not allow truncates",
|
|
|
|
RelationGetRelationName(rel))));
|
|
|
|
}
|
|
|
|
Assert(OidIsValid(serverid));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get connection to the foreign server. Connection manager will
|
|
|
|
* establish new connection if necessary.
|
|
|
|
*/
|
|
|
|
user = GetUserMapping(GetUserId(), serverid);
|
|
|
|
conn = GetConnection(user, false, NULL);
|
|
|
|
|
|
|
|
/* Construct the TRUNCATE command string */
|
|
|
|
initStringInfo(&sql);
|
2021-04-27 07:41:27 +02:00
|
|
|
deparseTruncateSql(&sql, rels, behavior, restart_seqs);
|
Allow TRUNCATE command to truncate foreign tables.
This commit introduces new foreign data wrapper API for TRUNCATE.
It extends TRUNCATE command so that it accepts foreign tables as
the targets to truncate and invokes that API. Also it extends postgres_fdw
so that it can issue TRUNCATE command to foreign servers, by adding
new routine for that TRUNCATE API.
The information about options specified in TRUNCATE command, e.g.,
ONLY, CACADE, etc is passed to FDW via API. The list of foreign tables to
truncate is also passed to FDW. FDW truncates the foreign data sources
that the passed foreign tables specify, based on those information.
For example, postgres_fdw constructs TRUNCATE command using them
and issues it to the foreign server.
For performance, TRUNCATE command invokes the FDW routine for
TRUNCATE once per foreign server that foreign tables to truncate belong to.
Author: Kazutaka Onishi, Kohei KaiGai, slightly modified by Fujii Masao
Reviewed-by: Bharath Rupireddy, Michael Paquier, Zhihong Yu, Alvaro Herrera, Stephen Frost, Ashutosh Bapat, Amit Langote, Daniel Gustafsson, Ibrar Ahmed, Fujii Masao
Discussion: https://postgr.es/m/CAOP8fzb_gkReLput7OvOK+8NHgw-RKqNv59vem7=524krQTcWA@mail.gmail.com
Discussion: https://postgr.es/m/CAJuF6cMWDDqU-vn_knZgma+2GMaout68YUgn1uyDnexRhqqM5Q@mail.gmail.com
2021-04-08 13:56:08 +02:00
|
|
|
|
|
|
|
/* Issue the TRUNCATE command to remote server */
|
|
|
|
do_sql_command(conn, sql.data);
|
|
|
|
|
|
|
|
pfree(sql.data);
|
|
|
|
}
|
2013-03-22 00:43:59 +01:00
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* estimate_path_cost_size
|
2016-02-09 20:00:50 +01:00
|
|
|
* Get cost and size estimates for a foreign scan on given foreign relation
|
2016-10-21 15:54:29 +02:00
|
|
|
* either a base relation or a join between foreign relations or an upper
|
|
|
|
* relation containing foreign relations.
|
2013-03-22 00:43:59 +01:00
|
|
|
*
|
2016-02-09 20:00:50 +01:00
|
|
|
* param_join_conds are the parameterization clauses with outer relations.
|
|
|
|
* pathkeys specify the expected sort order if any for given path being costed.
|
2019-04-02 12:20:30 +02:00
|
|
|
* fpextra specifies additional post-scan/join-processing steps such as the
|
2019-04-02 13:30:45 +02:00
|
|
|
* final sort and the LIMIT restriction.
|
2016-02-09 20:00:50 +01:00
|
|
|
*
|
2019-05-13 10:30:35 +02:00
|
|
|
* The function returns the cost and size estimates in p_rows, p_width,
|
2016-02-09 20:00:50 +01:00
|
|
|
* p_startup_cost and p_total_cost variables.
|
2013-03-22 00:43:59 +01:00
|
|
|
*/
|
|
|
|
static void
|
|
|
|
estimate_path_cost_size(PlannerInfo *root,
|
2016-02-09 20:00:50 +01:00
|
|
|
RelOptInfo *foreignrel,
|
|
|
|
List *param_join_conds,
|
2015-11-03 18:46:06 +01:00
|
|
|
List *pathkeys,
|
2019-04-02 12:20:30 +02:00
|
|
|
PgFdwPathExtraData *fpextra,
|
2013-03-22 00:43:59 +01:00
|
|
|
double *p_rows, int *p_width,
|
|
|
|
Cost *p_startup_cost, Cost *p_total_cost)
|
|
|
|
{
|
2016-02-09 20:00:50 +01:00
|
|
|
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) foreignrel->fdw_private;
|
2013-03-22 00:43:59 +01:00
|
|
|
double rows;
|
|
|
|
double retrieved_rows;
|
|
|
|
int width;
|
|
|
|
Cost startup_cost;
|
|
|
|
Cost total_cost;
|
|
|
|
|
2019-01-24 08:49:17 +01:00
|
|
|
/* Make sure the core code has set up the relation's reltarget */
|
|
|
|
Assert(foreignrel->reltarget);
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/*
|
|
|
|
* If the table or the server is configured to use remote estimates,
|
|
|
|
* connect to the foreign server and execute EXPLAIN to estimate the
|
|
|
|
* number of rows selected by the restriction+join clauses. Otherwise,
|
|
|
|
* estimate rows using whatever statistics we have locally, in a way
|
|
|
|
* similar to ordinary tables.
|
|
|
|
*/
|
|
|
|
if (fpinfo->use_remote_estimate)
|
|
|
|
{
|
2016-02-09 20:00:50 +01:00
|
|
|
List *remote_param_join_conds;
|
|
|
|
List *local_param_join_conds;
|
2013-03-22 00:43:59 +01:00
|
|
|
StringInfoData sql;
|
|
|
|
PGconn *conn;
|
2014-03-07 22:35:58 +01:00
|
|
|
Selectivity local_sel;
|
|
|
|
QualCost local_cost;
|
2016-02-09 20:00:50 +01:00
|
|
|
List *fdw_scan_tlist = NIL;
|
2016-01-30 16:32:38 +01:00
|
|
|
List *remote_conds;
|
2014-03-07 22:35:58 +01:00
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/* Required only to be passed to deparseSelectStmtForRel */
|
|
|
|
List *retrieved_attrs;
|
|
|
|
|
2014-03-07 22:35:58 +01:00
|
|
|
/*
|
2016-02-09 20:00:50 +01:00
|
|
|
* param_join_conds might contain both clauses that are safe to send
|
|
|
|
* across, and clauses that aren't.
|
2014-03-07 22:35:58 +01:00
|
|
|
*/
|
2016-02-09 20:00:50 +01:00
|
|
|
classifyConditions(root, foreignrel, param_join_conds,
|
|
|
|
&remote_param_join_conds, &local_param_join_conds);
|
|
|
|
|
|
|
|
/* Build the list of columns to be fetched from the foreign server. */
|
Abstract logic to allow for multiple kinds of child rels.
Currently, the only type of child relation is an "other member rel",
which is the child of a baserel, but in the future joins and even
upper relations may have child rels. To facilitate that, introduce
macros that test to test for particular RelOptKind values, and use
them in various places where they help to clarify the sense of a test.
(For example, a test may allow RELOPT_OTHER_MEMBER_REL either because
it intends to allow child rels, or because it intends to allow simple
rels.)
Also, remove find_childrel_top_parent, which will not work for a
child rel that is not a baserel. Instead, add a new RelOptInfo
member top_parent_relids to track the same kind of information in a
more generic manner.
Ashutosh Bapat, slightly tweaked by me. Review and testing of the
patch set from which this was taken by Rajkumar Raghuwanshi and Rafia
Sabih.
Discussion: http://postgr.es/m/CA+TgmoagTnF2yqR3PT2rv=om=wJiZ4-A+ATwdnriTGku1CLYxA@mail.gmail.com
2017-04-04 04:41:31 +02:00
|
|
|
if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel))
|
2016-02-09 20:00:50 +01:00
|
|
|
fdw_scan_tlist = build_tlist_to_deparse(foreignrel);
|
|
|
|
else
|
|
|
|
fdw_scan_tlist = NIL;
|
2013-03-22 00:43:59 +01:00
|
|
|
|
2016-01-30 16:32:38 +01:00
|
|
|
/*
|
|
|
|
* The complete list of remote conditions includes everything from
|
|
|
|
* baserestrictinfo plus any extra join_conds relevant to this
|
|
|
|
* particular path.
|
|
|
|
*/
|
Rationalize use of list_concat + list_copy combinations.
In the wake of commit 1cff1b95a, the result of list_concat no longer
shares the ListCells of the second input. Therefore, we can replace
"list_concat(x, list_copy(y))" with just "list_concat(x, y)".
To improve call sites that were list_copy'ing the first argument,
or both arguments, invent "list_concat_copy()" which produces a new
list sharing no ListCells with either input. (This is a bit faster
than "list_concat(list_copy(x), y)" because it makes the result list
the right size to start with.)
In call sites that were not list_copy'ing the second argument, the new
semantics mean that we are usually leaking the second List's storage,
since typically there is no remaining pointer to it. We considered
inventing another list_copy variant that would list_free the second
input, but concluded that for most call sites it isn't worth worrying
about, given the relative compactness of the new List representation.
(Note that in cases where such leakage would happen, the old code
already leaked the second List's header; so we're only discussing
the size of the leak not whether there is one. I did adjust two or
three places that had been troubling to free that header so that
they manually free the whole second List.)
Patch by me; thanks to David Rowley for review.
Discussion: https://postgr.es/m/11587.1550975080@sss.pgh.pa.us
2019-08-12 17:20:18 +02:00
|
|
|
remote_conds = list_concat(remote_param_join_conds,
|
2016-01-30 16:32:38 +01:00
|
|
|
fpinfo->remote_conds);
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/*
|
|
|
|
* Construct EXPLAIN query including the desired SELECT, FROM, and
|
2016-02-09 20:00:50 +01:00
|
|
|
* WHERE clauses. Params and other-relation Vars are replaced by dummy
|
|
|
|
* values, so don't request params_list.
|
2013-03-22 00:43:59 +01:00
|
|
|
*/
|
|
|
|
initStringInfo(&sql);
|
|
|
|
appendStringInfoString(&sql, "EXPLAIN ");
|
2016-02-09 20:00:50 +01:00
|
|
|
deparseSelectStmtForRel(&sql, root, foreignrel, fdw_scan_tlist,
|
2019-04-02 12:20:30 +02:00
|
|
|
remote_conds, pathkeys,
|
|
|
|
fpextra ? fpextra->has_final_sort : false,
|
2019-04-02 13:30:45 +02:00
|
|
|
fpextra ? fpextra->has_limit : false,
|
2019-04-02 12:20:30 +02:00
|
|
|
false, &retrieved_attrs, NULL);
|
2015-11-03 18:46:06 +01:00
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/* Get the remote estimate */
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
conn = GetConnection(fpinfo->user, false, NULL);
|
2013-03-22 00:43:59 +01:00
|
|
|
get_remote_estimate(sql.data, conn, &rows, &width,
|
|
|
|
&startup_cost, &total_cost);
|
|
|
|
ReleaseConnection(conn);
|
|
|
|
|
|
|
|
retrieved_rows = rows;
|
|
|
|
|
2014-03-07 22:35:58 +01:00
|
|
|
/* Factor in the selectivity of the locally-checked quals */
|
|
|
|
local_sel = clauselist_selectivity(root,
|
2016-02-09 20:00:50 +01:00
|
|
|
local_param_join_conds,
|
|
|
|
foreignrel->relid,
|
2014-03-07 22:35:58 +01:00
|
|
|
JOIN_INNER,
|
|
|
|
NULL);
|
|
|
|
local_sel *= fpinfo->local_conds_sel;
|
|
|
|
|
|
|
|
rows = clamp_row_est(rows * local_sel);
|
2013-03-22 00:43:59 +01:00
|
|
|
|
2014-03-07 22:35:58 +01:00
|
|
|
/* Add in the eval cost of the locally-checked quals */
|
2013-03-22 00:43:59 +01:00
|
|
|
startup_cost += fpinfo->local_conds_cost.startup;
|
|
|
|
total_cost += fpinfo->local_conds_cost.per_tuple * retrieved_rows;
|
2016-02-09 20:00:50 +01:00
|
|
|
cost_qual_eval(&local_cost, local_param_join_conds, root);
|
2014-03-07 22:35:58 +01:00
|
|
|
startup_cost += local_cost.startup;
|
|
|
|
total_cost += local_cost.per_tuple * retrieved_rows;
|
2019-01-24 08:49:17 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Add in tlist eval cost for each output row. In case of an
|
|
|
|
* aggregate, some of the tlist expressions such as grouping
|
|
|
|
* expressions will be evaluated remotely, so adjust the costs.
|
|
|
|
*/
|
|
|
|
startup_cost += foreignrel->reltarget->cost.startup;
|
|
|
|
total_cost += foreignrel->reltarget->cost.startup;
|
|
|
|
total_cost += foreignrel->reltarget->cost.per_tuple * rows;
|
|
|
|
if (IS_UPPER_REL(foreignrel))
|
|
|
|
{
|
|
|
|
QualCost tlist_cost;
|
|
|
|
|
|
|
|
cost_qual_eval(&tlist_cost, fdw_scan_tlist, root);
|
|
|
|
startup_cost -= tlist_cost.startup;
|
|
|
|
total_cost -= tlist_cost.startup;
|
|
|
|
total_cost -= tlist_cost.per_tuple * rows;
|
|
|
|
}
|
2013-03-22 00:43:59 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2016-02-09 20:00:50 +01:00
|
|
|
Cost run_cost = 0;
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/*
|
|
|
|
* We don't support join conditions in this mode (hence, no
|
|
|
|
* parameterized paths can be made).
|
|
|
|
*/
|
2016-02-09 20:00:50 +01:00
|
|
|
Assert(param_join_conds == NIL);
|
2013-03-22 00:43:59 +01:00
|
|
|
|
2016-03-09 16:51:49 +01:00
|
|
|
/*
|
2019-04-02 13:30:45 +02:00
|
|
|
* We will come here again and again with different set of pathkeys or
|
|
|
|
* additional post-scan/join-processing steps that caller wants to
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
* cost. We don't need to calculate the cost/size estimates for the
|
|
|
|
* underlying scan, join, or grouping each time. Instead, use those
|
|
|
|
* estimates if we have cached them already.
|
2016-03-09 16:51:49 +01:00
|
|
|
*/
|
2019-01-29 04:27:13 +01:00
|
|
|
if (fpinfo->rel_startup_cost >= 0 && fpinfo->rel_total_cost >= 0)
|
2016-03-09 16:51:49 +01:00
|
|
|
{
|
2021-02-05 07:30:00 +01:00
|
|
|
Assert(fpinfo->retrieved_rows >= 0);
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
|
|
|
|
rows = fpinfo->rows;
|
|
|
|
retrieved_rows = fpinfo->retrieved_rows;
|
|
|
|
width = fpinfo->width;
|
2016-03-09 16:51:49 +01:00
|
|
|
startup_cost = fpinfo->rel_startup_cost;
|
|
|
|
run_cost = fpinfo->rel_total_cost - fpinfo->rel_startup_cost;
|
2019-04-02 13:30:45 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we estimate the costs of a foreign scan or a foreign join
|
|
|
|
* with additional post-scan/join-processing steps, the scan or
|
|
|
|
* join costs obtained from the cache wouldn't yet contain the
|
|
|
|
* eval costs for the final scan/join target, which would've been
|
|
|
|
* updated by apply_scanjoin_target_to_paths(); add the eval costs
|
|
|
|
* now.
|
|
|
|
*/
|
|
|
|
if (fpextra && !IS_UPPER_REL(foreignrel))
|
|
|
|
{
|
|
|
|
/* Shouldn't get here unless we have LIMIT */
|
|
|
|
Assert(fpextra->has_limit);
|
|
|
|
Assert(foreignrel->reloptkind == RELOPT_BASEREL ||
|
|
|
|
foreignrel->reloptkind == RELOPT_JOINREL);
|
|
|
|
startup_cost += foreignrel->reltarget->cost.startup;
|
|
|
|
run_cost += foreignrel->reltarget->cost.per_tuple * rows;
|
|
|
|
}
|
2016-03-09 16:51:49 +01:00
|
|
|
}
|
Abstract logic to allow for multiple kinds of child rels.
Currently, the only type of child relation is an "other member rel",
which is the child of a baserel, but in the future joins and even
upper relations may have child rels. To facilitate that, introduce
macros that test to test for particular RelOptKind values, and use
them in various places where they help to clarify the sense of a test.
(For example, a test may allow RELOPT_OTHER_MEMBER_REL either because
it intends to allow child rels, or because it intends to allow simple
rels.)
Also, remove find_childrel_top_parent, which will not work for a
child rel that is not a baserel. Instead, add a new RelOptInfo
member top_parent_relids to track the same kind of information in a
more generic manner.
Ashutosh Bapat, slightly tweaked by me. Review and testing of the
patch set from which this was taken by Rajkumar Raghuwanshi and Rafia
Sabih.
Discussion: http://postgr.es/m/CA+TgmoagTnF2yqR3PT2rv=om=wJiZ4-A+ATwdnriTGku1CLYxA@mail.gmail.com
2017-04-04 04:41:31 +02:00
|
|
|
else if (IS_JOIN_REL(foreignrel))
|
2016-02-09 20:00:50 +01:00
|
|
|
{
|
|
|
|
PgFdwRelationInfo *fpinfo_i;
|
|
|
|
PgFdwRelationInfo *fpinfo_o;
|
|
|
|
QualCost join_cost;
|
|
|
|
QualCost remote_conds_cost;
|
|
|
|
double nrows;
|
|
|
|
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
/* Use rows/width estimates made by the core code. */
|
|
|
|
rows = foreignrel->rows;
|
|
|
|
width = foreignrel->reltarget->width;
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/* For join we expect inner and outer relations set */
|
|
|
|
Assert(fpinfo->innerrel && fpinfo->outerrel);
|
|
|
|
|
|
|
|
fpinfo_i = (PgFdwRelationInfo *) fpinfo->innerrel->fdw_private;
|
|
|
|
fpinfo_o = (PgFdwRelationInfo *) fpinfo->outerrel->fdw_private;
|
|
|
|
|
|
|
|
/* Estimate of number of rows in cross product */
|
|
|
|
nrows = fpinfo_i->rows * fpinfo_o->rows;
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Back into an estimate of the number of retrieved rows. Just in
|
2019-08-05 05:14:58 +02:00
|
|
|
* case this is nuts, clamp to at most nrows.
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
*/
|
|
|
|
retrieved_rows = clamp_row_est(rows / fpinfo->local_conds_sel);
|
2016-02-09 20:00:50 +01:00
|
|
|
retrieved_rows = Min(retrieved_rows, nrows);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The cost of foreign join is estimated as cost of generating
|
|
|
|
* rows for the joining relations + cost for applying quals on the
|
|
|
|
* rows.
|
|
|
|
*/
|
|
|
|
|
2016-08-24 22:29:10 +02:00
|
|
|
/*
|
|
|
|
* Calculate the cost of clauses pushed down to the foreign server
|
|
|
|
*/
|
2016-02-09 20:00:50 +01:00
|
|
|
cost_qual_eval(&remote_conds_cost, fpinfo->remote_conds, root);
|
|
|
|
/* Calculate the cost of applying join clauses */
|
|
|
|
cost_qual_eval(&join_cost, fpinfo->joinclauses, root);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Startup cost includes startup cost of joining relations and the
|
|
|
|
* startup cost for join and other clauses. We do not include the
|
|
|
|
* startup cost specific to join strategy (e.g. setting up hash
|
|
|
|
* tables) since we do not know what strategy the foreign server
|
|
|
|
* is going to use.
|
|
|
|
*/
|
|
|
|
startup_cost = fpinfo_i->rel_startup_cost + fpinfo_o->rel_startup_cost;
|
|
|
|
startup_cost += join_cost.startup;
|
|
|
|
startup_cost += remote_conds_cost.startup;
|
|
|
|
startup_cost += fpinfo->local_conds_cost.startup;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Run time cost includes:
|
|
|
|
*
|
|
|
|
* 1. Run time cost (total_cost - startup_cost) of relations being
|
|
|
|
* joined
|
|
|
|
*
|
|
|
|
* 2. Run time cost of applying join clauses on the cross product
|
|
|
|
* of the joining relations.
|
|
|
|
*
|
|
|
|
* 3. Run time cost of applying pushed down other clauses on the
|
|
|
|
* result of join
|
|
|
|
*
|
|
|
|
* 4. Run time cost of applying nonpushable other clauses locally
|
|
|
|
* on the result fetched from the foreign server.
|
|
|
|
*/
|
|
|
|
run_cost = fpinfo_i->rel_total_cost - fpinfo_i->rel_startup_cost;
|
|
|
|
run_cost += fpinfo_o->rel_total_cost - fpinfo_o->rel_startup_cost;
|
|
|
|
run_cost += nrows * join_cost.per_tuple;
|
|
|
|
nrows = clamp_row_est(nrows * fpinfo->joinclause_sel);
|
|
|
|
run_cost += nrows * remote_conds_cost.per_tuple;
|
|
|
|
run_cost += fpinfo->local_conds_cost.per_tuple * retrieved_rows;
|
2019-01-24 08:49:17 +01:00
|
|
|
|
|
|
|
/* Add in tlist eval cost for each output row */
|
|
|
|
startup_cost += foreignrel->reltarget->cost.startup;
|
|
|
|
run_cost += foreignrel->reltarget->cost.per_tuple * rows;
|
2016-02-09 20:00:50 +01:00
|
|
|
}
|
Abstract logic to allow for multiple kinds of child rels.
Currently, the only type of child relation is an "other member rel",
which is the child of a baserel, but in the future joins and even
upper relations may have child rels. To facilitate that, introduce
macros that test to test for particular RelOptKind values, and use
them in various places where they help to clarify the sense of a test.
(For example, a test may allow RELOPT_OTHER_MEMBER_REL either because
it intends to allow child rels, or because it intends to allow simple
rels.)
Also, remove find_childrel_top_parent, which will not work for a
child rel that is not a baserel. Instead, add a new RelOptInfo
member top_parent_relids to track the same kind of information in a
more generic manner.
Ashutosh Bapat, slightly tweaked by me. Review and testing of the
patch set from which this was taken by Rajkumar Raghuwanshi and Rafia
Sabih.
Discussion: http://postgr.es/m/CA+TgmoagTnF2yqR3PT2rv=om=wJiZ4-A+ATwdnriTGku1CLYxA@mail.gmail.com
2017-04-04 04:41:31 +02:00
|
|
|
else if (IS_UPPER_REL(foreignrel))
|
2016-10-21 15:54:29 +02:00
|
|
|
{
|
2019-05-09 11:39:23 +02:00
|
|
|
RelOptInfo *outerrel = fpinfo->outerrel;
|
2016-10-21 15:54:29 +02:00
|
|
|
PgFdwRelationInfo *ofpinfo;
|
|
|
|
AggClauseCosts aggcosts;
|
|
|
|
double input_rows;
|
|
|
|
int numGroupCols;
|
|
|
|
double numGroups = 1;
|
|
|
|
|
2019-05-09 11:39:23 +02:00
|
|
|
/* The upper relation should have its outer relation set */
|
|
|
|
Assert(outerrel);
|
|
|
|
/* and that outer relation should have its reltarget set */
|
|
|
|
Assert(outerrel->reltarget);
|
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
/*
|
|
|
|
* This cost model is mixture of costing done for sorted and
|
|
|
|
* hashed aggregates in cost_agg(). We are not sure which
|
|
|
|
* strategy will be considered at remote side, thus for
|
|
|
|
* simplicity, we put all startup related costs in startup_cost
|
|
|
|
* and all finalization and run cost are added in total_cost.
|
|
|
|
*/
|
|
|
|
|
2019-05-09 11:39:23 +02:00
|
|
|
ofpinfo = (PgFdwRelationInfo *) outerrel->fdw_private;
|
2016-10-21 15:54:29 +02:00
|
|
|
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
/* Get rows from input rel */
|
2016-10-21 15:54:29 +02:00
|
|
|
input_rows = ofpinfo->rows;
|
|
|
|
|
|
|
|
/* Collect statistics about aggregates for estimating costs. */
|
|
|
|
MemSet(&aggcosts, 0, sizeof(AggClauseCosts));
|
|
|
|
if (root->parse->hasAggs)
|
|
|
|
{
|
2020-11-24 09:45:00 +01:00
|
|
|
get_agg_clause_costs(root, AGGSPLIT_SIMPLE, &aggcosts);
|
2016-10-21 15:54:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Get number of grouping columns and possible number of groups */
|
|
|
|
numGroupCols = list_length(root->parse->groupClause);
|
|
|
|
numGroups = estimate_num_groups(root,
|
|
|
|
get_sortgrouplist_exprs(root->parse->groupClause,
|
|
|
|
fpinfo->grouped_tlist),
|
2021-03-30 09:52:46 +02:00
|
|
|
input_rows, NULL, NULL);
|
2016-10-21 15:54:29 +02:00
|
|
|
|
|
|
|
/*
|
postgres_fdw: Improve cost and size estimation for aggregate pushdown.
In commit 7012b132d07c2b4ea15b0b3cb1ea9f3278801d98, which added aggregate
pushdown to postgres_fdw, we didn't account for the evaluation cost and the
selectivity of HAVING quals attached to ForeignPaths performing aggregate
pushdown, as core had never accounted for that for AggPaths and GroupPaths.
And we didn't set these values of the locally-checked quals (ie, fpinfo's
local_conds_cost and local_conds_sel), which were initialized to zeros, but
since estimate_path_cost_size factors in these to estimate the result size
and the evaluation cost of such a ForeignPath when the use_remote_estimate
option is enabled, this caused it to produce underestimated results in that
case.
By commit 7b6c07547190f056b0464098bb5a2247129d7aa2 core was changed so that
it accounts for the evaluation cost and the selectivity of HAVING quals in
aggregation paths, so change the postgres_fdw's aggregate pushdown code as
well as such. This not only fixes the underestimation issue mentioned
above, but improves the estimation using local statistics in that function
when that option is disabled.
This would be a bug fix rather than an improvement, but apply it to HEAD
only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/5BFD3EAD.2060301%40lab.ntt.co.jp
2018-12-04 09:18:58 +01:00
|
|
|
* Get the retrieved_rows and rows estimates. If there are HAVING
|
|
|
|
* quals, account for their selectivity.
|
2016-10-21 15:54:29 +02:00
|
|
|
*/
|
postgres_fdw: Improve cost and size estimation for aggregate pushdown.
In commit 7012b132d07c2b4ea15b0b3cb1ea9f3278801d98, which added aggregate
pushdown to postgres_fdw, we didn't account for the evaluation cost and the
selectivity of HAVING quals attached to ForeignPaths performing aggregate
pushdown, as core had never accounted for that for AggPaths and GroupPaths.
And we didn't set these values of the locally-checked quals (ie, fpinfo's
local_conds_cost and local_conds_sel), which were initialized to zeros, but
since estimate_path_cost_size factors in these to estimate the result size
and the evaluation cost of such a ForeignPath when the use_remote_estimate
option is enabled, this caused it to produce underestimated results in that
case.
By commit 7b6c07547190f056b0464098bb5a2247129d7aa2 core was changed so that
it accounts for the evaluation cost and the selectivity of HAVING quals in
aggregation paths, so change the postgres_fdw's aggregate pushdown code as
well as such. This not only fixes the underestimation issue mentioned
above, but improves the estimation using local statistics in that function
when that option is disabled.
This would be a bug fix rather than an improvement, but apply it to HEAD
only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/5BFD3EAD.2060301%40lab.ntt.co.jp
2018-12-04 09:18:58 +01:00
|
|
|
if (root->parse->havingQual)
|
|
|
|
{
|
|
|
|
/* Factor in the selectivity of the remotely-checked quals */
|
|
|
|
retrieved_rows =
|
|
|
|
clamp_row_est(numGroups *
|
|
|
|
clauselist_selectivity(root,
|
|
|
|
fpinfo->remote_conds,
|
|
|
|
0,
|
|
|
|
JOIN_INNER,
|
|
|
|
NULL));
|
|
|
|
/* Factor in the selectivity of the locally-checked quals */
|
|
|
|
rows = clamp_row_est(retrieved_rows * fpinfo->local_conds_sel);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
rows = retrieved_rows = numGroups;
|
|
|
|
}
|
2016-10-21 15:54:29 +02:00
|
|
|
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
/* Use width estimate made by the core code. */
|
|
|
|
width = foreignrel->reltarget->width;
|
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
/*-----
|
|
|
|
* Startup cost includes:
|
2019-05-09 11:39:23 +02:00
|
|
|
* 1. Startup cost for underneath input relation, adjusted for
|
|
|
|
* tlist replacement by apply_scanjoin_target_to_paths()
|
2016-10-21 15:54:29 +02:00
|
|
|
* 2. Cost of performing aggregation, per cost_agg()
|
|
|
|
*-----
|
|
|
|
*/
|
|
|
|
startup_cost = ofpinfo->rel_startup_cost;
|
2019-05-09 11:39:23 +02:00
|
|
|
startup_cost += outerrel->reltarget->cost.startup;
|
2016-10-21 15:54:29 +02:00
|
|
|
startup_cost += aggcosts.transCost.startup;
|
|
|
|
startup_cost += aggcosts.transCost.per_tuple * input_rows;
|
2019-02-10 00:32:23 +01:00
|
|
|
startup_cost += aggcosts.finalCost.startup;
|
2016-10-21 15:54:29 +02:00
|
|
|
startup_cost += (cpu_operator_cost * numGroupCols) * input_rows;
|
|
|
|
|
|
|
|
/*-----
|
|
|
|
* Run time cost includes:
|
2019-05-09 11:39:23 +02:00
|
|
|
* 1. Run time cost of underneath input relation, adjusted for
|
|
|
|
* tlist replacement by apply_scanjoin_target_to_paths()
|
2016-10-21 15:54:29 +02:00
|
|
|
* 2. Run time cost of performing aggregation, per cost_agg()
|
|
|
|
*-----
|
|
|
|
*/
|
|
|
|
run_cost = ofpinfo->rel_total_cost - ofpinfo->rel_startup_cost;
|
2019-05-09 11:39:23 +02:00
|
|
|
run_cost += outerrel->reltarget->cost.per_tuple * input_rows;
|
2019-02-10 00:32:23 +01:00
|
|
|
run_cost += aggcosts.finalCost.per_tuple * numGroups;
|
2016-10-21 15:54:29 +02:00
|
|
|
run_cost += cpu_tuple_cost * numGroups;
|
postgres_fdw: Improve cost and size estimation for aggregate pushdown.
In commit 7012b132d07c2b4ea15b0b3cb1ea9f3278801d98, which added aggregate
pushdown to postgres_fdw, we didn't account for the evaluation cost and the
selectivity of HAVING quals attached to ForeignPaths performing aggregate
pushdown, as core had never accounted for that for AggPaths and GroupPaths.
And we didn't set these values of the locally-checked quals (ie, fpinfo's
local_conds_cost and local_conds_sel), which were initialized to zeros, but
since estimate_path_cost_size factors in these to estimate the result size
and the evaluation cost of such a ForeignPath when the use_remote_estimate
option is enabled, this caused it to produce underestimated results in that
case.
By commit 7b6c07547190f056b0464098bb5a2247129d7aa2 core was changed so that
it accounts for the evaluation cost and the selectivity of HAVING quals in
aggregation paths, so change the postgres_fdw's aggregate pushdown code as
well as such. This not only fixes the underestimation issue mentioned
above, but improves the estimation using local statistics in that function
when that option is disabled.
This would be a bug fix rather than an improvement, but apply it to HEAD
only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/5BFD3EAD.2060301%40lab.ntt.co.jp
2018-12-04 09:18:58 +01:00
|
|
|
|
2019-01-23 12:39:00 +01:00
|
|
|
/* Account for the eval cost of HAVING quals, if any */
|
postgres_fdw: Improve cost and size estimation for aggregate pushdown.
In commit 7012b132d07c2b4ea15b0b3cb1ea9f3278801d98, which added aggregate
pushdown to postgres_fdw, we didn't account for the evaluation cost and the
selectivity of HAVING quals attached to ForeignPaths performing aggregate
pushdown, as core had never accounted for that for AggPaths and GroupPaths.
And we didn't set these values of the locally-checked quals (ie, fpinfo's
local_conds_cost and local_conds_sel), which were initialized to zeros, but
since estimate_path_cost_size factors in these to estimate the result size
and the evaluation cost of such a ForeignPath when the use_remote_estimate
option is enabled, this caused it to produce underestimated results in that
case.
By commit 7b6c07547190f056b0464098bb5a2247129d7aa2 core was changed so that
it accounts for the evaluation cost and the selectivity of HAVING quals in
aggregation paths, so change the postgres_fdw's aggregate pushdown code as
well as such. This not only fixes the underestimation issue mentioned
above, but improves the estimation using local statistics in that function
when that option is disabled.
This would be a bug fix rather than an improvement, but apply it to HEAD
only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/5BFD3EAD.2060301%40lab.ntt.co.jp
2018-12-04 09:18:58 +01:00
|
|
|
if (root->parse->havingQual)
|
|
|
|
{
|
|
|
|
QualCost remote_cost;
|
|
|
|
|
|
|
|
/* Add in the eval cost of the remotely-checked quals */
|
|
|
|
cost_qual_eval(&remote_cost, fpinfo->remote_conds, root);
|
|
|
|
startup_cost += remote_cost.startup;
|
|
|
|
run_cost += remote_cost.per_tuple * numGroups;
|
|
|
|
/* Add in the eval cost of the locally-checked quals */
|
|
|
|
startup_cost += fpinfo->local_conds_cost.startup;
|
|
|
|
run_cost += fpinfo->local_conds_cost.per_tuple * retrieved_rows;
|
|
|
|
}
|
2019-01-24 08:49:17 +01:00
|
|
|
|
|
|
|
/* Add in tlist eval cost for each output row */
|
|
|
|
startup_cost += foreignrel->reltarget->cost.startup;
|
|
|
|
run_cost += foreignrel->reltarget->cost.per_tuple * rows;
|
2016-10-21 15:54:29 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
Cost cpu_per_tuple;
|
|
|
|
|
|
|
|
/* Use rows/width estimates made by set_baserel_size_estimates. */
|
|
|
|
rows = foreignrel->rows;
|
|
|
|
width = foreignrel->reltarget->width;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Back into an estimate of the number of retrieved rows. Just in
|
|
|
|
* case this is nuts, clamp to at most foreignrel->tuples.
|
|
|
|
*/
|
|
|
|
retrieved_rows = clamp_row_est(rows / fpinfo->local_conds_sel);
|
2016-10-21 15:54:29 +02:00
|
|
|
retrieved_rows = Min(retrieved_rows, foreignrel->tuples);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Cost as though this were a seqscan, which is pessimistic. We
|
|
|
|
* effectively imagine the local_conds are being evaluated
|
|
|
|
* remotely, too.
|
|
|
|
*/
|
|
|
|
startup_cost = 0;
|
|
|
|
run_cost = 0;
|
|
|
|
run_cost += seq_page_cost * foreignrel->pages;
|
|
|
|
|
|
|
|
startup_cost += foreignrel->baserestrictcost.startup;
|
|
|
|
cpu_per_tuple = cpu_tuple_cost + foreignrel->baserestrictcost.per_tuple;
|
|
|
|
run_cost += cpu_per_tuple * foreignrel->tuples;
|
2019-01-24 08:49:17 +01:00
|
|
|
|
|
|
|
/* Add in tlist eval cost for each output row */
|
|
|
|
startup_cost += foreignrel->reltarget->cost.startup;
|
|
|
|
run_cost += foreignrel->reltarget->cost.per_tuple * rows;
|
2016-10-21 15:54:29 +02:00
|
|
|
}
|
2013-03-22 00:43:59 +01:00
|
|
|
|
2015-11-03 18:46:06 +01:00
|
|
|
/*
|
|
|
|
* Without remote estimates, we have no real way to estimate the cost
|
|
|
|
* of generating sorted output. It could be free if the query plan
|
|
|
|
* the remote side would have chosen generates properly-sorted output
|
|
|
|
* anyway, but in most cases it will cost something. Estimate a value
|
|
|
|
* high enough that we won't pick the sorted path when the ordering
|
|
|
|
* isn't locally useful, but low enough that we'll err on the side of
|
|
|
|
* pushing down the ORDER BY clause when it's useful to do so.
|
|
|
|
*/
|
|
|
|
if (pathkeys != NIL)
|
|
|
|
{
|
2019-04-02 12:20:30 +02:00
|
|
|
if (IS_UPPER_REL(foreignrel))
|
|
|
|
{
|
|
|
|
Assert(foreignrel->reloptkind == RELOPT_UPPER_REL &&
|
|
|
|
fpinfo->stage == UPPERREL_GROUP_AGG);
|
|
|
|
adjust_foreign_grouping_path_cost(root, pathkeys,
|
|
|
|
retrieved_rows, width,
|
2019-04-02 13:30:45 +02:00
|
|
|
fpextra->limit_tuples,
|
2019-04-02 12:20:30 +02:00
|
|
|
&startup_cost, &run_cost);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
startup_cost *= DEFAULT_FDW_SORT_MULTIPLIER;
|
|
|
|
run_cost *= DEFAULT_FDW_SORT_MULTIPLIER;
|
|
|
|
}
|
2015-11-03 18:46:06 +01:00
|
|
|
}
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
total_cost = startup_cost + run_cost;
|
2019-04-02 13:30:45 +02:00
|
|
|
|
|
|
|
/* Adjust the cost estimates if we have LIMIT */
|
|
|
|
if (fpextra && fpextra->has_limit)
|
|
|
|
{
|
|
|
|
adjust_limit_rows_costs(&rows, &startup_cost, &total_cost,
|
|
|
|
fpextra->offset_est, fpextra->count_est);
|
|
|
|
retrieved_rows = rows;
|
|
|
|
}
|
2013-03-22 00:43:59 +01:00
|
|
|
}
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
2019-04-02 12:20:30 +02:00
|
|
|
* If this includes the final sort step, the given target, which will be
|
|
|
|
* applied to the resulting path, might have different expressions from
|
|
|
|
* the foreignrel's reltarget (see make_sort_input_target()); adjust tlist
|
|
|
|
* eval costs.
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
2019-04-02 13:30:45 +02:00
|
|
|
if (fpextra && fpextra->has_final_sort &&
|
|
|
|
fpextra->target != foreignrel->reltarget)
|
2019-04-02 12:20:30 +02:00
|
|
|
{
|
|
|
|
QualCost oldcost = foreignrel->reltarget->cost;
|
|
|
|
QualCost newcost = fpextra->target->cost;
|
|
|
|
|
|
|
|
startup_cost += newcost.startup - oldcost.startup;
|
|
|
|
total_cost += newcost.startup - oldcost.startup;
|
|
|
|
total_cost += (newcost.per_tuple - oldcost.per_tuple) * rows;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
* Cache the retrieved rows and cost estimates for scans, joins, or
|
|
|
|
* groupings without any parameterization, pathkeys, or additional
|
|
|
|
* post-scan/join-processing steps, before adding the costs for
|
|
|
|
* transferring data from the foreign server. These estimates are useful
|
|
|
|
* for costing remote joins involving this relation or costing other
|
|
|
|
* remote operations on this relation such as remote sorts and remote
|
|
|
|
* LIMIT restrictions, when the costs can not be obtained from the foreign
|
|
|
|
* server. This function will be called at least once for every foreign
|
|
|
|
* relation without any parameterization, pathkeys, or additional
|
|
|
|
* post-scan/join-processing steps.
|
2019-04-02 12:20:30 +02:00
|
|
|
*/
|
|
|
|
if (pathkeys == NIL && param_join_conds == NIL && fpextra == NULL)
|
2016-03-09 16:51:49 +01:00
|
|
|
{
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
fpinfo->retrieved_rows = retrieved_rows;
|
2016-03-09 16:51:49 +01:00
|
|
|
fpinfo->rel_startup_cost = startup_cost;
|
|
|
|
fpinfo->rel_total_cost = total_cost;
|
|
|
|
}
|
2016-02-09 20:00:50 +01:00
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/*
|
|
|
|
* Add some additional cost factors to account for connection overhead
|
|
|
|
* (fdw_startup_cost), transferring data across the network
|
|
|
|
* (fdw_tuple_cost per retrieved row), and local manipulation of the data
|
|
|
|
* (cpu_tuple_cost per retrieved row).
|
|
|
|
*/
|
|
|
|
startup_cost += fpinfo->fdw_startup_cost;
|
|
|
|
total_cost += fpinfo->fdw_startup_cost;
|
|
|
|
total_cost += fpinfo->fdw_tuple_cost * retrieved_rows;
|
|
|
|
total_cost += cpu_tuple_cost * retrieved_rows;
|
|
|
|
|
2019-04-02 13:30:45 +02:00
|
|
|
/*
|
2019-05-26 14:58:18 +02:00
|
|
|
* If we have LIMIT, we should prefer performing the restriction remotely
|
2019-04-02 13:30:45 +02:00
|
|
|
* rather than locally, as the former avoids extra row fetches from the
|
|
|
|
* remote that the latter might cause. But since the core code doesn't
|
|
|
|
* account for such fetches when estimating the costs of the local
|
|
|
|
* restriction (see create_limit_path()), there would be no difference
|
|
|
|
* between the costs of the local restriction and the costs of the remote
|
|
|
|
* restriction estimated above if we don't use remote estimates (except
|
|
|
|
* for the case where the foreignrel is a grouping relation, the given
|
|
|
|
* pathkeys is not NIL, and the effects of a bounded sort for that rel is
|
|
|
|
* accounted for in costing the remote restriction). Tweak the costs of
|
|
|
|
* the remote restriction to ensure we'll prefer it if LIMIT is a useful
|
|
|
|
* one.
|
|
|
|
*/
|
|
|
|
if (!fpinfo->use_remote_estimate &&
|
|
|
|
fpextra && fpextra->has_limit &&
|
|
|
|
fpextra->limit_tuples > 0 &&
|
|
|
|
fpextra->limit_tuples < fpinfo->rows)
|
|
|
|
{
|
|
|
|
Assert(fpinfo->rows > 0);
|
|
|
|
total_cost -= (total_cost - startup_cost) * 0.05 *
|
|
|
|
(fpinfo->rows - fpextra->limit_tuples) / fpinfo->rows;
|
|
|
|
}
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/* Return results. */
|
|
|
|
*p_rows = rows;
|
|
|
|
*p_width = width;
|
|
|
|
*p_startup_cost = startup_cost;
|
|
|
|
*p_total_cost = total_cost;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Estimate costs of executing a SQL statement remotely.
|
|
|
|
* The given "sql" must be an EXPLAIN command.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
|
|
|
static void
|
|
|
|
get_remote_estimate(const char *sql, PGconn *conn,
|
|
|
|
double *rows, int *width,
|
|
|
|
Cost *startup_cost, Cost *total_cost)
|
|
|
|
{
|
|
|
|
PGresult *volatile res = NULL;
|
|
|
|
|
|
|
|
/* PGresult must be released before leaving this function. */
|
|
|
|
PG_TRY();
|
|
|
|
{
|
|
|
|
char *line;
|
|
|
|
char *p;
|
|
|
|
int n;
|
|
|
|
|
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* Execute EXPLAIN remotely.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(conn, sql, NULL);
|
2013-02-21 11:26:23 +01:00
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
2014-02-04 03:30:02 +01:00
|
|
|
pgfdw_report_error(ERROR, res, conn, false, sql);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract cost numbers for topmost plan node. Note we search for a
|
|
|
|
* left paren from the end of the line to avoid being confused by
|
|
|
|
* other uses of parentheses.
|
|
|
|
*/
|
|
|
|
line = PQgetvalue(res, 0, 0);
|
|
|
|
p = strrchr(line, '(');
|
|
|
|
if (p == NULL)
|
|
|
|
elog(ERROR, "could not interpret EXPLAIN output: \"%s\"", line);
|
|
|
|
n = sscanf(p, "(cost=%lf..%lf rows=%lf width=%d)",
|
|
|
|
startup_cost, total_cost, rows, width);
|
|
|
|
if (n != 4)
|
|
|
|
elog(ERROR, "could not interpret EXPLAIN output: \"%s\"", line);
|
|
|
|
}
|
2019-11-01 11:09:52 +01:00
|
|
|
PG_FINALLY();
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
|
|
|
if (res)
|
|
|
|
PQclear(res);
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
|
|
|
}
|
|
|
|
|
2019-04-02 12:20:30 +02:00
|
|
|
/*
|
|
|
|
* Adjust the cost estimates of a foreign grouping path to include the cost of
|
|
|
|
* generating properly-sorted output.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
adjust_foreign_grouping_path_cost(PlannerInfo *root,
|
|
|
|
List *pathkeys,
|
|
|
|
double retrieved_rows,
|
|
|
|
double width,
|
2019-04-02 13:30:45 +02:00
|
|
|
double limit_tuples,
|
2019-04-02 12:20:30 +02:00
|
|
|
Cost *p_startup_cost,
|
|
|
|
Cost *p_run_cost)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If the GROUP BY clause isn't sort-able, the plan chosen by the remote
|
|
|
|
* side is unlikely to generate properly-sorted output, so it would need
|
|
|
|
* an explicit sort; adjust the given costs with cost_sort(). Likewise,
|
|
|
|
* if the GROUP BY clause is sort-able but isn't a superset of the given
|
|
|
|
* pathkeys, adjust the costs with that function. Otherwise, adjust the
|
|
|
|
* costs by applying the same heuristic as for the scan or join case.
|
|
|
|
*/
|
|
|
|
if (!grouping_is_sortable(root->parse->groupClause) ||
|
|
|
|
!pathkeys_contained_in(pathkeys, root->group_pathkeys))
|
|
|
|
{
|
|
|
|
Path sort_path; /* dummy for result of cost_sort */
|
|
|
|
|
|
|
|
cost_sort(&sort_path,
|
|
|
|
root,
|
|
|
|
pathkeys,
|
|
|
|
*p_startup_cost + *p_run_cost,
|
|
|
|
retrieved_rows,
|
|
|
|
width,
|
|
|
|
0.0,
|
|
|
|
work_mem,
|
2019-04-02 13:30:45 +02:00
|
|
|
limit_tuples);
|
2019-04-02 12:20:30 +02:00
|
|
|
|
|
|
|
*p_startup_cost = sort_path.startup_cost;
|
|
|
|
*p_run_cost = sort_path.total_cost - sort_path.startup_cost;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The default extra cost seems too large for foreign-grouping cases;
|
|
|
|
* add 1/4th of that default.
|
|
|
|
*/
|
|
|
|
double sort_multiplier = 1.0 + (DEFAULT_FDW_SORT_MULTIPLIER
|
|
|
|
- 1.0) * 0.25;
|
|
|
|
|
|
|
|
*p_startup_cost *= sort_multiplier;
|
|
|
|
*p_run_cost *= sort_multiplier;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
/*
|
|
|
|
* Detect whether we want to process an EquivalenceClass member.
|
|
|
|
*
|
|
|
|
* This is a callback for use by generate_implied_equalities_for_column.
|
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
ec_member_matches_foreign(PlannerInfo *root, RelOptInfo *rel,
|
|
|
|
EquivalenceClass *ec, EquivalenceMember *em,
|
|
|
|
void *arg)
|
|
|
|
{
|
|
|
|
ec_member_foreign_arg *state = (ec_member_foreign_arg *) arg;
|
|
|
|
Expr *expr = em->em_expr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we've identified what we're processing in the current scan, we only
|
|
|
|
* want to match that expression.
|
|
|
|
*/
|
|
|
|
if (state->current != NULL)
|
|
|
|
return equal(expr, state->current);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Otherwise, ignore anything we've already processed.
|
|
|
|
*/
|
|
|
|
if (list_member(state->already_used, expr))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* This is the new target to process. */
|
|
|
|
state->current = expr;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* Create cursor for node's query with current parameter values.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
create_cursor(ForeignScanState *node)
|
|
|
|
{
|
2013-03-10 19:14:53 +01:00
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
2013-03-22 00:43:59 +01:00
|
|
|
ExprContext *econtext = node->ss.ps.ps_ExprContext;
|
2013-03-10 19:14:53 +01:00
|
|
|
int numParams = fsstate->numParams;
|
|
|
|
const char **values = fsstate->param_values;
|
|
|
|
PGconn *conn = fsstate->conn;
|
2013-02-21 11:26:23 +01:00
|
|
|
StringInfoData buf;
|
|
|
|
PGresult *res;
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/* First, process a pending asynchronous request, if any. */
|
|
|
|
if (fsstate->conn_state->pendingAreq)
|
|
|
|
process_pending_request(fsstate->conn_state->pendingAreq);
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* Construct array of query parameter values in text format. We do the
|
|
|
|
* conversions in the short-lived per-tuple context, so as not to cause a
|
|
|
|
* memory leak over repeated scans.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
2013-03-22 00:43:59 +01:00
|
|
|
if (numParams > 0)
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
2013-03-22 00:43:59 +01:00
|
|
|
MemoryContext oldcontext;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
oldcontext = MemoryContextSwitchTo(econtext->ecxt_per_tuple_memory);
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
process_query_params(econtext,
|
|
|
|
fsstate->param_flinfo,
|
|
|
|
fsstate->param_exprs,
|
|
|
|
values);
|
2013-03-12 02:31:28 +01:00
|
|
|
|
2013-03-22 00:43:59 +01:00
|
|
|
MemoryContextSwitchTo(oldcontext);
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Construct the DECLARE CURSOR command */
|
|
|
|
initStringInfo(&buf);
|
|
|
|
appendStringInfo(&buf, "DECLARE c%u CURSOR FOR\n%s",
|
2013-03-22 05:31:11 +01:00
|
|
|
fsstate->cursor_number, fsstate->query);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
2013-03-22 00:43:59 +01:00
|
|
|
* Notice that we pass NULL for paramTypes, thus forcing the remote server
|
|
|
|
* to infer types for all parameters. Since we explicitly cast every
|
|
|
|
* parameter (see deparse.c), the "inference" is trivial and will produce
|
|
|
|
* the desired result. This allows us to avoid assuming that the remote
|
|
|
|
* server has the same OIDs we do for the parameters' types.
|
2016-04-21 16:46:09 +02:00
|
|
|
*/
|
|
|
|
if (!PQsendQueryParams(conn, buf.data, numParams,
|
|
|
|
NULL, values, NULL, NULL, 0))
|
|
|
|
pgfdw_report_error(ERROR, NULL, conn, false, buf.data);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the result, and check for success.
|
2013-03-22 00:43:59 +01:00
|
|
|
*
|
2013-02-21 11:26:23 +01:00
|
|
|
* We don't use a PG_TRY block here, so be careful not to throw error
|
|
|
|
* without releasing the PGresult.
|
|
|
|
*/
|
2016-04-21 16:46:09 +02:00
|
|
|
res = pgfdw_get_result(conn, buf.data);
|
2013-02-21 11:26:23 +01:00
|
|
|
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
2014-02-04 03:30:02 +01:00
|
|
|
pgfdw_report_error(ERROR, res, conn, true, fsstate->query);
|
2013-02-21 11:26:23 +01:00
|
|
|
PQclear(res);
|
|
|
|
|
|
|
|
/* Mark the cursor as created, and show no tuples have been retrieved */
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->cursor_exists = true;
|
|
|
|
fsstate->tuples = NULL;
|
|
|
|
fsstate->num_tuples = 0;
|
|
|
|
fsstate->next_tuple = 0;
|
|
|
|
fsstate->fetch_ct_2 = 0;
|
|
|
|
fsstate->eof_reached = false;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Clean up */
|
|
|
|
pfree(buf.data);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Fetch some more rows from the node's cursor.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
fetch_more_data(ForeignScanState *node)
|
|
|
|
{
|
2013-03-10 19:14:53 +01:00
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
2013-02-21 11:26:23 +01:00
|
|
|
PGresult *volatile res = NULL;
|
|
|
|
MemoryContext oldcontext;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We'll store the tuples in the batch_cxt. First, flush the previous
|
|
|
|
* batch.
|
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->tuples = NULL;
|
|
|
|
MemoryContextReset(fsstate->batch_cxt);
|
|
|
|
oldcontext = MemoryContextSwitchTo(fsstate->batch_cxt);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* PGresult must be released before leaving this function. */
|
|
|
|
PG_TRY();
|
|
|
|
{
|
2013-03-10 19:14:53 +01:00
|
|
|
PGconn *conn = fsstate->conn;
|
2013-02-21 11:26:23 +01:00
|
|
|
int numrows;
|
|
|
|
int i;
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
if (fsstate->async_capable)
|
|
|
|
{
|
|
|
|
Assert(fsstate->conn_state->pendingAreq);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/*
|
|
|
|
* The query was already sent by an earlier call to
|
|
|
|
* fetch_more_data_begin. So now we just fetch the result.
|
|
|
|
*/
|
|
|
|
res = pgfdw_get_result(conn, fsstate->query);
|
|
|
|
/* On error, report the original query, not the FETCH. */
|
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
|
|
|
pgfdw_report_error(ERROR, res, conn, false, fsstate->query);
|
|
|
|
|
|
|
|
/* Reset per-connection state */
|
|
|
|
fsstate->conn_state->pendingAreq = NULL;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
char sql[64];
|
|
|
|
|
|
|
|
/* This is a regular synchronous fetch. */
|
|
|
|
snprintf(sql, sizeof(sql), "FETCH %d FROM c%u",
|
|
|
|
fsstate->fetch_size, fsstate->cursor_number);
|
|
|
|
|
|
|
|
res = pgfdw_exec_query(conn, sql, fsstate->conn_state);
|
|
|
|
/* On error, report the original query, not the FETCH. */
|
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
|
|
|
pgfdw_report_error(ERROR, res, conn, false, fsstate->query);
|
|
|
|
}
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Convert the data into HeapTuples */
|
|
|
|
numrows = PQntuples(res);
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->tuples = (HeapTuple *) palloc0(numrows * sizeof(HeapTuple));
|
|
|
|
fsstate->num_tuples = numrows;
|
|
|
|
fsstate->next_tuple = 0;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
for (i = 0; i < numrows; i++)
|
|
|
|
{
|
2016-02-10 14:17:43 +01:00
|
|
|
Assert(IsA(node->ss.ps.plan, ForeignScan));
|
2016-02-09 20:00:50 +01:00
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->tuples[i] =
|
2013-02-21 11:26:23 +01:00
|
|
|
make_tuple_from_result_row(res, i,
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->rel,
|
|
|
|
fsstate->attinmeta,
|
2013-03-22 05:31:11 +01:00
|
|
|
fsstate->retrieved_attrs,
|
2016-02-09 20:00:50 +01:00
|
|
|
node,
|
2013-03-10 19:14:53 +01:00
|
|
|
fsstate->temp_cxt);
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Update fetch_ct_2 */
|
2013-03-10 19:14:53 +01:00
|
|
|
if (fsstate->fetch_ct_2 < 2)
|
|
|
|
fsstate->fetch_ct_2++;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Must be EOF if we didn't get as many tuples as we asked for. */
|
2016-02-03 15:01:59 +01:00
|
|
|
fsstate->eof_reached = (numrows < fsstate->fetch_size);
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
2019-11-01 11:09:52 +01:00
|
|
|
PG_FINALLY();
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
|
|
|
if (res)
|
|
|
|
PQclear(res);
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
|
|
|
|
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
}
|
|
|
|
|
2013-03-12 02:31:28 +01:00
|
|
|
/*
|
|
|
|
* Force assorted GUC parameters to settings that ensure that we'll output
|
|
|
|
* data values in a form that is unambiguous to the remote server.
|
|
|
|
*
|
|
|
|
* This is rather expensive and annoying to do once per row, but there's
|
|
|
|
* little choice if we want to be sure values are transmitted accurately;
|
|
|
|
* we can't leave the settings in place between rows for fear of affecting
|
|
|
|
* user-visible computations.
|
|
|
|
*
|
|
|
|
* We use the equivalent of a function SET option to allow the settings to
|
|
|
|
* persist only until the caller calls reset_transmission_modes(). If an
|
|
|
|
* error is thrown in between, guc.c will take care of undoing the settings.
|
|
|
|
*
|
|
|
|
* The return value is the nestlevel that must be passed to
|
|
|
|
* reset_transmission_modes() to undo things.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
set_transmission_modes(void)
|
|
|
|
{
|
|
|
|
int nestlevel = NewGUCNestLevel();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The values set here should match what pg_dump does. See also
|
|
|
|
* configure_remote_session in connection.c.
|
|
|
|
*/
|
|
|
|
if (DateStyle != USE_ISO_DATES)
|
|
|
|
(void) set_config_option("datestyle", "ISO",
|
|
|
|
PGC_USERSET, PGC_S_SESSION,
|
2014-11-24 22:13:11 +01:00
|
|
|
GUC_ACTION_SAVE, true, 0, false);
|
2013-03-12 02:31:28 +01:00
|
|
|
if (IntervalStyle != INTSTYLE_POSTGRES)
|
|
|
|
(void) set_config_option("intervalstyle", "postgres",
|
|
|
|
PGC_USERSET, PGC_S_SESSION,
|
2014-11-24 22:13:11 +01:00
|
|
|
GUC_ACTION_SAVE, true, 0, false);
|
2013-03-12 02:31:28 +01:00
|
|
|
if (extra_float_digits < 3)
|
|
|
|
(void) set_config_option("extra_float_digits", "3",
|
|
|
|
PGC_USERSET, PGC_S_SESSION,
|
2014-11-24 22:13:11 +01:00
|
|
|
GUC_ACTION_SAVE, true, 0, false);
|
2013-03-12 02:31:28 +01:00
|
|
|
|
|
|
|
return nestlevel;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Undo the effects of set_transmission_modes().
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
reset_transmission_modes(int nestlevel)
|
|
|
|
{
|
|
|
|
AtEOXact_GUC(true, nestlevel);
|
|
|
|
}
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* Utility routine to close a cursor.
|
|
|
|
*/
|
|
|
|
static void
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
close_cursor(PGconn *conn, unsigned int cursor_number,
|
|
|
|
PgFdwConnState *conn_state)
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
|
|
|
char sql[64];
|
|
|
|
PGresult *res;
|
|
|
|
|
|
|
|
snprintf(sql, sizeof(sql), "CLOSE c%u", cursor_number);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't use a PG_TRY block here, so be careful not to throw error
|
|
|
|
* without releasing the PGresult.
|
|
|
|
*/
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(conn, sql, conn_state);
|
2013-02-21 11:26:23 +01:00
|
|
|
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
2014-02-04 03:30:02 +01:00
|
|
|
pgfdw_report_error(ERROR, res, conn, true, sql);
|
2013-02-21 11:26:23 +01:00
|
|
|
PQclear(res);
|
|
|
|
}
|
|
|
|
|
2018-04-06 17:29:43 +02:00
|
|
|
/*
|
|
|
|
* create_foreign_modify
|
|
|
|
* Construct an execution state of a foreign insert/update/delete
|
|
|
|
* operation
|
|
|
|
*/
|
|
|
|
static PgFdwModifyState *
|
|
|
|
create_foreign_modify(EState *estate,
|
Fix interaction of foreign tuple routing with remote triggers.
Without these fixes, changes to the inserted tuple made by remote
triggers are ignored when building local RETURNING tuples.
In the core code, call ExecInitRoutingInfo at a later point from
within ExecInitPartitionInfo so that the FDW callback gets invoked
after the returning list has been built. But move CheckValidResultRel
out of ExecInitRoutingInfo so that it can happen at an earlier stage.
In postgres_fdw, refactor assorted deparsing functions to work with
the RTE rather than the PlannerInfo, which saves us having to
construct a fake PlannerInfo in cases where we don't have a real one.
Then, we can pass down a constructed RTE that yields the correct
deparse result when no real one exists. Unfortunately, this
necessitates a hack that understands how the core code manages RT
indexes for update tuple routing, which is ugly, but we don't have a
better idea right now.
Original report, analysis, and patch by Etsuro Fujita. Heavily
refactored by me. Then worked over some more by Amit Langote.
Discussion: http://postgr.es/m/5AD4882B.10002@lab.ntt.co.jp
2018-05-01 19:21:46 +02:00
|
|
|
RangeTblEntry *rte,
|
2018-04-06 17:29:43 +02:00
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
CmdType operation,
|
|
|
|
Plan *subplan,
|
|
|
|
char *query,
|
|
|
|
List *target_attrs,
|
2021-01-20 23:05:46 +01:00
|
|
|
int values_end,
|
2018-04-06 17:29:43 +02:00
|
|
|
bool has_returning,
|
|
|
|
List *retrieved_attrs)
|
|
|
|
{
|
|
|
|
PgFdwModifyState *fmstate;
|
|
|
|
Relation rel = resultRelInfo->ri_RelationDesc;
|
|
|
|
TupleDesc tupdesc = RelationGetDescr(rel);
|
|
|
|
Oid userid;
|
|
|
|
ForeignTable *table;
|
|
|
|
UserMapping *user;
|
|
|
|
AttrNumber n_params;
|
|
|
|
Oid typefnoid;
|
|
|
|
bool isvarlena;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
/* Begin constructing PgFdwModifyState. */
|
|
|
|
fmstate = (PgFdwModifyState *) palloc0(sizeof(PgFdwModifyState));
|
|
|
|
fmstate->rel = rel;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Identify which user to do the remote access as. This should match what
|
|
|
|
* ExecCheckRTEPerms() does.
|
|
|
|
*/
|
|
|
|
userid = rte->checkAsUser ? rte->checkAsUser : GetUserId();
|
|
|
|
|
|
|
|
/* Get info about foreign table. */
|
|
|
|
table = GetForeignTable(RelationGetRelid(rel));
|
|
|
|
user = GetUserMapping(userid, table->serverid);
|
|
|
|
|
|
|
|
/* Open connection; report that we'll create a prepared statement. */
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
fmstate->conn = GetConnection(user, true, &fmstate->conn_state);
|
2018-04-06 17:29:43 +02:00
|
|
|
fmstate->p_name = NULL; /* prepared statement not made yet */
|
|
|
|
|
|
|
|
/* Set up remote query information. */
|
|
|
|
fmstate->query = query;
|
2021-01-20 23:05:46 +01:00
|
|
|
if (operation == CMD_INSERT)
|
2021-05-07 22:29:43 +02:00
|
|
|
{
|
|
|
|
fmstate->query = pstrdup(fmstate->query);
|
2021-01-20 23:05:46 +01:00
|
|
|
fmstate->orig_query = pstrdup(fmstate->query);
|
2021-05-07 22:29:43 +02:00
|
|
|
}
|
2018-04-06 17:29:43 +02:00
|
|
|
fmstate->target_attrs = target_attrs;
|
2021-01-20 23:05:46 +01:00
|
|
|
fmstate->values_end = values_end;
|
2018-04-06 17:29:43 +02:00
|
|
|
fmstate->has_returning = has_returning;
|
|
|
|
fmstate->retrieved_attrs = retrieved_attrs;
|
|
|
|
|
|
|
|
/* Create context for per-tuple temp workspace. */
|
|
|
|
fmstate->temp_cxt = AllocSetContextCreate(estate->es_query_cxt,
|
|
|
|
"postgres_fdw temporary data",
|
|
|
|
ALLOCSET_SMALL_SIZES);
|
|
|
|
|
|
|
|
/* Prepare for input conversion of RETURNING results. */
|
|
|
|
if (fmstate->has_returning)
|
|
|
|
fmstate->attinmeta = TupleDescGetAttInMetadata(tupdesc);
|
|
|
|
|
|
|
|
/* Prepare for output conversion of parameters used in prepared stmt. */
|
|
|
|
n_params = list_length(fmstate->target_attrs) + 1;
|
|
|
|
fmstate->p_flinfo = (FmgrInfo *) palloc0(sizeof(FmgrInfo) * n_params);
|
|
|
|
fmstate->p_nums = 0;
|
|
|
|
|
|
|
|
if (operation == CMD_UPDATE || operation == CMD_DELETE)
|
|
|
|
{
|
|
|
|
Assert(subplan != NULL);
|
|
|
|
|
|
|
|
/* Find the ctid resjunk column in the subplan's result */
|
|
|
|
fmstate->ctidAttno = ExecFindJunkAttributeInTlist(subplan->targetlist,
|
|
|
|
"ctid");
|
|
|
|
if (!AttributeNumberIsValid(fmstate->ctidAttno))
|
|
|
|
elog(ERROR, "could not find junk ctid column");
|
|
|
|
|
|
|
|
/* First transmittable parameter will be ctid */
|
|
|
|
getTypeOutputInfo(TIDOID, &typefnoid, &isvarlena);
|
|
|
|
fmgr_info(typefnoid, &fmstate->p_flinfo[fmstate->p_nums]);
|
|
|
|
fmstate->p_nums++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (operation == CMD_INSERT || operation == CMD_UPDATE)
|
|
|
|
{
|
|
|
|
/* Set up for remaining transmittable parameters */
|
|
|
|
foreach(lc, fmstate->target_attrs)
|
|
|
|
{
|
|
|
|
int attnum = lfirst_int(lc);
|
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1);
|
|
|
|
|
|
|
|
Assert(!attr->attisdropped);
|
|
|
|
|
|
|
|
getTypeOutputInfo(attr->atttypid, &typefnoid, &isvarlena);
|
|
|
|
fmgr_info(typefnoid, &fmstate->p_flinfo[fmstate->p_nums]);
|
|
|
|
fmstate->p_nums++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Assert(fmstate->p_nums <= n_params);
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
/* Set batch_size from foreign server/table options. */
|
|
|
|
if (operation == CMD_INSERT)
|
|
|
|
fmstate->batch_size = get_batch_size_option(rel);
|
|
|
|
|
|
|
|
fmstate->num_slots = 1;
|
|
|
|
|
2019-04-24 11:31:50 +02:00
|
|
|
/* Initialize auxiliary state */
|
|
|
|
fmstate->aux_fmstate = NULL;
|
|
|
|
|
2018-04-06 17:29:43 +02:00
|
|
|
return fmstate;
|
|
|
|
}
|
|
|
|
|
2019-01-17 06:37:33 +01:00
|
|
|
/*
|
|
|
|
* execute_foreign_modify
|
|
|
|
* Perform foreign-table modification as required, and fetch RETURNING
|
|
|
|
* result if any. (This is the shared guts of postgresExecForeignInsert,
|
2021-01-20 23:05:46 +01:00
|
|
|
* postgresExecForeignBatchInsert, postgresExecForeignUpdate, and
|
|
|
|
* postgresExecForeignDelete.)
|
2019-01-17 06:37:33 +01:00
|
|
|
*/
|
2021-01-20 23:05:46 +01:00
|
|
|
static TupleTableSlot **
|
2019-01-17 06:37:33 +01:00
|
|
|
execute_foreign_modify(EState *estate,
|
|
|
|
ResultRelInfo *resultRelInfo,
|
|
|
|
CmdType operation,
|
2021-01-20 23:05:46 +01:00
|
|
|
TupleTableSlot **slots,
|
|
|
|
TupleTableSlot **planSlots,
|
|
|
|
int *numSlots)
|
2019-01-17 06:37:33 +01:00
|
|
|
{
|
|
|
|
PgFdwModifyState *fmstate = (PgFdwModifyState *) resultRelInfo->ri_FdwState;
|
Split create_foreignscan_path() into three functions.
Up to now postgres_fdw has been using create_foreignscan_path() to
generate not only base-relation paths, but also paths for foreign joins
and foreign upperrels. This is wrong, because create_foreignscan_path()
calls get_baserel_parampathinfo() which will only do the right thing for
baserels. It accidentally fails to fail for unparameterized paths, which
are the only ones postgres_fdw (thought it) was handling, but we really
need different APIs for the baserel and join cases.
In HEAD, the best thing to do seems to be to split up the baserel,
joinrel, and upperrel cases into three functions so that they can
have different APIs. I haven't actually given create_foreign_join_path
a different API in this commit: we should spend a bit of time thinking
about just what we want to do there, since perhaps FDWs would want to
do something different from the build-up-a-join-pairwise approach that
get_joinrel_parampathinfo expects. In the meantime, since postgres_fdw
isn't prepared to generate parameterized joins anyway, just give it a
defense against trying to plan joins with lateral refs.
In addition (and this is what triggered this whole mess) fix bug #15613
from Srinivasan S A, by teaching file_fdw and postgres_fdw that plain
baserel foreign paths still have outer refs if the relation has
lateral_relids. Add some assertions in relnode.c to catch future
occurrences of the same error --- in particular, to catch other FDWs
doing that, but also as backstop against core-code mistakes like the
one fixed by commit bdd9a99aa.
Bug #15613 also needs to be fixed in the back branches, but the
appropriate fix will look quite a bit different there, since we don't
want to assume that existing FDWs get the word right away.
Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org
2019-02-07 18:59:47 +01:00
|
|
|
ItemPointer ctid = NULL;
|
2019-01-17 06:37:33 +01:00
|
|
|
const char **p_values;
|
|
|
|
PGresult *res;
|
|
|
|
int n_rows;
|
2021-01-20 23:05:46 +01:00
|
|
|
StringInfoData sql;
|
2019-01-17 06:37:33 +01:00
|
|
|
|
|
|
|
/* The operation should be INSERT, UPDATE, or DELETE */
|
|
|
|
Assert(operation == CMD_INSERT ||
|
|
|
|
operation == CMD_UPDATE ||
|
|
|
|
operation == CMD_DELETE);
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/* First, process a pending asynchronous request, if any. */
|
|
|
|
if (fmstate->conn_state->pendingAreq)
|
|
|
|
process_pending_request(fmstate->conn_state->pendingAreq);
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
/*
|
|
|
|
* If the existing query was deparsed and prepared for a different number
|
|
|
|
* of rows, rebuild it for the proper number.
|
|
|
|
*/
|
|
|
|
if (operation == CMD_INSERT && fmstate->num_slots != *numSlots)
|
|
|
|
{
|
|
|
|
/* Destroy the prepared statement created previously */
|
|
|
|
if (fmstate->p_name)
|
|
|
|
deallocate_query(fmstate);
|
|
|
|
|
|
|
|
/* Build INSERT string with numSlots records in its VALUES clause. */
|
|
|
|
initStringInfo(&sql);
|
|
|
|
rebuildInsertSql(&sql, fmstate->orig_query, fmstate->values_end,
|
|
|
|
fmstate->p_nums, *numSlots - 1);
|
|
|
|
pfree(fmstate->query);
|
|
|
|
fmstate->query = sql.data;
|
|
|
|
fmstate->num_slots = *numSlots;
|
|
|
|
}
|
|
|
|
|
2019-01-17 06:37:33 +01:00
|
|
|
/* Set up the prepared statement on the remote server, if we didn't yet */
|
|
|
|
if (!fmstate->p_name)
|
|
|
|
prepare_foreign_modify(fmstate);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For UPDATE/DELETE, get the ctid that was passed up as a resjunk column
|
|
|
|
*/
|
|
|
|
if (operation == CMD_UPDATE || operation == CMD_DELETE)
|
|
|
|
{
|
|
|
|
Datum datum;
|
|
|
|
bool isNull;
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
datum = ExecGetJunkAttribute(planSlots[0],
|
2019-01-17 06:37:33 +01:00
|
|
|
fmstate->ctidAttno,
|
|
|
|
&isNull);
|
|
|
|
/* shouldn't ever get a null result... */
|
|
|
|
if (isNull)
|
|
|
|
elog(ERROR, "ctid is NULL");
|
|
|
|
ctid = (ItemPointer) DatumGetPointer(datum);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Convert parameters needed by prepared statement to text form */
|
2021-01-20 23:05:46 +01:00
|
|
|
p_values = convert_prep_stmt_params(fmstate, ctid, slots, *numSlots);
|
2019-01-17 06:37:33 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Execute the prepared statement.
|
|
|
|
*/
|
|
|
|
if (!PQsendQueryPrepared(fmstate->conn,
|
|
|
|
fmstate->p_name,
|
2021-01-20 23:05:46 +01:00
|
|
|
fmstate->p_nums * (*numSlots),
|
2019-01-17 06:37:33 +01:00
|
|
|
p_values,
|
|
|
|
NULL,
|
|
|
|
NULL,
|
|
|
|
0))
|
|
|
|
pgfdw_report_error(ERROR, NULL, fmstate->conn, false, fmstate->query);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the result, and check for success.
|
|
|
|
*
|
|
|
|
* We don't use a PG_TRY block here, so be careful not to throw error
|
|
|
|
* without releasing the PGresult.
|
|
|
|
*/
|
|
|
|
res = pgfdw_get_result(fmstate->conn, fmstate->query);
|
|
|
|
if (PQresultStatus(res) !=
|
|
|
|
(fmstate->has_returning ? PGRES_TUPLES_OK : PGRES_COMMAND_OK))
|
|
|
|
pgfdw_report_error(ERROR, res, fmstate->conn, true, fmstate->query);
|
|
|
|
|
|
|
|
/* Check number of rows affected, and fetch RETURNING tuple if any */
|
|
|
|
if (fmstate->has_returning)
|
|
|
|
{
|
2021-01-20 23:05:46 +01:00
|
|
|
Assert(*numSlots == 1);
|
2019-01-17 06:37:33 +01:00
|
|
|
n_rows = PQntuples(res);
|
|
|
|
if (n_rows > 0)
|
2021-01-20 23:05:46 +01:00
|
|
|
store_returning_result(fmstate, slots[0], res);
|
2019-01-17 06:37:33 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
n_rows = atoi(PQcmdTuples(res));
|
|
|
|
|
|
|
|
/* And clean up */
|
|
|
|
PQclear(res);
|
|
|
|
|
|
|
|
MemoryContextReset(fmstate->temp_cxt);
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
*numSlots = n_rows;
|
|
|
|
|
2019-01-17 06:37:33 +01:00
|
|
|
/*
|
|
|
|
* Return NULL if nothing was inserted/updated/deleted on the remote end
|
|
|
|
*/
|
2021-01-20 23:05:46 +01:00
|
|
|
return (n_rows > 0) ? slots : NULL;
|
2019-01-17 06:37:33 +01:00
|
|
|
}
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/*
|
|
|
|
* prepare_foreign_modify
|
|
|
|
* Establish a prepared statement for execution of INSERT/UPDATE/DELETE
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
prepare_foreign_modify(PgFdwModifyState *fmstate)
|
|
|
|
{
|
|
|
|
char prep_name[NAMEDATALEN];
|
|
|
|
char *p_name;
|
|
|
|
PGresult *res;
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/*
|
|
|
|
* The caller would already have processed a pending asynchronous request
|
|
|
|
* if any, so no need to do it here.
|
|
|
|
*/
|
|
|
|
|
2013-03-10 19:14:53 +01:00
|
|
|
/* Construct name we'll use for the prepared statement. */
|
|
|
|
snprintf(prep_name, sizeof(prep_name), "pgsql_fdw_prep_%u",
|
|
|
|
GetPrepStmtNumber(fmstate->conn));
|
|
|
|
p_name = pstrdup(prep_name);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We intentionally do not specify parameter types here, but leave the
|
|
|
|
* remote server to derive them by default. This avoids possible problems
|
|
|
|
* with the remote server using different type OIDs than we do. All of
|
|
|
|
* the prepared statements we use in this module are simple enough that
|
|
|
|
* the remote server will make the right choices.
|
2016-04-21 16:46:09 +02:00
|
|
|
*/
|
|
|
|
if (!PQsendPrepare(fmstate->conn,
|
|
|
|
p_name,
|
|
|
|
fmstate->query,
|
|
|
|
0,
|
|
|
|
NULL))
|
|
|
|
pgfdw_report_error(ERROR, NULL, fmstate->conn, false, fmstate->query);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the result, and check for success.
|
2013-03-10 19:14:53 +01:00
|
|
|
*
|
|
|
|
* We don't use a PG_TRY block here, so be careful not to throw error
|
|
|
|
* without releasing the PGresult.
|
|
|
|
*/
|
2016-04-21 16:46:09 +02:00
|
|
|
res = pgfdw_get_result(fmstate->conn, fmstate->query);
|
2013-03-10 19:14:53 +01:00
|
|
|
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
2014-02-04 03:30:02 +01:00
|
|
|
pgfdw_report_error(ERROR, res, fmstate->conn, true, fmstate->query);
|
2013-03-10 19:14:53 +01:00
|
|
|
PQclear(res);
|
|
|
|
|
|
|
|
/* This action shows that the prepare has been done. */
|
|
|
|
fmstate->p_name = p_name;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* convert_prep_stmt_params
|
|
|
|
* Create array of text strings representing parameter values
|
|
|
|
*
|
|
|
|
* tupleid is ctid to send, or NULL if none
|
|
|
|
* slot is slot to get remaining parameters from, or NULL if none
|
|
|
|
*
|
|
|
|
* Data is constructed in temp_cxt; caller should reset that after use.
|
|
|
|
*/
|
|
|
|
static const char **
|
|
|
|
convert_prep_stmt_params(PgFdwModifyState *fmstate,
|
|
|
|
ItemPointer tupleid,
|
2021-01-20 23:05:46 +01:00
|
|
|
TupleTableSlot **slots,
|
|
|
|
int numSlots)
|
2013-03-10 19:14:53 +01:00
|
|
|
{
|
|
|
|
const char **p_values;
|
2021-01-20 23:05:46 +01:00
|
|
|
int i;
|
|
|
|
int j;
|
2013-03-10 19:14:53 +01:00
|
|
|
int pindex = 0;
|
|
|
|
MemoryContext oldcontext;
|
|
|
|
|
|
|
|
oldcontext = MemoryContextSwitchTo(fmstate->temp_cxt);
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
p_values = (const char **) palloc(sizeof(char *) * fmstate->p_nums * numSlots);
|
|
|
|
|
|
|
|
/* ctid is provided only for UPDATE/DELETE, which don't allow batching */
|
|
|
|
Assert(!(tupleid != NULL && numSlots > 1));
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
/* 1st parameter should be ctid, if it's in use */
|
|
|
|
if (tupleid != NULL)
|
|
|
|
{
|
2021-01-20 23:05:46 +01:00
|
|
|
Assert(numSlots == 1);
|
2013-03-12 02:31:28 +01:00
|
|
|
/* don't need set_transmission_modes for TID output */
|
2013-03-10 19:14:53 +01:00
|
|
|
p_values[pindex] = OutputFunctionCall(&fmstate->p_flinfo[pindex],
|
|
|
|
PointerGetDatum(tupleid));
|
|
|
|
pindex++;
|
|
|
|
}
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
/* get following parameters from slots */
|
|
|
|
if (slots != NULL && fmstate->target_attrs != NIL)
|
2013-03-10 19:14:53 +01:00
|
|
|
{
|
2013-03-12 02:31:28 +01:00
|
|
|
int nestlevel;
|
2013-03-10 19:14:53 +01:00
|
|
|
ListCell *lc;
|
|
|
|
|
2013-03-12 02:31:28 +01:00
|
|
|
nestlevel = set_transmission_modes();
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
for (i = 0; i < numSlots; i++)
|
2013-03-10 19:14:53 +01:00
|
|
|
{
|
2021-01-20 23:05:46 +01:00
|
|
|
j = (tupleid != NULL) ? 1 : 0;
|
|
|
|
foreach(lc, fmstate->target_attrs)
|
|
|
|
{
|
|
|
|
int attnum = lfirst_int(lc);
|
|
|
|
Datum value;
|
|
|
|
bool isnull;
|
2013-03-10 19:14:53 +01:00
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
value = slot_getattr(slots[i], attnum, &isnull);
|
|
|
|
if (isnull)
|
|
|
|
p_values[pindex] = NULL;
|
|
|
|
else
|
|
|
|
p_values[pindex] = OutputFunctionCall(&fmstate->p_flinfo[j],
|
|
|
|
value);
|
|
|
|
pindex++;
|
|
|
|
j++;
|
|
|
|
}
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
2013-03-12 02:31:28 +01:00
|
|
|
|
|
|
|
reset_transmission_modes(nestlevel);
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
Assert(pindex == fmstate->p_nums * numSlots);
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
|
|
|
|
return p_values;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* store_returning_result
|
|
|
|
* Store the result of a RETURNING clause
|
|
|
|
*
|
|
|
|
* On error, be sure to release the PGresult on the way out. Callers do not
|
|
|
|
* have PG_TRY blocks to ensure this happens.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
store_returning_result(PgFdwModifyState *fmstate,
|
|
|
|
TupleTableSlot *slot, PGresult *res)
|
|
|
|
{
|
|
|
|
PG_TRY();
|
|
|
|
{
|
|
|
|
HeapTuple newtup;
|
|
|
|
|
|
|
|
newtup = make_tuple_from_result_row(res, 0,
|
|
|
|
fmstate->rel,
|
|
|
|
fmstate->attinmeta,
|
2013-03-22 05:31:11 +01:00
|
|
|
fmstate->retrieved_attrs,
|
2016-02-09 20:00:50 +01:00
|
|
|
NULL,
|
2013-03-10 19:14:53 +01:00
|
|
|
fmstate->temp_cxt);
|
2019-05-22 18:55:34 +02:00
|
|
|
|
2019-02-27 05:30:28 +01:00
|
|
|
/*
|
|
|
|
* The returning slot will not necessarily be suitable to store
|
|
|
|
* heaptuples directly, so allow for conversion.
|
|
|
|
*/
|
2019-04-19 20:33:37 +02:00
|
|
|
ExecForceStoreHeapTuple(newtup, slot, true);
|
2013-03-10 19:14:53 +01:00
|
|
|
}
|
|
|
|
PG_CATCH();
|
|
|
|
{
|
|
|
|
if (res)
|
|
|
|
PQclear(res);
|
|
|
|
PG_RE_THROW();
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
|
|
|
}
|
|
|
|
|
2018-04-06 17:29:43 +02:00
|
|
|
/*
|
|
|
|
* finish_foreign_modify
|
|
|
|
* Release resources for a foreign insert/update/delete operation
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
finish_foreign_modify(PgFdwModifyState *fmstate)
|
|
|
|
{
|
|
|
|
Assert(fmstate != NULL);
|
|
|
|
|
|
|
|
/* If we created a prepared statement, destroy it */
|
2021-01-20 23:05:46 +01:00
|
|
|
deallocate_query(fmstate);
|
2018-04-06 17:29:43 +02:00
|
|
|
|
|
|
|
/* Release remote connection */
|
|
|
|
ReleaseConnection(fmstate->conn);
|
|
|
|
fmstate->conn = NULL;
|
|
|
|
}
|
|
|
|
|
2021-01-20 23:05:46 +01:00
|
|
|
/*
|
|
|
|
* deallocate_query
|
|
|
|
* Deallocate a prepared statement for a foreign insert/update/delete
|
|
|
|
* operation
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
deallocate_query(PgFdwModifyState *fmstate)
|
|
|
|
{
|
|
|
|
char sql[64];
|
|
|
|
PGresult *res;
|
|
|
|
|
|
|
|
/* do nothing if the query is not allocated */
|
|
|
|
if (!fmstate->p_name)
|
|
|
|
return;
|
|
|
|
|
|
|
|
snprintf(sql, sizeof(sql), "DEALLOCATE %s", fmstate->p_name);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't use a PG_TRY block here, so be careful not to throw error
|
|
|
|
* without releasing the PGresult.
|
|
|
|
*/
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(fmstate->conn, sql, fmstate->conn_state);
|
2021-01-20 23:05:46 +01:00
|
|
|
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
|
|
|
pgfdw_report_error(ERROR, res, fmstate->conn, true, sql);
|
|
|
|
PQclear(res);
|
2021-01-26 10:43:01 +01:00
|
|
|
pfree(fmstate->p_name);
|
2021-01-20 23:05:46 +01:00
|
|
|
fmstate->p_name = NULL;
|
|
|
|
}
|
|
|
|
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
/*
|
|
|
|
* build_remote_returning
|
|
|
|
* Build a RETURNING targetlist of a remote query for performing an
|
|
|
|
* UPDATE/DELETE .. RETURNING on a join directly
|
|
|
|
*/
|
|
|
|
static List *
|
|
|
|
build_remote_returning(Index rtindex, Relation rel, List *returningList)
|
|
|
|
{
|
|
|
|
bool have_wholerow = false;
|
|
|
|
List *tlist = NIL;
|
|
|
|
List *vars;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
Assert(returningList);
|
|
|
|
|
|
|
|
vars = pull_var_clause((Node *) returningList, PVC_INCLUDE_PLACEHOLDERS);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there's a whole-row reference to the target relation, then we'll
|
|
|
|
* need all the columns of the relation.
|
|
|
|
*/
|
|
|
|
foreach(lc, vars)
|
|
|
|
{
|
|
|
|
Var *var = (Var *) lfirst(lc);
|
|
|
|
|
|
|
|
if (IsA(var, Var) &&
|
|
|
|
var->varno == rtindex &&
|
|
|
|
var->varattno == InvalidAttrNumber)
|
|
|
|
{
|
|
|
|
have_wholerow = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (have_wholerow)
|
|
|
|
{
|
|
|
|
TupleDesc tupdesc = RelationGetDescr(rel);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 1; i <= tupdesc->natts; i++)
|
|
|
|
{
|
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupdesc, i - 1);
|
|
|
|
Var *var;
|
|
|
|
|
|
|
|
/* Ignore dropped attributes. */
|
|
|
|
if (attr->attisdropped)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
var = makeVar(rtindex,
|
|
|
|
i,
|
|
|
|
attr->atttypid,
|
|
|
|
attr->atttypmod,
|
|
|
|
attr->attcollation,
|
|
|
|
0);
|
|
|
|
|
|
|
|
tlist = lappend(tlist,
|
|
|
|
makeTargetEntry((Expr *) var,
|
|
|
|
list_length(tlist) + 1,
|
|
|
|
NULL,
|
|
|
|
false));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Now add any remaining columns to tlist. */
|
|
|
|
foreach(lc, vars)
|
|
|
|
{
|
|
|
|
Var *var = (Var *) lfirst(lc);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No need for whole-row references to the target relation. We don't
|
|
|
|
* need system columns other than ctid and oid either, since those are
|
|
|
|
* set locally.
|
|
|
|
*/
|
|
|
|
if (IsA(var, Var) &&
|
|
|
|
var->varno == rtindex &&
|
|
|
|
var->varattno <= InvalidAttrNumber &&
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
var->varattno != SelfItemPointerAttributeNumber)
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
continue; /* don't need it */
|
|
|
|
|
|
|
|
if (tlist_member((Expr *) var, tlist))
|
|
|
|
continue; /* already got it */
|
|
|
|
|
|
|
|
tlist = lappend(tlist,
|
|
|
|
makeTargetEntry((Expr *) var,
|
|
|
|
list_length(tlist) + 1,
|
|
|
|
NULL,
|
|
|
|
false));
|
|
|
|
}
|
|
|
|
|
|
|
|
list_free(vars);
|
|
|
|
|
|
|
|
return tlist;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* rebuild_fdw_scan_tlist
|
|
|
|
* Build new fdw_scan_tlist of given foreign-scan plan node from given
|
|
|
|
* tlist
|
|
|
|
*
|
|
|
|
* There might be columns that the fdw_scan_tlist of the given foreign-scan
|
|
|
|
* plan node contains that the given tlist doesn't. The fdw_scan_tlist would
|
|
|
|
* have contained resjunk columns such as 'ctid' of the target relation and
|
|
|
|
* 'wholerow' of non-target relations, but the tlist might not contain them,
|
|
|
|
* for example. So, adjust the tlist so it contains all the columns specified
|
|
|
|
* in the fdw_scan_tlist; else setrefs.c will get confused.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
rebuild_fdw_scan_tlist(ForeignScan *fscan, List *tlist)
|
|
|
|
{
|
|
|
|
List *new_tlist = tlist;
|
|
|
|
List *old_tlist = fscan->fdw_scan_tlist;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
foreach(lc, old_tlist)
|
|
|
|
{
|
|
|
|
TargetEntry *tle = (TargetEntry *) lfirst(lc);
|
|
|
|
|
|
|
|
if (tlist_member(tle->expr, new_tlist))
|
|
|
|
continue; /* already got it */
|
|
|
|
|
|
|
|
new_tlist = lappend(new_tlist,
|
|
|
|
makeTargetEntry(tle->expr,
|
|
|
|
list_length(new_tlist) + 1,
|
|
|
|
NULL,
|
|
|
|
false));
|
|
|
|
}
|
|
|
|
fscan->fdw_scan_tlist = new_tlist;
|
|
|
|
}
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* Execute a direct UPDATE/DELETE statement.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
execute_dml_stmt(ForeignScanState *node)
|
|
|
|
{
|
|
|
|
PgFdwDirectModifyState *dmstate = (PgFdwDirectModifyState *) node->fdw_state;
|
|
|
|
ExprContext *econtext = node->ss.ps.ps_ExprContext;
|
|
|
|
int numParams = dmstate->numParams;
|
|
|
|
const char **values = dmstate->param_values;
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/* First, process a pending asynchronous request, if any. */
|
|
|
|
if (dmstate->conn_state->pendingAreq)
|
|
|
|
process_pending_request(dmstate->conn_state->pendingAreq);
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* Construct array of query parameter values in text format.
|
|
|
|
*/
|
|
|
|
if (numParams > 0)
|
|
|
|
process_query_params(econtext,
|
|
|
|
dmstate->param_flinfo,
|
|
|
|
dmstate->param_exprs,
|
|
|
|
values);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Notice that we pass NULL for paramTypes, thus forcing the remote server
|
|
|
|
* to infer types for all parameters. Since we explicitly cast every
|
|
|
|
* parameter (see deparse.c), the "inference" is trivial and will produce
|
|
|
|
* the desired result. This allows us to avoid assuming that the remote
|
|
|
|
* server has the same OIDs we do for the parameters' types.
|
2016-04-21 16:46:09 +02:00
|
|
|
*/
|
|
|
|
if (!PQsendQueryParams(dmstate->conn, dmstate->query, numParams,
|
|
|
|
NULL, values, NULL, NULL, 0))
|
|
|
|
pgfdw_report_error(ERROR, NULL, dmstate->conn, false, dmstate->query);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the result, and check for success.
|
2016-03-18 18:48:58 +01:00
|
|
|
*
|
|
|
|
* We don't use a PG_TRY block here, so be careful not to throw error
|
|
|
|
* without releasing the PGresult.
|
|
|
|
*/
|
2016-04-21 16:46:09 +02:00
|
|
|
dmstate->result = pgfdw_get_result(dmstate->conn, dmstate->query);
|
2016-03-18 18:48:58 +01:00
|
|
|
if (PQresultStatus(dmstate->result) !=
|
|
|
|
(dmstate->has_returning ? PGRES_TUPLES_OK : PGRES_COMMAND_OK))
|
|
|
|
pgfdw_report_error(ERROR, dmstate->result, dmstate->conn, true,
|
|
|
|
dmstate->query);
|
|
|
|
|
|
|
|
/* Get the number of rows affected. */
|
|
|
|
if (dmstate->has_returning)
|
|
|
|
dmstate->num_tuples = PQntuples(dmstate->result);
|
|
|
|
else
|
|
|
|
dmstate->num_tuples = atoi(PQcmdTuples(dmstate->result));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the result of a RETURNING clause.
|
|
|
|
*/
|
|
|
|
static TupleTableSlot *
|
|
|
|
get_returning_data(ForeignScanState *node)
|
|
|
|
{
|
|
|
|
PgFdwDirectModifyState *dmstate = (PgFdwDirectModifyState *) node->fdw_state;
|
|
|
|
EState *estate = node->ss.ps.state;
|
2020-10-14 09:58:38 +02:00
|
|
|
ResultRelInfo *resultRelInfo = node->resultRelInfo;
|
2016-03-18 18:48:58 +01:00
|
|
|
TupleTableSlot *slot = node->ss.ss_ScanTupleSlot;
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
TupleTableSlot *resultSlot;
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
Assert(resultRelInfo->ri_projectReturning);
|
|
|
|
|
|
|
|
/* If we didn't get any tuples, must be end of data. */
|
|
|
|
if (dmstate->next_tuple >= dmstate->num_tuples)
|
|
|
|
return ExecClearTuple(slot);
|
|
|
|
|
|
|
|
/* Increment the command es_processed count if necessary. */
|
|
|
|
if (dmstate->set_processed)
|
|
|
|
estate->es_processed += 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Store a RETURNING tuple. If has_returning is false, just emit a dummy
|
|
|
|
* tuple. (has_returning is false when the local query is of the form
|
|
|
|
* "UPDATE/DELETE .. RETURNING 1" for example.)
|
|
|
|
*/
|
|
|
|
if (!dmstate->has_returning)
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
{
|
2016-03-18 18:48:58 +01:00
|
|
|
ExecStoreAllNullTuple(slot);
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
resultSlot = slot;
|
|
|
|
}
|
2016-03-18 18:48:58 +01:00
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* On error, be sure to release the PGresult on the way out. Callers
|
|
|
|
* do not have PG_TRY blocks to ensure this happens.
|
|
|
|
*/
|
|
|
|
PG_TRY();
|
|
|
|
{
|
|
|
|
HeapTuple newtup;
|
|
|
|
|
|
|
|
newtup = make_tuple_from_result_row(dmstate->result,
|
|
|
|
dmstate->next_tuple,
|
|
|
|
dmstate->rel,
|
|
|
|
dmstate->attinmeta,
|
|
|
|
dmstate->retrieved_attrs,
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
node,
|
2016-03-18 18:48:58 +01:00
|
|
|
dmstate->temp_cxt);
|
2018-09-26 01:27:48 +02:00
|
|
|
ExecStoreHeapTuple(newtup, slot, false);
|
2016-03-18 18:48:58 +01:00
|
|
|
}
|
|
|
|
PG_CATCH();
|
|
|
|
{
|
|
|
|
if (dmstate->result)
|
|
|
|
PQclear(dmstate->result);
|
|
|
|
PG_RE_THROW();
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
|
|
|
|
/* Get the updated/deleted tuple. */
|
|
|
|
if (dmstate->rel)
|
|
|
|
resultSlot = slot;
|
|
|
|
else
|
2020-10-14 09:58:38 +02:00
|
|
|
resultSlot = apply_returning_filter(dmstate, resultRelInfo, slot, estate);
|
2016-03-18 18:48:58 +01:00
|
|
|
}
|
|
|
|
dmstate->next_tuple++;
|
|
|
|
|
|
|
|
/* Make slot available for evaluation of the local query RETURNING list. */
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
resultRelInfo->ri_projectReturning->pi_exprContext->ecxt_scantuple =
|
|
|
|
resultSlot;
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
return slot;
|
|
|
|
}
|
|
|
|
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
/*
|
|
|
|
* Initialize a filter to extract an updated/deleted tuple from a scan tuple.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
init_returning_filter(PgFdwDirectModifyState *dmstate,
|
|
|
|
List *fdw_scan_tlist,
|
|
|
|
Index rtindex)
|
|
|
|
{
|
|
|
|
TupleDesc resultTupType = RelationGetDescr(dmstate->resultRel);
|
|
|
|
ListCell *lc;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate the mapping between the fdw_scan_tlist's entries and the
|
|
|
|
* result tuple's attributes.
|
|
|
|
*
|
|
|
|
* The "map" is an array of indexes of the result tuple's attributes in
|
|
|
|
* fdw_scan_tlist, i.e., one entry for every attribute of the result
|
|
|
|
* tuple. We store zero for any attributes that don't have the
|
|
|
|
* corresponding entries in that list, marking that a NULL is needed in
|
|
|
|
* the result tuple.
|
|
|
|
*
|
|
|
|
* Also get the indexes of the entries for ctid and oid if any.
|
|
|
|
*/
|
|
|
|
dmstate->attnoMap = (AttrNumber *)
|
|
|
|
palloc0(resultTupType->natts * sizeof(AttrNumber));
|
|
|
|
|
|
|
|
dmstate->ctidAttno = dmstate->oidAttno = 0;
|
|
|
|
|
|
|
|
i = 1;
|
|
|
|
dmstate->hasSystemCols = false;
|
|
|
|
foreach(lc, fdw_scan_tlist)
|
|
|
|
{
|
|
|
|
TargetEntry *tle = (TargetEntry *) lfirst(lc);
|
|
|
|
Var *var = (Var *) tle->expr;
|
|
|
|
|
|
|
|
Assert(IsA(var, Var));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the Var is a column of the target relation to be retrieved from
|
|
|
|
* the foreign server, get the index of the entry.
|
|
|
|
*/
|
|
|
|
if (var->varno == rtindex &&
|
|
|
|
list_member_int(dmstate->retrieved_attrs, i))
|
|
|
|
{
|
|
|
|
int attrno = var->varattno;
|
|
|
|
|
|
|
|
if (attrno < 0)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We don't retrieve system columns other than ctid and oid.
|
|
|
|
*/
|
|
|
|
if (attrno == SelfItemPointerAttributeNumber)
|
|
|
|
dmstate->ctidAttno = i;
|
|
|
|
else
|
|
|
|
Assert(false);
|
|
|
|
dmstate->hasSystemCols = true;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We don't retrieve whole-row references to the target
|
|
|
|
* relation either.
|
|
|
|
*/
|
|
|
|
Assert(attrno > 0);
|
|
|
|
|
|
|
|
dmstate->attnoMap[attrno - 1] = i;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract and return an updated/deleted tuple from a scan tuple.
|
|
|
|
*/
|
|
|
|
static TupleTableSlot *
|
|
|
|
apply_returning_filter(PgFdwDirectModifyState *dmstate,
|
2020-10-14 09:58:38 +02:00
|
|
|
ResultRelInfo *resultRelInfo,
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
TupleTableSlot *slot,
|
|
|
|
EState *estate)
|
|
|
|
{
|
|
|
|
TupleDesc resultTupType = RelationGetDescr(dmstate->resultRel);
|
|
|
|
TupleTableSlot *resultSlot;
|
|
|
|
Datum *values;
|
|
|
|
bool *isnull;
|
|
|
|
Datum *old_values;
|
|
|
|
bool *old_isnull;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/*
|
2019-02-27 05:30:28 +01:00
|
|
|
* Use the return tuple slot as a place to store the result tuple.
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
*/
|
2020-10-14 09:58:38 +02:00
|
|
|
resultSlot = ExecGetReturningSlot(estate, resultRelInfo);
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract all the values of the scan tuple.
|
|
|
|
*/
|
|
|
|
slot_getallattrs(slot);
|
|
|
|
old_values = slot->tts_values;
|
|
|
|
old_isnull = slot->tts_isnull;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Prepare to build the result tuple.
|
|
|
|
*/
|
|
|
|
ExecClearTuple(resultSlot);
|
|
|
|
values = resultSlot->tts_values;
|
|
|
|
isnull = resultSlot->tts_isnull;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Transpose data into proper fields of the result tuple.
|
|
|
|
*/
|
|
|
|
for (i = 0; i < resultTupType->natts; i++)
|
|
|
|
{
|
|
|
|
int j = dmstate->attnoMap[i];
|
|
|
|
|
|
|
|
if (j == 0)
|
|
|
|
{
|
|
|
|
values[i] = (Datum) 0;
|
|
|
|
isnull[i] = true;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
values[i] = old_values[j - 1];
|
|
|
|
isnull[i] = old_isnull[j - 1];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Build the virtual tuple.
|
|
|
|
*/
|
|
|
|
ExecStoreVirtualTuple(resultSlot);
|
|
|
|
|
|
|
|
/*
|
Split create_foreignscan_path() into three functions.
Up to now postgres_fdw has been using create_foreignscan_path() to
generate not only base-relation paths, but also paths for foreign joins
and foreign upperrels. This is wrong, because create_foreignscan_path()
calls get_baserel_parampathinfo() which will only do the right thing for
baserels. It accidentally fails to fail for unparameterized paths, which
are the only ones postgres_fdw (thought it) was handling, but we really
need different APIs for the baserel and join cases.
In HEAD, the best thing to do seems to be to split up the baserel,
joinrel, and upperrel cases into three functions so that they can
have different APIs. I haven't actually given create_foreign_join_path
a different API in this commit: we should spend a bit of time thinking
about just what we want to do there, since perhaps FDWs would want to
do something different from the build-up-a-join-pairwise approach that
get_joinrel_parampathinfo expects. In the meantime, since postgres_fdw
isn't prepared to generate parameterized joins anyway, just give it a
defense against trying to plan joins with lateral refs.
In addition (and this is what triggered this whole mess) fix bug #15613
from Srinivasan S A, by teaching file_fdw and postgres_fdw that plain
baserel foreign paths still have outer refs if the relation has
lateral_relids. Add some assertions in relnode.c to catch future
occurrences of the same error --- in particular, to catch other FDWs
doing that, but also as backstop against core-code mistakes like the
one fixed by commit bdd9a99aa.
Bug #15613 also needs to be fixed in the back branches, but the
appropriate fix will look quite a bit different there, since we don't
want to assume that existing FDWs get the word right away.
Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org
2019-02-07 18:59:47 +01:00
|
|
|
* If we have any system columns to return, materialize a heap tuple in
|
|
|
|
* the slot from column values set above and install system columns in
|
|
|
|
* that tuple.
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
*/
|
|
|
|
if (dmstate->hasSystemCols)
|
|
|
|
{
|
Rejigger materializing and fetching a HeapTuple from a slot.
Previously materializing a slot always returned a HeapTuple. As
current work aims to reduce the reliance on HeapTuples (so other
storage systems can work efficiently), that needs to change. Thus
split the tasks of materializing a slot (i.e. making it independent
from the underlying storage / other memory contexts) from fetching a
HeapTuple from the slot. For brevity, allow to fetch a HeapTuple from
a slot and materializing the slot at the same time, controlled by a
parameter.
For now some callers of ExecFetchSlotHeapTuple, with materialize =
true, expect that changes to the heap tuple will be reflected in the
underlying slot. Those places will be adapted in due course, so while
not pretty, that's OK for now.
Also rename ExecFetchSlotTuple to ExecFetchSlotHeapTupleDatum and
ExecFetchSlotTupleDatum to ExecFetchSlotHeapTupleDatum, as it's likely
that future storage methods will need similar methods. There already
is ExecFetchSlotMinimalTuple, so the new names make the naming scheme
more coherent.
Author: Ashutosh Bapat and Andres Freund, with changes by Amit Khandekar
Discussion: https://postgr.es/m/20181105210039.hh4vvi4vwoq5ba2q@alap3.anarazel.de
2018-11-15 23:26:14 +01:00
|
|
|
HeapTuple resultTup = ExecFetchSlotHeapTuple(resultSlot, true, NULL);
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
|
|
|
|
/* ctid */
|
|
|
|
if (dmstate->ctidAttno)
|
|
|
|
{
|
|
|
|
ItemPointer ctid = NULL;
|
|
|
|
|
|
|
|
ctid = (ItemPointer) DatumGetPointer(old_values[dmstate->ctidAttno - 1]);
|
|
|
|
resultTup->t_self = *ctid;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* And remaining columns
|
|
|
|
*
|
|
|
|
* Note: since we currently don't allow the target relation to appear
|
|
|
|
* on the nullable side of an outer join, any system columns wouldn't
|
|
|
|
* go to NULL.
|
|
|
|
*
|
|
|
|
* Note: no need to care about tableoid here because it will be
|
|
|
|
* initialized in ExecProcessReturning().
|
|
|
|
*/
|
|
|
|
HeapTupleHeaderSetXmin(resultTup->t_data, InvalidTransactionId);
|
|
|
|
HeapTupleHeaderSetXmax(resultTup->t_data, InvalidTransactionId);
|
|
|
|
HeapTupleHeaderSetCmin(resultTup->t_data, InvalidTransactionId);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* And return the result tuple.
|
|
|
|
*/
|
|
|
|
return resultSlot;
|
|
|
|
}
|
|
|
|
|
2016-03-18 18:48:58 +01:00
|
|
|
/*
|
|
|
|
* Prepare for processing of parameters used in remote query.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
prepare_query_params(PlanState *node,
|
|
|
|
List *fdw_exprs,
|
|
|
|
int numParams,
|
|
|
|
FmgrInfo **param_flinfo,
|
|
|
|
List **param_exprs,
|
|
|
|
const char ***param_values)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
Assert(numParams > 0);
|
|
|
|
|
|
|
|
/* Prepare for output conversion of parameters used in remote query. */
|
|
|
|
*param_flinfo = (FmgrInfo *) palloc0(sizeof(FmgrInfo) * numParams);
|
|
|
|
|
|
|
|
i = 0;
|
|
|
|
foreach(lc, fdw_exprs)
|
|
|
|
{
|
|
|
|
Node *param_expr = (Node *) lfirst(lc);
|
|
|
|
Oid typefnoid;
|
|
|
|
bool isvarlena;
|
|
|
|
|
|
|
|
getTypeOutputInfo(exprType(param_expr), &typefnoid, &isvarlena);
|
|
|
|
fmgr_info(typefnoid, &(*param_flinfo)[i]);
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Prepare remote-parameter expressions for evaluation. (Note: in
|
|
|
|
* practice, we expect that all these expressions will be just Params, so
|
|
|
|
* we could possibly do something more efficient than using the full
|
|
|
|
* expression-eval machinery for this. But probably there would be little
|
|
|
|
* benefit, and it'd require postgres_fdw to know more than is desirable
|
|
|
|
* about Param evaluation.)
|
|
|
|
*/
|
Faster expression evaluation and targetlist projection.
This replaces the old, recursive tree-walk based evaluation, with
non-recursive, opcode dispatch based, expression evaluation.
Projection is now implemented as part of expression evaluation.
This both leads to significant performance improvements, and makes
future just-in-time compilation of expressions easier.
The speed gains primarily come from:
- non-recursive implementation reduces stack usage / overhead
- simple sub-expressions are implemented with a single jump, without
function calls
- sharing some state between different sub-expressions
- reduced amount of indirect/hard to predict memory accesses by laying
out operation metadata sequentially; including the avoidance of
nearly all of the previously used linked lists
- more code has been moved to expression initialization, avoiding
constant re-checks at evaluation time
Future just-in-time compilation (JIT) has become easier, as
demonstrated by released patches intended to be merged in a later
release, for primarily two reasons: Firstly, due to a stricter split
between expression initialization and evaluation, less code has to be
handled by the JIT. Secondly, due to the non-recursive nature of the
generated "instructions", less performance-critical code-paths can
easily be shared between interpreted and compiled evaluation.
The new framework allows for significant future optimizations. E.g.:
- basic infrastructure for to later reduce the per executor-startup
overhead of expression evaluation, by caching state in prepared
statements. That'd be helpful in OLTPish scenarios where
initialization overhead is measurable.
- optimizing the generated "code". A number of proposals for potential
work has already been made.
- optimizing the interpreter. Similarly a number of proposals have
been made here too.
The move of logic into the expression initialization step leads to some
backward-incompatible changes:
- Function permission checks are now done during expression
initialization, whereas previously they were done during
execution. In edge cases this can lead to errors being raised that
previously wouldn't have been, e.g. a NULL array being coerced to a
different array type previously didn't perform checks.
- The set of domain constraints to be checked, is now evaluated once
during expression initialization, previously it was re-built
every time a domain check was evaluated. For normal queries this
doesn't change much, but e.g. for plpgsql functions, which caches
ExprStates, the old set could stick around longer. The behavior
around might still change.
Author: Andres Freund, with significant changes by Tom Lane,
changes by Heikki Linnakangas
Reviewed-By: Tom Lane, Heikki Linnakangas
Discussion: https://postgr.es/m/20161206034955.bh33paeralxbtluv@alap3.anarazel.de
2017-03-14 23:45:36 +01:00
|
|
|
*param_exprs = ExecInitExprList(fdw_exprs, node);
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/* Allocate buffer for text form of query parameters. */
|
|
|
|
*param_values = (const char **) palloc0(numParams * sizeof(char *));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Construct array of query parameter values in text format.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
process_query_params(ExprContext *econtext,
|
|
|
|
FmgrInfo *param_flinfo,
|
|
|
|
List *param_exprs,
|
|
|
|
const char **param_values)
|
|
|
|
{
|
|
|
|
int nestlevel;
|
|
|
|
int i;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
nestlevel = set_transmission_modes();
|
|
|
|
|
|
|
|
i = 0;
|
|
|
|
foreach(lc, param_exprs)
|
|
|
|
{
|
|
|
|
ExprState *expr_state = (ExprState *) lfirst(lc);
|
|
|
|
Datum expr_value;
|
|
|
|
bool isNull;
|
|
|
|
|
|
|
|
/* Evaluate the parameter expression */
|
2017-01-19 23:12:38 +01:00
|
|
|
expr_value = ExecEvalExpr(expr_state, econtext, &isNull);
|
2016-03-18 18:48:58 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get string representation of each parameter value by invoking
|
|
|
|
* type-specific output function, unless the value is null.
|
|
|
|
*/
|
|
|
|
if (isNull)
|
|
|
|
param_values[i] = NULL;
|
|
|
|
else
|
|
|
|
param_values[i] = OutputFunctionCall(¶m_flinfo[i], expr_value);
|
2016-03-21 16:59:49 +01:00
|
|
|
|
|
|
|
i++;
|
2016-03-18 18:48:58 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
reset_transmission_modes(nestlevel);
|
|
|
|
}
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* postgresAnalyzeForeignTable
|
|
|
|
* Test whether analyzing this foreign table is supported
|
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
postgresAnalyzeForeignTable(Relation relation,
|
|
|
|
AcquireSampleRowsFunc *func,
|
|
|
|
BlockNumber *totalpages)
|
|
|
|
{
|
2013-02-22 16:56:06 +01:00
|
|
|
ForeignTable *table;
|
|
|
|
UserMapping *user;
|
|
|
|
PGconn *conn;
|
|
|
|
StringInfoData sql;
|
|
|
|
PGresult *volatile res = NULL;
|
|
|
|
|
|
|
|
/* Return the row-analysis function pointer */
|
2013-02-21 11:26:23 +01:00
|
|
|
*func = postgresAcquireSampleRowsFunc;
|
|
|
|
|
2013-02-22 16:56:06 +01:00
|
|
|
/*
|
2013-03-10 19:14:53 +01:00
|
|
|
* Now we have to get the number of pages. It's annoying that the ANALYZE
|
2013-02-22 16:56:06 +01:00
|
|
|
* API requires us to return that now, because it forces some duplication
|
|
|
|
* of effort between this routine and postgresAcquireSampleRowsFunc. But
|
|
|
|
* it's probably not worth redefining that API at this point.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the connection to use. We do the remote access as the table's
|
|
|
|
* owner, even if the ANALYZE was started by some other user.
|
|
|
|
*/
|
|
|
|
table = GetForeignTable(RelationGetRelid(relation));
|
Avoid multiple foreign server connections when all use same user mapping.
Previously, postgres_fdw's connection cache was keyed by user OID and
server OID, but this can lead to multiple connections when it's not
really necessary. In particular, if all relevant users are mapped to
the public user mapping, then their connection options are certainly
the same, so one connection can be used for all of them.
While we're cleaning things up here, drop the "server" argument to
GetConnection(), which isn't really needed. This saves a few cycles
because callers no longer have to look this up; the function itself
does, but only when establishing a new connection, not when reusing
an existing one.
Ashutosh Bapat, with a few small changes by me.
2016-01-28 18:05:19 +01:00
|
|
|
user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
conn = GetConnection(user, false, NULL);
|
2013-02-22 16:56:06 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Construct command to get page count for relation.
|
|
|
|
*/
|
|
|
|
initStringInfo(&sql);
|
|
|
|
deparseAnalyzeSizeSql(&sql, relation);
|
|
|
|
|
|
|
|
/* In what follows, do not risk leaking any PGresults. */
|
|
|
|
PG_TRY();
|
|
|
|
{
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(conn, sql.data, NULL);
|
2013-02-22 16:56:06 +01:00
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
2014-02-04 03:30:02 +01:00
|
|
|
pgfdw_report_error(ERROR, res, conn, false, sql.data);
|
2013-02-22 16:56:06 +01:00
|
|
|
|
|
|
|
if (PQntuples(res) != 1 || PQnfields(res) != 1)
|
|
|
|
elog(ERROR, "unexpected result from deparseAnalyzeSizeSql query");
|
|
|
|
*totalpages = strtoul(PQgetvalue(res, 0, 0), NULL, 10);
|
|
|
|
}
|
2019-11-01 11:09:52 +01:00
|
|
|
PG_FINALLY();
|
2013-02-22 16:56:06 +01:00
|
|
|
{
|
|
|
|
if (res)
|
|
|
|
PQclear(res);
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
|
|
|
|
|
|
|
ReleaseConnection(conn);
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Acquire a random sample of rows from foreign table managed by postgres_fdw.
|
|
|
|
*
|
|
|
|
* We fetch the whole table from the remote side and pick out some sample rows.
|
|
|
|
*
|
|
|
|
* Selected rows are returned in the caller-allocated array rows[],
|
|
|
|
* which must have at least targrows entries.
|
|
|
|
* The actual number of rows selected is returned as the function result.
|
|
|
|
* We also count the total number of rows in the table and return it into
|
|
|
|
* *totalrows. Note that *totaldeadrows is always set to 0.
|
|
|
|
*
|
|
|
|
* Note that the returned list of rows is not always in order by physical
|
|
|
|
* position in the table. Therefore, correlation estimates derived later
|
|
|
|
* may be meaningless, but it's OK because we don't use the estimates
|
|
|
|
* currently (the planner only pays attention to correlation for indexscans).
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
postgresAcquireSampleRowsFunc(Relation relation, int elevel,
|
|
|
|
HeapTuple *rows, int targrows,
|
|
|
|
double *totalrows,
|
|
|
|
double *totaldeadrows)
|
|
|
|
{
|
|
|
|
PgFdwAnalyzeState astate;
|
|
|
|
ForeignTable *table;
|
2016-02-03 15:01:59 +01:00
|
|
|
ForeignServer *server;
|
2013-02-21 11:26:23 +01:00
|
|
|
UserMapping *user;
|
|
|
|
PGconn *conn;
|
|
|
|
unsigned int cursor_number;
|
|
|
|
StringInfoData sql;
|
|
|
|
PGresult *volatile res = NULL;
|
|
|
|
|
|
|
|
/* Initialize workspace state */
|
|
|
|
astate.rel = relation;
|
|
|
|
astate.attinmeta = TupleDescGetAttInMetadata(RelationGetDescr(relation));
|
|
|
|
|
|
|
|
astate.rows = rows;
|
|
|
|
astate.targrows = targrows;
|
|
|
|
astate.numrows = 0;
|
|
|
|
astate.samplerows = 0;
|
|
|
|
astate.rowstoskip = -1; /* -1 means not set yet */
|
2015-05-15 04:02:54 +02:00
|
|
|
reservoir_init_selection_state(&astate.rstate, targrows);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Remember ANALYZE context, and create a per-tuple temp context */
|
|
|
|
astate.anl_cxt = CurrentMemoryContext;
|
|
|
|
astate.temp_cxt = AllocSetContextCreate(CurrentMemoryContext,
|
|
|
|
"postgres_fdw temporary data",
|
Add macros to make AllocSetContextCreate() calls simpler and safer.
I found that half a dozen (nearly 5%) of our AllocSetContextCreate calls
had typos in the context-sizing parameters. While none of these led to
especially significant problems, they did create minor inefficiencies,
and it's now clear that expecting people to copy-and-paste those calls
accurately is not a great idea. Let's reduce the risk of future errors
by introducing single macros that encapsulate the common use-cases.
Three such macros are enough to cover all but two special-purpose contexts;
those two calls can be left as-is, I think.
While this patch doesn't in itself improve matters for third-party
extensions, it doesn't break anything for them either, and they can
gradually adopt the simplified notation over time.
In passing, change TopMemoryContext to use the default allocation
parameters. Formerly it could only be extended 8K at a time. That was
probably reasonable when this code was written; but nowadays we create
many more contexts than we did then, so that it's not unusual to have a
couple hundred K in TopMemoryContext, even without considering various
dubious code that sticks other things there. There seems no good reason
not to let it use growing blocks like most other contexts.
Back-patch to 9.6, mostly because that's still close enough to HEAD that
it's easy to do so, and keeping the branches in sync can be expected to
avoid some future back-patching pain. The bugs fixed by these changes
don't seem to be significant enough to justify fixing them further back.
Discussion: <21072.1472321324@sss.pgh.pa.us>
2016-08-27 23:50:38 +02:00
|
|
|
ALLOCSET_SMALL_SIZES);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the connection to use. We do the remote access as the table's
|
|
|
|
* owner, even if the ANALYZE was started by some other user.
|
|
|
|
*/
|
|
|
|
table = GetForeignTable(RelationGetRelid(relation));
|
2016-02-03 15:01:59 +01:00
|
|
|
server = GetForeignServer(table->serverid);
|
Avoid multiple foreign server connections when all use same user mapping.
Previously, postgres_fdw's connection cache was keyed by user OID and
server OID, but this can lead to multiple connections when it's not
really necessary. In particular, if all relevant users are mapped to
the public user mapping, then their connection options are certainly
the same, so one connection can be used for all of them.
While we're cleaning things up here, drop the "server" argument to
GetConnection(), which isn't really needed. This saves a few cycles
because callers no longer have to look this up; the function itself
does, but only when establishing a new connection, not when reusing
an existing one.
Ashutosh Bapat, with a few small changes by me.
2016-01-28 18:05:19 +01:00
|
|
|
user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
conn = GetConnection(user, false, NULL);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Construct cursor that retrieves whole rows from remote.
|
|
|
|
*/
|
|
|
|
cursor_number = GetCursorNumber(conn);
|
|
|
|
initStringInfo(&sql);
|
|
|
|
appendStringInfo(&sql, "DECLARE c%u CURSOR FOR ", cursor_number);
|
2013-03-22 05:31:11 +01:00
|
|
|
deparseAnalyzeSql(&sql, relation, &astate.retrieved_attrs);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* In what follows, do not risk leaking any PGresults. */
|
|
|
|
PG_TRY();
|
|
|
|
{
|
2019-07-03 10:51:00 +02:00
|
|
|
char fetch_sql[64];
|
|
|
|
int fetch_size;
|
|
|
|
ListCell *lc;
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(conn, sql.data, NULL);
|
2013-02-21 11:26:23 +01:00
|
|
|
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
2014-02-04 03:30:02 +01:00
|
|
|
pgfdw_report_error(ERROR, res, conn, false, sql.data);
|
2013-02-21 11:26:23 +01:00
|
|
|
PQclear(res);
|
|
|
|
res = NULL;
|
|
|
|
|
2019-07-03 10:51:00 +02:00
|
|
|
/*
|
|
|
|
* Determine the fetch size. The default is arbitrary, but shouldn't
|
|
|
|
* be enormous.
|
|
|
|
*/
|
|
|
|
fetch_size = 100;
|
|
|
|
foreach(lc, server->options)
|
|
|
|
{
|
|
|
|
DefElem *def = (DefElem *) lfirst(lc);
|
|
|
|
|
|
|
|
if (strcmp(def->defname, "fetch_size") == 0)
|
|
|
|
{
|
|
|
|
fetch_size = strtol(defGetString(def), NULL, 10);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
foreach(lc, table->options)
|
|
|
|
{
|
|
|
|
DefElem *def = (DefElem *) lfirst(lc);
|
|
|
|
|
|
|
|
if (strcmp(def->defname, "fetch_size") == 0)
|
|
|
|
{
|
|
|
|
fetch_size = strtol(defGetString(def), NULL, 10);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Construct command to fetch rows from remote. */
|
|
|
|
snprintf(fetch_sql, sizeof(fetch_sql), "FETCH %d FROM c%u",
|
|
|
|
fetch_size, cursor_number);
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/* Retrieve and process rows a batch at a time. */
|
|
|
|
for (;;)
|
|
|
|
{
|
|
|
|
int numrows;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/* Allow users to cancel long query */
|
|
|
|
CHECK_FOR_INTERRUPTS();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* XXX possible future improvement: if rowstoskip is large, we
|
|
|
|
* could issue a MOVE rather than physically fetching the rows,
|
|
|
|
* then just adjust rowstoskip and samplerows appropriately.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* Fetch some rows */
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(conn, fetch_sql, NULL);
|
2013-02-21 11:26:23 +01:00
|
|
|
/* On error, report the original query, not the FETCH. */
|
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
2014-02-04 03:30:02 +01:00
|
|
|
pgfdw_report_error(ERROR, res, conn, false, sql.data);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/* Process whatever we got. */
|
|
|
|
numrows = PQntuples(res);
|
|
|
|
for (i = 0; i < numrows; i++)
|
|
|
|
analyze_row_processor(res, i, &astate);
|
|
|
|
|
|
|
|
PQclear(res);
|
|
|
|
res = NULL;
|
|
|
|
|
|
|
|
/* Must be EOF if we didn't get all the rows requested. */
|
|
|
|
if (numrows < fetch_size)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Close the cursor, just to be tidy. */
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
close_cursor(conn, cursor_number, NULL);
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
|
|
|
PG_CATCH();
|
|
|
|
{
|
|
|
|
if (res)
|
|
|
|
PQclear(res);
|
|
|
|
PG_RE_THROW();
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
|
|
|
|
|
|
|
ReleaseConnection(conn);
|
|
|
|
|
|
|
|
/* We assume that we have no dead tuple. */
|
|
|
|
*totaldeadrows = 0.0;
|
|
|
|
|
|
|
|
/* We've retrieved all living tuples from foreign server. */
|
|
|
|
*totalrows = astate.samplerows;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Emit some interesting relation info
|
|
|
|
*/
|
|
|
|
ereport(elevel,
|
|
|
|
(errmsg("\"%s\": table contains %.0f rows, %d rows in sample",
|
|
|
|
RelationGetRelationName(relation),
|
|
|
|
astate.samplerows, astate.numrows)));
|
|
|
|
|
|
|
|
return astate.numrows;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Collect sample rows from the result of query.
|
|
|
|
* - Use all tuples in sample until target # of samples are collected.
|
|
|
|
* - Subsequently, replace already-sampled tuples randomly.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
analyze_row_processor(PGresult *res, int row, PgFdwAnalyzeState *astate)
|
|
|
|
{
|
|
|
|
int targrows = astate->targrows;
|
|
|
|
int pos; /* array index to store tuple in */
|
|
|
|
MemoryContext oldcontext;
|
|
|
|
|
|
|
|
/* Always increment sample row counter. */
|
|
|
|
astate->samplerows += 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Determine the slot where this sample row should be stored. Set pos to
|
|
|
|
* negative value to indicate the row should be skipped.
|
|
|
|
*/
|
|
|
|
if (astate->numrows < targrows)
|
|
|
|
{
|
|
|
|
/* First targrows rows are always included into the sample */
|
|
|
|
pos = astate->numrows++;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Now we start replacing tuples in the sample until we reach the end
|
|
|
|
* of the relation. Same algorithm as in acquire_sample_rows in
|
|
|
|
* analyze.c; see Jeff Vitter's paper.
|
|
|
|
*/
|
|
|
|
if (astate->rowstoskip < 0)
|
2015-05-15 04:02:54 +02:00
|
|
|
astate->rowstoskip = reservoir_get_next_S(&astate->rstate, astate->samplerows, targrows);
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
if (astate->rowstoskip <= 0)
|
|
|
|
{
|
|
|
|
/* Choose a random reservoir element to replace. */
|
2015-05-15 20:37:10 +02:00
|
|
|
pos = (int) (targrows * sampler_random_fract(astate->rstate.randstate));
|
2013-02-21 11:26:23 +01:00
|
|
|
Assert(pos >= 0 && pos < targrows);
|
|
|
|
heap_freetuple(astate->rows[pos]);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* Skip this tuple. */
|
|
|
|
pos = -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
astate->rowstoskip -= 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pos >= 0)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Create sample tuple from current result row, and store it in the
|
|
|
|
* position determined above. The tuple has to be created in anl_cxt.
|
|
|
|
*/
|
|
|
|
oldcontext = MemoryContextSwitchTo(astate->anl_cxt);
|
|
|
|
|
|
|
|
astate->rows[pos] = make_tuple_from_result_row(res, row,
|
|
|
|
astate->rel,
|
|
|
|
astate->attinmeta,
|
2013-03-22 05:31:11 +01:00
|
|
|
astate->retrieved_attrs,
|
2016-02-09 20:00:50 +01:00
|
|
|
NULL,
|
2013-02-21 11:26:23 +01:00
|
|
|
astate->temp_cxt);
|
|
|
|
|
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-07-10 21:01:31 +02:00
|
|
|
/*
|
|
|
|
* Import a foreign schema
|
|
|
|
*/
|
|
|
|
static List *
|
|
|
|
postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
|
|
|
|
{
|
|
|
|
List *commands = NIL;
|
|
|
|
bool import_collate = true;
|
|
|
|
bool import_default = false;
|
|
|
|
bool import_not_null = true;
|
|
|
|
ForeignServer *server;
|
|
|
|
UserMapping *mapping;
|
|
|
|
PGconn *conn;
|
|
|
|
StringInfoData buf;
|
|
|
|
PGresult *volatile res = NULL;
|
|
|
|
int numrows,
|
|
|
|
i;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
/* Parse statement options */
|
|
|
|
foreach(lc, stmt->options)
|
|
|
|
{
|
|
|
|
DefElem *def = (DefElem *) lfirst(lc);
|
|
|
|
|
|
|
|
if (strcmp(def->defname, "import_collate") == 0)
|
|
|
|
import_collate = defGetBoolean(def);
|
|
|
|
else if (strcmp(def->defname, "import_default") == 0)
|
|
|
|
import_default = defGetBoolean(def);
|
|
|
|
else if (strcmp(def->defname, "import_not_null") == 0)
|
|
|
|
import_not_null = defGetBoolean(def);
|
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FDW_INVALID_OPTION_NAME),
|
|
|
|
errmsg("invalid option \"%s\"", def->defname)));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get connection to the foreign server. Connection manager will
|
|
|
|
* establish new connection if necessary.
|
|
|
|
*/
|
|
|
|
server = GetForeignServer(serverOid);
|
|
|
|
mapping = GetUserMapping(GetUserId(), server->serverid);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
conn = GetConnection(mapping, false, NULL);
|
2014-07-10 21:01:31 +02:00
|
|
|
|
|
|
|
/* Don't attempt to import collation if remote server hasn't got it */
|
|
|
|
if (PQserverVersion(conn) < 90100)
|
|
|
|
import_collate = false;
|
|
|
|
|
|
|
|
/* Create workspace for strings */
|
|
|
|
initStringInfo(&buf);
|
|
|
|
|
|
|
|
/* In what follows, do not risk leaking any PGresults. */
|
|
|
|
PG_TRY();
|
|
|
|
{
|
|
|
|
/* Check that the schema really exists */
|
|
|
|
appendStringInfoString(&buf, "SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = ");
|
|
|
|
deparseStringLiteral(&buf, stmt->remote_schema);
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(conn, buf.data, NULL);
|
2014-07-10 21:01:31 +02:00
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
|
|
|
pgfdw_report_error(ERROR, res, conn, false, buf.data);
|
|
|
|
|
|
|
|
if (PQntuples(res) != 1)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FDW_SCHEMA_NOT_FOUND),
|
|
|
|
errmsg("schema \"%s\" is not present on foreign server \"%s\"",
|
|
|
|
stmt->remote_schema, server->servername)));
|
|
|
|
|
|
|
|
PQclear(res);
|
|
|
|
res = NULL;
|
|
|
|
resetStringInfo(&buf);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Fetch all table data from this schema, possibly restricted by
|
2015-12-31 23:59:10 +01:00
|
|
|
* EXCEPT or LIMIT TO. (We don't actually need to pay any attention
|
|
|
|
* to EXCEPT/LIMIT TO here, because the core code will filter the
|
|
|
|
* statements we return according to those lists anyway. But it
|
|
|
|
* should save a few cycles to not process excluded tables in the
|
|
|
|
* first place.)
|
2014-07-10 21:01:31 +02:00
|
|
|
*
|
2021-04-06 19:32:10 +02:00
|
|
|
* Import table data for partitions only when they are explicitly
|
|
|
|
* specified in LIMIT TO clause. Otherwise ignore them and only
|
|
|
|
* include the definitions of the root partitioned tables to allow
|
|
|
|
* access to the complete remote data set locally in the schema
|
|
|
|
* imported.
|
2017-03-31 21:01:35 +02:00
|
|
|
*
|
2014-07-10 21:01:31 +02:00
|
|
|
* Note: because we run the connection with search_path restricted to
|
|
|
|
* pg_catalog, the format_type() and pg_get_expr() outputs will always
|
|
|
|
* include a schema name for types/functions in other schemas, which
|
|
|
|
* is what we want.
|
|
|
|
*/
|
|
|
|
if (import_collate)
|
|
|
|
appendStringInfoString(&buf,
|
|
|
|
"SELECT relname, "
|
|
|
|
" attname, "
|
|
|
|
" format_type(atttypid, atttypmod), "
|
|
|
|
" attnotnull, "
|
|
|
|
" pg_get_expr(adbin, adrelid), "
|
|
|
|
" collname, "
|
|
|
|
" collnsp.nspname "
|
|
|
|
"FROM pg_class c "
|
|
|
|
" JOIN pg_namespace n ON "
|
|
|
|
" relnamespace = n.oid "
|
|
|
|
" LEFT JOIN pg_attribute a ON "
|
|
|
|
" attrelid = c.oid AND attnum > 0 "
|
|
|
|
" AND NOT attisdropped "
|
|
|
|
" LEFT JOIN pg_attrdef ad ON "
|
|
|
|
" adrelid = c.oid AND adnum = attnum "
|
|
|
|
" LEFT JOIN pg_collation coll ON "
|
|
|
|
" coll.oid = attcollation "
|
|
|
|
" LEFT JOIN pg_namespace collnsp ON "
|
|
|
|
" collnsp.oid = collnamespace ");
|
|
|
|
else
|
|
|
|
appendStringInfoString(&buf,
|
|
|
|
"SELECT relname, "
|
|
|
|
" attname, "
|
|
|
|
" format_type(atttypid, atttypmod), "
|
|
|
|
" attnotnull, "
|
|
|
|
" pg_get_expr(adbin, adrelid), "
|
|
|
|
" NULL, NULL "
|
|
|
|
"FROM pg_class c "
|
|
|
|
" JOIN pg_namespace n ON "
|
|
|
|
" relnamespace = n.oid "
|
|
|
|
" LEFT JOIN pg_attribute a ON "
|
|
|
|
" attrelid = c.oid AND attnum > 0 "
|
|
|
|
" AND NOT attisdropped "
|
|
|
|
" LEFT JOIN pg_attrdef ad ON "
|
|
|
|
" adrelid = c.oid AND adnum = attnum ");
|
|
|
|
|
|
|
|
appendStringInfoString(&buf,
|
2017-03-10 05:36:44 +01:00
|
|
|
"WHERE c.relkind IN ("
|
|
|
|
CppAsString2(RELKIND_RELATION) ","
|
|
|
|
CppAsString2(RELKIND_VIEW) ","
|
|
|
|
CppAsString2(RELKIND_FOREIGN_TABLE) ","
|
2017-03-31 21:01:35 +02:00
|
|
|
CppAsString2(RELKIND_MATVIEW) ","
|
|
|
|
CppAsString2(RELKIND_PARTITIONED_TABLE) ") "
|
2014-07-10 21:01:31 +02:00
|
|
|
" AND n.nspname = ");
|
|
|
|
deparseStringLiteral(&buf, stmt->remote_schema);
|
|
|
|
|
2017-03-31 21:01:35 +02:00
|
|
|
/* Partitions are supported since Postgres 10 */
|
2021-04-06 19:32:10 +02:00
|
|
|
if (PQserverVersion(conn) >= 100000 &&
|
|
|
|
stmt->list_type != FDW_IMPORT_SCHEMA_LIMIT_TO)
|
2017-03-31 21:01:35 +02:00
|
|
|
appendStringInfoString(&buf, " AND NOT c.relispartition ");
|
|
|
|
|
2014-07-10 21:01:31 +02:00
|
|
|
/* Apply restrictions for LIMIT TO and EXCEPT */
|
|
|
|
if (stmt->list_type == FDW_IMPORT_SCHEMA_LIMIT_TO ||
|
|
|
|
stmt->list_type == FDW_IMPORT_SCHEMA_EXCEPT)
|
|
|
|
{
|
|
|
|
bool first_item = true;
|
|
|
|
|
|
|
|
appendStringInfoString(&buf, " AND c.relname ");
|
|
|
|
if (stmt->list_type == FDW_IMPORT_SCHEMA_EXCEPT)
|
|
|
|
appendStringInfoString(&buf, "NOT ");
|
|
|
|
appendStringInfoString(&buf, "IN (");
|
|
|
|
|
|
|
|
/* Append list of table names within IN clause */
|
|
|
|
foreach(lc, stmt->table_list)
|
|
|
|
{
|
|
|
|
RangeVar *rv = (RangeVar *) lfirst(lc);
|
|
|
|
|
|
|
|
if (first_item)
|
|
|
|
first_item = false;
|
|
|
|
else
|
|
|
|
appendStringInfoString(&buf, ", ");
|
|
|
|
deparseStringLiteral(&buf, rv->relname);
|
|
|
|
}
|
2015-05-12 02:38:55 +02:00
|
|
|
appendStringInfoChar(&buf, ')');
|
2014-07-10 21:01:31 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Append ORDER BY at the end of query to ensure output ordering */
|
2015-07-02 11:32:48 +02:00
|
|
|
appendStringInfoString(&buf, " ORDER BY c.relname, a.attnum");
|
2014-07-10 21:01:31 +02:00
|
|
|
|
|
|
|
/* Fetch the data */
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
res = pgfdw_exec_query(conn, buf.data, NULL);
|
2014-07-10 21:01:31 +02:00
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
|
|
|
pgfdw_report_error(ERROR, res, conn, false, buf.data);
|
|
|
|
|
|
|
|
/* Process results */
|
|
|
|
numrows = PQntuples(res);
|
|
|
|
/* note: incrementation of i happens in inner loop's while() test */
|
|
|
|
for (i = 0; i < numrows;)
|
|
|
|
{
|
|
|
|
char *tablename = PQgetvalue(res, i, 0);
|
|
|
|
bool first_item = true;
|
|
|
|
|
|
|
|
resetStringInfo(&buf);
|
|
|
|
appendStringInfo(&buf, "CREATE FOREIGN TABLE %s (\n",
|
|
|
|
quote_identifier(tablename));
|
|
|
|
|
|
|
|
/* Scan all rows for this table */
|
|
|
|
do
|
|
|
|
{
|
|
|
|
char *attname;
|
|
|
|
char *typename;
|
|
|
|
char *attnotnull;
|
|
|
|
char *attdefault;
|
|
|
|
char *collname;
|
|
|
|
char *collnamespace;
|
|
|
|
|
|
|
|
/* If table has no columns, we'll see nulls here */
|
|
|
|
if (PQgetisnull(res, i, 1))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
attname = PQgetvalue(res, i, 1);
|
|
|
|
typename = PQgetvalue(res, i, 2);
|
|
|
|
attnotnull = PQgetvalue(res, i, 3);
|
|
|
|
attdefault = PQgetisnull(res, i, 4) ? (char *) NULL :
|
|
|
|
PQgetvalue(res, i, 4);
|
|
|
|
collname = PQgetisnull(res, i, 5) ? (char *) NULL :
|
|
|
|
PQgetvalue(res, i, 5);
|
|
|
|
collnamespace = PQgetisnull(res, i, 6) ? (char *) NULL :
|
|
|
|
PQgetvalue(res, i, 6);
|
|
|
|
|
|
|
|
if (first_item)
|
|
|
|
first_item = false;
|
|
|
|
else
|
|
|
|
appendStringInfoString(&buf, ",\n");
|
|
|
|
|
|
|
|
/* Print column name and type */
|
|
|
|
appendStringInfo(&buf, " %s %s",
|
|
|
|
quote_identifier(attname),
|
|
|
|
typename);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Add column_name option so that renaming the foreign table's
|
|
|
|
* column doesn't break the association to the underlying
|
|
|
|
* column.
|
|
|
|
*/
|
|
|
|
appendStringInfoString(&buf, " OPTIONS (column_name ");
|
|
|
|
deparseStringLiteral(&buf, attname);
|
2015-05-12 02:38:55 +02:00
|
|
|
appendStringInfoChar(&buf, ')');
|
2014-07-10 21:01:31 +02:00
|
|
|
|
|
|
|
/* Add COLLATE if needed */
|
|
|
|
if (import_collate && collname != NULL && collnamespace != NULL)
|
|
|
|
appendStringInfo(&buf, " COLLATE %s.%s",
|
|
|
|
quote_identifier(collnamespace),
|
|
|
|
quote_identifier(collname));
|
|
|
|
|
|
|
|
/* Add DEFAULT if needed */
|
|
|
|
if (import_default && attdefault != NULL)
|
|
|
|
appendStringInfo(&buf, " DEFAULT %s", attdefault);
|
|
|
|
|
|
|
|
/* Add NOT NULL if needed */
|
|
|
|
if (import_not_null && attnotnull[0] == 't')
|
|
|
|
appendStringInfoString(&buf, " NOT NULL");
|
|
|
|
}
|
|
|
|
while (++i < numrows &&
|
|
|
|
strcmp(PQgetvalue(res, i, 0), tablename) == 0);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Add server name and table-level options. We specify remote
|
|
|
|
* schema and table name as options (the latter to ensure that
|
|
|
|
* renaming the foreign table doesn't break the association).
|
|
|
|
*/
|
|
|
|
appendStringInfo(&buf, "\n) SERVER %s\nOPTIONS (",
|
|
|
|
quote_identifier(server->servername));
|
|
|
|
|
|
|
|
appendStringInfoString(&buf, "schema_name ");
|
|
|
|
deparseStringLiteral(&buf, stmt->remote_schema);
|
|
|
|
appendStringInfoString(&buf, ", table_name ");
|
|
|
|
deparseStringLiteral(&buf, tablename);
|
|
|
|
|
|
|
|
appendStringInfoString(&buf, ");");
|
|
|
|
|
|
|
|
commands = lappend(commands, pstrdup(buf.data));
|
|
|
|
}
|
|
|
|
}
|
2019-11-01 11:09:52 +01:00
|
|
|
PG_FINALLY();
|
2014-07-10 21:01:31 +02:00
|
|
|
{
|
|
|
|
if (res)
|
|
|
|
PQclear(res);
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
|
|
|
|
|
|
|
ReleaseConnection(conn);
|
|
|
|
|
|
|
|
return commands;
|
|
|
|
}
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
|
|
|
* Assess whether the join between inner and outer relations can be pushed down
|
|
|
|
* to the foreign server. As a side effect, save information we obtain in this
|
|
|
|
* function to PgFdwRelationInfo passed in.
|
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel, JoinType jointype,
|
|
|
|
RelOptInfo *outerrel, RelOptInfo *innerrel,
|
|
|
|
JoinPathExtraData *extra)
|
|
|
|
{
|
|
|
|
PgFdwRelationInfo *fpinfo;
|
|
|
|
PgFdwRelationInfo *fpinfo_o;
|
|
|
|
PgFdwRelationInfo *fpinfo_i;
|
|
|
|
ListCell *lc;
|
|
|
|
List *joinclauses;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We support pushing down INNER, LEFT, RIGHT and FULL OUTER joins.
|
|
|
|
* Constructing queries representing SEMI and ANTI joins is hard, hence
|
|
|
|
* not considered right now.
|
|
|
|
*/
|
|
|
|
if (jointype != JOIN_INNER && jointype != JOIN_LEFT &&
|
|
|
|
jointype != JOIN_RIGHT && jointype != JOIN_FULL)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If either of the joining relations is marked as unsafe to pushdown, the
|
|
|
|
* join can not be pushed down.
|
|
|
|
*/
|
|
|
|
fpinfo = (PgFdwRelationInfo *) joinrel->fdw_private;
|
|
|
|
fpinfo_o = (PgFdwRelationInfo *) outerrel->fdw_private;
|
|
|
|
fpinfo_i = (PgFdwRelationInfo *) innerrel->fdw_private;
|
|
|
|
if (!fpinfo_o || !fpinfo_o->pushdown_safe ||
|
|
|
|
!fpinfo_i || !fpinfo_i->pushdown_safe)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If joining relations have local conditions, those conditions are
|
|
|
|
* required to be applied before joining the relations. Hence the join can
|
|
|
|
* not be pushed down.
|
|
|
|
*/
|
|
|
|
if (fpinfo_o->local_conds || fpinfo_i->local_conds)
|
|
|
|
return false;
|
|
|
|
|
2017-04-25 04:50:07 +02:00
|
|
|
/*
|
|
|
|
* Merge FDW options. We might be tempted to do this after we have deemed
|
|
|
|
* the foreign join to be OK. But we must do this beforehand so that we
|
|
|
|
* know which quals can be evaluated on the foreign server, which might
|
|
|
|
* depend on shippable_extensions.
|
|
|
|
*/
|
|
|
|
fpinfo->server = fpinfo_o->server;
|
|
|
|
merge_fdw_options(fpinfo, fpinfo_o, fpinfo_i);
|
|
|
|
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
/*
|
|
|
|
* Separate restrict list into join quals and pushed-down (other) quals.
|
|
|
|
*
|
|
|
|
* Join quals belonging to an outer join must all be shippable, else we
|
|
|
|
* cannot execute the join remotely. Add such quals to 'joinclauses'.
|
|
|
|
*
|
|
|
|
* Add other quals to fpinfo->remote_conds if they are shippable, else to
|
|
|
|
* fpinfo->local_conds. In an inner join it's okay to execute conditions
|
|
|
|
* either locally or remotely; the same is true for pushed-down conditions
|
|
|
|
* at an outer join.
|
|
|
|
*
|
|
|
|
* Note we might return failure after having already scribbled on
|
|
|
|
* fpinfo->remote_conds and fpinfo->local_conds. That's okay because we
|
|
|
|
* won't consult those lists again if we deem the join unshippable.
|
|
|
|
*/
|
|
|
|
joinclauses = NIL;
|
|
|
|
foreach(lc, extra->restrictlist)
|
2016-02-09 20:00:50 +01:00
|
|
|
{
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc);
|
|
|
|
bool is_remote_clause = is_foreign_expr(root, joinrel,
|
|
|
|
rinfo->clause);
|
2016-02-09 20:00:50 +01:00
|
|
|
|
2018-04-20 21:19:16 +02:00
|
|
|
if (IS_OUTER_JOIN(jointype) &&
|
|
|
|
!RINFO_IS_PUSHED_DOWN(rinfo, joinrel->relids))
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
{
|
|
|
|
if (!is_remote_clause)
|
|
|
|
return false;
|
|
|
|
joinclauses = lappend(joinclauses, rinfo);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (is_remote_clause)
|
|
|
|
fpinfo->remote_conds = lappend(fpinfo->remote_conds, rinfo);
|
|
|
|
else
|
|
|
|
fpinfo->local_conds = lappend(fpinfo->local_conds, rinfo);
|
|
|
|
}
|
2016-02-09 20:00:50 +01:00
|
|
|
}
|
|
|
|
|
2016-06-14 17:48:27 +02:00
|
|
|
/*
|
|
|
|
* deparseExplicitTargetList() isn't smart enough to handle anything other
|
|
|
|
* than a Var. In particular, if there's some PlaceHolderVar that would
|
|
|
|
* need to be evaluated within this join tree (because there's an upper
|
|
|
|
* reference to a quantity that may go to NULL as a result of an outer
|
|
|
|
* join), then we can't try to push the join down because we'll fail when
|
|
|
|
* we get to deparseExplicitTargetList(). However, a PlaceHolderVar that
|
|
|
|
* needs to be evaluated *at the top* of this join tree is OK, because we
|
|
|
|
* can do that locally after fetching the results from the remote side.
|
|
|
|
*/
|
|
|
|
foreach(lc, root->placeholder_list)
|
|
|
|
{
|
|
|
|
PlaceHolderInfo *phinfo = lfirst(lc);
|
2018-02-22 16:03:14 +01:00
|
|
|
Relids relids;
|
|
|
|
|
|
|
|
/* PlaceHolderInfo refers to parent relids, not child relids. */
|
|
|
|
relids = IS_OTHER_REL(joinrel) ?
|
|
|
|
joinrel->top_parent_relids : joinrel->relids;
|
2016-06-14 17:48:27 +02:00
|
|
|
|
|
|
|
if (bms_is_subset(phinfo->ph_eval_at, relids) &&
|
|
|
|
bms_nonempty_difference(relids, phinfo->ph_eval_at))
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/* Save the join clauses, for later use. */
|
|
|
|
fpinfo->joinclauses = joinclauses;
|
|
|
|
|
|
|
|
fpinfo->outerrel = outerrel;
|
|
|
|
fpinfo->innerrel = innerrel;
|
|
|
|
fpinfo->jointype = jointype;
|
|
|
|
|
2017-03-16 18:34:59 +01:00
|
|
|
/*
|
|
|
|
* By default, both the input relations are not required to be deparsed as
|
|
|
|
* subqueries, but there might be some relations covered by the input
|
|
|
|
* relations that are required to be deparsed as subqueries, so save the
|
|
|
|
* relids of those relations for later use by the deparser.
|
|
|
|
*/
|
|
|
|
fpinfo->make_outerrel_subquery = false;
|
|
|
|
fpinfo->make_innerrel_subquery = false;
|
|
|
|
Assert(bms_is_subset(fpinfo_o->lower_subquery_rels, outerrel->relids));
|
|
|
|
Assert(bms_is_subset(fpinfo_i->lower_subquery_rels, innerrel->relids));
|
|
|
|
fpinfo->lower_subquery_rels = bms_union(fpinfo_o->lower_subquery_rels,
|
|
|
|
fpinfo_i->lower_subquery_rels);
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
|
|
|
* Pull the other remote conditions from the joining relations into join
|
2016-04-21 05:34:07 +02:00
|
|
|
* clauses or other remote clauses (remote_conds) of this relation
|
2017-03-16 18:34:59 +01:00
|
|
|
* wherever possible. This avoids building subqueries at every join step.
|
2016-02-09 20:00:50 +01:00
|
|
|
*
|
|
|
|
* For an inner join, clauses from both the relations are added to the
|
2016-04-21 05:34:07 +02:00
|
|
|
* other remote clauses. For LEFT and RIGHT OUTER join, the clauses from
|
|
|
|
* the outer side are added to remote_conds since those can be evaluated
|
|
|
|
* after the join is evaluated. The clauses from inner side are added to
|
2016-11-04 14:03:42 +01:00
|
|
|
* the joinclauses, since they need to be evaluated while constructing the
|
2016-02-09 20:00:50 +01:00
|
|
|
* join.
|
|
|
|
*
|
2016-04-21 05:34:07 +02:00
|
|
|
* For a FULL OUTER JOIN, the other clauses from either relation can not
|
|
|
|
* be added to the joinclauses or remote_conds, since each relation acts
|
2017-03-16 18:34:59 +01:00
|
|
|
* as an outer relation for the other.
|
2016-04-21 05:34:07 +02:00
|
|
|
*
|
2016-02-09 20:00:50 +01:00
|
|
|
* The joining sides can not have local conditions, thus no need to test
|
|
|
|
* shippability of the clauses being pulled up.
|
|
|
|
*/
|
|
|
|
switch (jointype)
|
|
|
|
{
|
|
|
|
case JOIN_INNER:
|
|
|
|
fpinfo->remote_conds = list_concat(fpinfo->remote_conds,
|
Rationalize use of list_concat + list_copy combinations.
In the wake of commit 1cff1b95a, the result of list_concat no longer
shares the ListCells of the second input. Therefore, we can replace
"list_concat(x, list_copy(y))" with just "list_concat(x, y)".
To improve call sites that were list_copy'ing the first argument,
or both arguments, invent "list_concat_copy()" which produces a new
list sharing no ListCells with either input. (This is a bit faster
than "list_concat(list_copy(x), y)" because it makes the result list
the right size to start with.)
In call sites that were not list_copy'ing the second argument, the new
semantics mean that we are usually leaking the second List's storage,
since typically there is no remaining pointer to it. We considered
inventing another list_copy variant that would list_free the second
input, but concluded that for most call sites it isn't worth worrying
about, given the relative compactness of the new List representation.
(Note that in cases where such leakage would happen, the old code
already leaked the second List's header; so we're only discussing
the size of the leak not whether there is one. I did adjust two or
three places that had been troubling to free that header so that
they manually free the whole second List.)
Patch by me; thanks to David Rowley for review.
Discussion: https://postgr.es/m/11587.1550975080@sss.pgh.pa.us
2019-08-12 17:20:18 +02:00
|
|
|
fpinfo_i->remote_conds);
|
2016-02-09 20:00:50 +01:00
|
|
|
fpinfo->remote_conds = list_concat(fpinfo->remote_conds,
|
Rationalize use of list_concat + list_copy combinations.
In the wake of commit 1cff1b95a, the result of list_concat no longer
shares the ListCells of the second input. Therefore, we can replace
"list_concat(x, list_copy(y))" with just "list_concat(x, y)".
To improve call sites that were list_copy'ing the first argument,
or both arguments, invent "list_concat_copy()" which produces a new
list sharing no ListCells with either input. (This is a bit faster
than "list_concat(list_copy(x), y)" because it makes the result list
the right size to start with.)
In call sites that were not list_copy'ing the second argument, the new
semantics mean that we are usually leaking the second List's storage,
since typically there is no remaining pointer to it. We considered
inventing another list_copy variant that would list_free the second
input, but concluded that for most call sites it isn't worth worrying
about, given the relative compactness of the new List representation.
(Note that in cases where such leakage would happen, the old code
already leaked the second List's header; so we're only discussing
the size of the leak not whether there is one. I did adjust two or
three places that had been troubling to free that header so that
they manually free the whole second List.)
Patch by me; thanks to David Rowley for review.
Discussion: https://postgr.es/m/11587.1550975080@sss.pgh.pa.us
2019-08-12 17:20:18 +02:00
|
|
|
fpinfo_o->remote_conds);
|
2016-02-09 20:00:50 +01:00
|
|
|
break;
|
|
|
|
|
|
|
|
case JOIN_LEFT:
|
|
|
|
fpinfo->joinclauses = list_concat(fpinfo->joinclauses,
|
Rationalize use of list_concat + list_copy combinations.
In the wake of commit 1cff1b95a, the result of list_concat no longer
shares the ListCells of the second input. Therefore, we can replace
"list_concat(x, list_copy(y))" with just "list_concat(x, y)".
To improve call sites that were list_copy'ing the first argument,
or both arguments, invent "list_concat_copy()" which produces a new
list sharing no ListCells with either input. (This is a bit faster
than "list_concat(list_copy(x), y)" because it makes the result list
the right size to start with.)
In call sites that were not list_copy'ing the second argument, the new
semantics mean that we are usually leaking the second List's storage,
since typically there is no remaining pointer to it. We considered
inventing another list_copy variant that would list_free the second
input, but concluded that for most call sites it isn't worth worrying
about, given the relative compactness of the new List representation.
(Note that in cases where such leakage would happen, the old code
already leaked the second List's header; so we're only discussing
the size of the leak not whether there is one. I did adjust two or
three places that had been troubling to free that header so that
they manually free the whole second List.)
Patch by me; thanks to David Rowley for review.
Discussion: https://postgr.es/m/11587.1550975080@sss.pgh.pa.us
2019-08-12 17:20:18 +02:00
|
|
|
fpinfo_i->remote_conds);
|
2016-02-09 20:00:50 +01:00
|
|
|
fpinfo->remote_conds = list_concat(fpinfo->remote_conds,
|
Rationalize use of list_concat + list_copy combinations.
In the wake of commit 1cff1b95a, the result of list_concat no longer
shares the ListCells of the second input. Therefore, we can replace
"list_concat(x, list_copy(y))" with just "list_concat(x, y)".
To improve call sites that were list_copy'ing the first argument,
or both arguments, invent "list_concat_copy()" which produces a new
list sharing no ListCells with either input. (This is a bit faster
than "list_concat(list_copy(x), y)" because it makes the result list
the right size to start with.)
In call sites that were not list_copy'ing the second argument, the new
semantics mean that we are usually leaking the second List's storage,
since typically there is no remaining pointer to it. We considered
inventing another list_copy variant that would list_free the second
input, but concluded that for most call sites it isn't worth worrying
about, given the relative compactness of the new List representation.
(Note that in cases where such leakage would happen, the old code
already leaked the second List's header; so we're only discussing
the size of the leak not whether there is one. I did adjust two or
three places that had been troubling to free that header so that
they manually free the whole second List.)
Patch by me; thanks to David Rowley for review.
Discussion: https://postgr.es/m/11587.1550975080@sss.pgh.pa.us
2019-08-12 17:20:18 +02:00
|
|
|
fpinfo_o->remote_conds);
|
2016-02-09 20:00:50 +01:00
|
|
|
break;
|
|
|
|
|
|
|
|
case JOIN_RIGHT:
|
|
|
|
fpinfo->joinclauses = list_concat(fpinfo->joinclauses,
|
Rationalize use of list_concat + list_copy combinations.
In the wake of commit 1cff1b95a, the result of list_concat no longer
shares the ListCells of the second input. Therefore, we can replace
"list_concat(x, list_copy(y))" with just "list_concat(x, y)".
To improve call sites that were list_copy'ing the first argument,
or both arguments, invent "list_concat_copy()" which produces a new
list sharing no ListCells with either input. (This is a bit faster
than "list_concat(list_copy(x), y)" because it makes the result list
the right size to start with.)
In call sites that were not list_copy'ing the second argument, the new
semantics mean that we are usually leaking the second List's storage,
since typically there is no remaining pointer to it. We considered
inventing another list_copy variant that would list_free the second
input, but concluded that for most call sites it isn't worth worrying
about, given the relative compactness of the new List representation.
(Note that in cases where such leakage would happen, the old code
already leaked the second List's header; so we're only discussing
the size of the leak not whether there is one. I did adjust two or
three places that had been troubling to free that header so that
they manually free the whole second List.)
Patch by me; thanks to David Rowley for review.
Discussion: https://postgr.es/m/11587.1550975080@sss.pgh.pa.us
2019-08-12 17:20:18 +02:00
|
|
|
fpinfo_o->remote_conds);
|
2016-02-09 20:00:50 +01:00
|
|
|
fpinfo->remote_conds = list_concat(fpinfo->remote_conds,
|
Rationalize use of list_concat + list_copy combinations.
In the wake of commit 1cff1b95a, the result of list_concat no longer
shares the ListCells of the second input. Therefore, we can replace
"list_concat(x, list_copy(y))" with just "list_concat(x, y)".
To improve call sites that were list_copy'ing the first argument,
or both arguments, invent "list_concat_copy()" which produces a new
list sharing no ListCells with either input. (This is a bit faster
than "list_concat(list_copy(x), y)" because it makes the result list
the right size to start with.)
In call sites that were not list_copy'ing the second argument, the new
semantics mean that we are usually leaking the second List's storage,
since typically there is no remaining pointer to it. We considered
inventing another list_copy variant that would list_free the second
input, but concluded that for most call sites it isn't worth worrying
about, given the relative compactness of the new List representation.
(Note that in cases where such leakage would happen, the old code
already leaked the second List's header; so we're only discussing
the size of the leak not whether there is one. I did adjust two or
three places that had been troubling to free that header so that
they manually free the whole second List.)
Patch by me; thanks to David Rowley for review.
Discussion: https://postgr.es/m/11587.1550975080@sss.pgh.pa.us
2019-08-12 17:20:18 +02:00
|
|
|
fpinfo_i->remote_conds);
|
2016-02-09 20:00:50 +01:00
|
|
|
break;
|
|
|
|
|
|
|
|
case JOIN_FULL:
|
2017-03-16 18:34:59 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* In this case, if any of the input relations has conditions, we
|
|
|
|
* need to deparse that relation as a subquery so that the
|
|
|
|
* conditions can be evaluated before the join. Remember it in
|
|
|
|
* the fpinfo of this relation so that the deparser can take
|
|
|
|
* appropriate action. Also, save the relids of base relations
|
|
|
|
* covered by that relation for later use by the deparser.
|
|
|
|
*/
|
|
|
|
if (fpinfo_o->remote_conds)
|
|
|
|
{
|
|
|
|
fpinfo->make_outerrel_subquery = true;
|
|
|
|
fpinfo->lower_subquery_rels =
|
|
|
|
bms_add_members(fpinfo->lower_subquery_rels,
|
|
|
|
outerrel->relids);
|
|
|
|
}
|
|
|
|
if (fpinfo_i->remote_conds)
|
|
|
|
{
|
|
|
|
fpinfo->make_innerrel_subquery = true;
|
|
|
|
fpinfo->lower_subquery_rels =
|
|
|
|
bms_add_members(fpinfo->lower_subquery_rels,
|
|
|
|
innerrel->relids);
|
|
|
|
}
|
2016-02-09 20:00:50 +01:00
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
2017-06-30 20:51:15 +02:00
|
|
|
/* Should not happen, we have just checked this above */
|
2016-02-09 20:00:50 +01:00
|
|
|
elog(ERROR, "unsupported join type %d", jointype);
|
|
|
|
}
|
|
|
|
|
2016-04-21 05:34:07 +02:00
|
|
|
/*
|
2016-06-14 17:48:27 +02:00
|
|
|
* For an inner join, all restrictions can be treated alike. Treating the
|
|
|
|
* pushed down conditions as join conditions allows a top level full outer
|
|
|
|
* join to be deparsed without requiring subqueries.
|
2016-04-21 05:34:07 +02:00
|
|
|
*/
|
|
|
|
if (jointype == JOIN_INNER)
|
|
|
|
{
|
|
|
|
Assert(!fpinfo->joinclauses);
|
|
|
|
fpinfo->joinclauses = fpinfo->remote_conds;
|
|
|
|
fpinfo->remote_conds = NIL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Mark that this join can be pushed down safely */
|
|
|
|
fpinfo->pushdown_safe = true;
|
|
|
|
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
/* Get user mapping */
|
|
|
|
if (fpinfo->use_remote_estimate)
|
|
|
|
{
|
|
|
|
if (fpinfo_o->use_remote_estimate)
|
|
|
|
fpinfo->user = fpinfo_o->user;
|
|
|
|
else
|
|
|
|
fpinfo->user = fpinfo_i->user;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
fpinfo->user = NULL;
|
|
|
|
|
2016-04-21 05:34:07 +02:00
|
|
|
/*
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
* Set # of retrieved rows and cached relation costs to some negative
|
|
|
|
* value, so that we can detect when they are set to some sensible values,
|
|
|
|
* during one (usually the first) of the calls to estimate_path_cost_size.
|
2016-04-21 05:34:07 +02:00
|
|
|
*/
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
fpinfo->retrieved_rows = -1;
|
2016-04-21 05:34:07 +02:00
|
|
|
fpinfo->rel_startup_cost = -1;
|
|
|
|
fpinfo->rel_total_cost = -1;
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
|
|
|
* Set the string describing this join relation to be used in EXPLAIN
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
* output of corresponding ForeignScan. Note that the decoration we add
|
|
|
|
* to the base relation names mustn't include any digits, or it'll confuse
|
|
|
|
* postgresExplainForeignScan.
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
fpinfo->relation_name = psprintf("(%s) %s JOIN (%s)",
|
|
|
|
fpinfo_o->relation_name,
|
|
|
|
get_jointype_name(fpinfo->jointype),
|
|
|
|
fpinfo_i->relation_name);
|
2016-02-09 20:00:50 +01:00
|
|
|
|
2017-03-16 18:34:59 +01:00
|
|
|
/*
|
|
|
|
* Set the relation index. This is defined as the position of this
|
|
|
|
* joinrel in the join_rel_list list plus the length of the rtable list.
|
|
|
|
* Note that since this joinrel is at the end of the join_rel_list list
|
|
|
|
* when we are called, we can get the position by list_length.
|
|
|
|
*/
|
|
|
|
Assert(fpinfo->relation_index == 0); /* shouldn't be set yet */
|
|
|
|
fpinfo->relation_index =
|
|
|
|
list_length(root->parse->rtable) + list_length(root->join_rel_list);
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-03-09 16:51:49 +01:00
|
|
|
static void
|
|
|
|
add_paths_with_pathkeys_for_rel(PlannerInfo *root, RelOptInfo *rel,
|
|
|
|
Path *epq_path)
|
|
|
|
{
|
|
|
|
List *useful_pathkeys_list = NIL; /* List of all pathkeys */
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
useful_pathkeys_list = get_useful_pathkeys_for_relation(root, rel);
|
|
|
|
|
|
|
|
/* Create one path for each set of pathkeys we found above. */
|
|
|
|
foreach(lc, useful_pathkeys_list)
|
|
|
|
{
|
|
|
|
double rows;
|
|
|
|
int width;
|
|
|
|
Cost startup_cost;
|
|
|
|
Cost total_cost;
|
|
|
|
List *useful_pathkeys = lfirst(lc);
|
2018-01-17 22:18:39 +01:00
|
|
|
Path *sorted_epq_path;
|
2016-03-09 16:51:49 +01:00
|
|
|
|
2019-04-02 12:20:30 +02:00
|
|
|
estimate_path_cost_size(root, rel, NIL, useful_pathkeys, NULL,
|
2016-03-09 16:51:49 +01:00
|
|
|
&rows, &width, &startup_cost, &total_cost);
|
|
|
|
|
2018-01-17 22:18:39 +01:00
|
|
|
/*
|
|
|
|
* The EPQ path must be at least as well sorted as the path itself, in
|
|
|
|
* case it gets used as input to a mergejoin.
|
|
|
|
*/
|
|
|
|
sorted_epq_path = epq_path;
|
|
|
|
if (sorted_epq_path != NULL &&
|
|
|
|
!pathkeys_contained_in(useful_pathkeys,
|
|
|
|
sorted_epq_path->pathkeys))
|
|
|
|
sorted_epq_path = (Path *)
|
|
|
|
create_sort_path(root,
|
|
|
|
rel,
|
|
|
|
sorted_epq_path,
|
|
|
|
useful_pathkeys,
|
|
|
|
-1.0);
|
|
|
|
|
Split create_foreignscan_path() into three functions.
Up to now postgres_fdw has been using create_foreignscan_path() to
generate not only base-relation paths, but also paths for foreign joins
and foreign upperrels. This is wrong, because create_foreignscan_path()
calls get_baserel_parampathinfo() which will only do the right thing for
baserels. It accidentally fails to fail for unparameterized paths, which
are the only ones postgres_fdw (thought it) was handling, but we really
need different APIs for the baserel and join cases.
In HEAD, the best thing to do seems to be to split up the baserel,
joinrel, and upperrel cases into three functions so that they can
have different APIs. I haven't actually given create_foreign_join_path
a different API in this commit: we should spend a bit of time thinking
about just what we want to do there, since perhaps FDWs would want to
do something different from the build-up-a-join-pairwise approach that
get_joinrel_parampathinfo expects. In the meantime, since postgres_fdw
isn't prepared to generate parameterized joins anyway, just give it a
defense against trying to plan joins with lateral refs.
In addition (and this is what triggered this whole mess) fix bug #15613
from Srinivasan S A, by teaching file_fdw and postgres_fdw that plain
baserel foreign paths still have outer refs if the relation has
lateral_relids. Add some assertions in relnode.c to catch future
occurrences of the same error --- in particular, to catch other FDWs
doing that, but also as backstop against core-code mistakes like the
one fixed by commit bdd9a99aa.
Bug #15613 also needs to be fixed in the back branches, but the
appropriate fix will look quite a bit different there, since we don't
want to assume that existing FDWs get the word right away.
Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org
2019-02-07 18:59:47 +01:00
|
|
|
if (IS_SIMPLE_REL(rel))
|
|
|
|
add_path(rel, (Path *)
|
|
|
|
create_foreignscan_path(root, rel,
|
|
|
|
NULL,
|
|
|
|
rows,
|
|
|
|
startup_cost,
|
|
|
|
total_cost,
|
|
|
|
useful_pathkeys,
|
|
|
|
rel->lateral_relids,
|
|
|
|
sorted_epq_path,
|
|
|
|
NIL));
|
|
|
|
else
|
|
|
|
add_path(rel, (Path *)
|
|
|
|
create_foreign_join_path(root, rel,
|
|
|
|
NULL,
|
|
|
|
rows,
|
|
|
|
startup_cost,
|
|
|
|
total_cost,
|
|
|
|
useful_pathkeys,
|
|
|
|
rel->lateral_relids,
|
|
|
|
sorted_epq_path,
|
|
|
|
NIL));
|
2016-03-09 16:51:49 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-25 04:50:07 +02:00
|
|
|
/*
|
|
|
|
* Parse options from foreign server and apply them to fpinfo.
|
|
|
|
*
|
|
|
|
* New options might also require tweaking merge_fdw_options().
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
apply_server_options(PgFdwRelationInfo *fpinfo)
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
foreach(lc, fpinfo->server->options)
|
|
|
|
{
|
|
|
|
DefElem *def = (DefElem *) lfirst(lc);
|
|
|
|
|
|
|
|
if (strcmp(def->defname, "use_remote_estimate") == 0)
|
|
|
|
fpinfo->use_remote_estimate = defGetBoolean(def);
|
|
|
|
else if (strcmp(def->defname, "fdw_startup_cost") == 0)
|
|
|
|
fpinfo->fdw_startup_cost = strtod(defGetString(def), NULL);
|
|
|
|
else if (strcmp(def->defname, "fdw_tuple_cost") == 0)
|
|
|
|
fpinfo->fdw_tuple_cost = strtod(defGetString(def), NULL);
|
|
|
|
else if (strcmp(def->defname, "extensions") == 0)
|
|
|
|
fpinfo->shippable_extensions =
|
|
|
|
ExtractExtensionList(defGetString(def), false);
|
|
|
|
else if (strcmp(def->defname, "fetch_size") == 0)
|
|
|
|
fpinfo->fetch_size = strtol(defGetString(def), NULL, 10);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
else if (strcmp(def->defname, "async_capable") == 0)
|
|
|
|
fpinfo->async_capable = defGetBoolean(def);
|
2017-04-25 04:50:07 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Parse options from foreign table and apply them to fpinfo.
|
|
|
|
*
|
|
|
|
* New options might also require tweaking merge_fdw_options().
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
apply_table_options(PgFdwRelationInfo *fpinfo)
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
foreach(lc, fpinfo->table->options)
|
|
|
|
{
|
|
|
|
DefElem *def = (DefElem *) lfirst(lc);
|
|
|
|
|
|
|
|
if (strcmp(def->defname, "use_remote_estimate") == 0)
|
|
|
|
fpinfo->use_remote_estimate = defGetBoolean(def);
|
|
|
|
else if (strcmp(def->defname, "fetch_size") == 0)
|
|
|
|
fpinfo->fetch_size = strtol(defGetString(def), NULL, 10);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
else if (strcmp(def->defname, "async_capable") == 0)
|
|
|
|
fpinfo->async_capable = defGetBoolean(def);
|
2017-04-25 04:50:07 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Merge FDW options from input relations into a new set of options for a join
|
|
|
|
* or an upper rel.
|
|
|
|
*
|
|
|
|
* For a join relation, FDW-specific information about the inner and outer
|
|
|
|
* relations is provided using fpinfo_i and fpinfo_o. For an upper relation,
|
|
|
|
* fpinfo_o provides the information for the input relation; fpinfo_i is
|
|
|
|
* expected to NULL.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
merge_fdw_options(PgFdwRelationInfo *fpinfo,
|
|
|
|
const PgFdwRelationInfo *fpinfo_o,
|
|
|
|
const PgFdwRelationInfo *fpinfo_i)
|
|
|
|
{
|
|
|
|
/* We must always have fpinfo_o. */
|
|
|
|
Assert(fpinfo_o);
|
|
|
|
|
|
|
|
/* fpinfo_i may be NULL, but if present the servers must both match. */
|
|
|
|
Assert(!fpinfo_i ||
|
|
|
|
fpinfo_i->server->serverid == fpinfo_o->server->serverid);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy the server specific FDW options. (For a join, both relations come
|
|
|
|
* from the same server, so the server options should have the same value
|
|
|
|
* for both relations.)
|
|
|
|
*/
|
|
|
|
fpinfo->fdw_startup_cost = fpinfo_o->fdw_startup_cost;
|
|
|
|
fpinfo->fdw_tuple_cost = fpinfo_o->fdw_tuple_cost;
|
|
|
|
fpinfo->shippable_extensions = fpinfo_o->shippable_extensions;
|
|
|
|
fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate;
|
|
|
|
fpinfo->fetch_size = fpinfo_o->fetch_size;
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
fpinfo->async_capable = fpinfo_o->async_capable;
|
2017-04-25 04:50:07 +02:00
|
|
|
|
|
|
|
/* Merge the table level options from either side of the join. */
|
|
|
|
if (fpinfo_i)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We'll prefer to use remote estimates for this join if any table
|
|
|
|
* from either side of the join is using remote estimates. This is
|
|
|
|
* most likely going to be preferred since they're already willing to
|
|
|
|
* pay the price of a round trip to get the remote EXPLAIN. In any
|
|
|
|
* case it's not entirely clear how we might otherwise handle this
|
|
|
|
* best.
|
|
|
|
*/
|
|
|
|
fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate ||
|
|
|
|
fpinfo_i->use_remote_estimate;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set fetch size to maximum of the joining sides, since we are
|
|
|
|
* expecting the rows returned by the join to be proportional to the
|
|
|
|
* relation sizes.
|
|
|
|
*/
|
|
|
|
fpinfo->fetch_size = Max(fpinfo_o->fetch_size, fpinfo_i->fetch_size);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We'll prefer to consider this join async-capable if any table from
|
2021-04-23 05:00:00 +02:00
|
|
|
* either side of the join is considered async-capable. This would be
|
|
|
|
* reasonable because in that case the foreign server would have its
|
|
|
|
* own resources to scan that table asynchronously, and the join could
|
|
|
|
* also be computed asynchronously using the resources.
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
*/
|
|
|
|
fpinfo->async_capable = fpinfo_o->async_capable ||
|
|
|
|
fpinfo_i->async_capable;
|
2017-04-25 04:50:07 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
|
|
|
* postgresGetForeignJoinPaths
|
|
|
|
* Add possible ForeignPath to joinrel, if join is safe to push down.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresGetForeignJoinPaths(PlannerInfo *root,
|
|
|
|
RelOptInfo *joinrel,
|
|
|
|
RelOptInfo *outerrel,
|
|
|
|
RelOptInfo *innerrel,
|
|
|
|
JoinType jointype,
|
|
|
|
JoinPathExtraData *extra)
|
|
|
|
{
|
|
|
|
PgFdwRelationInfo *fpinfo;
|
|
|
|
ForeignPath *joinpath;
|
|
|
|
double rows;
|
|
|
|
int width;
|
|
|
|
Cost startup_cost;
|
|
|
|
Cost total_cost;
|
|
|
|
Path *epq_path; /* Path to create plan to be executed when
|
|
|
|
* EvalPlanQual gets triggered. */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Skip if this join combination has been considered already.
|
|
|
|
*/
|
|
|
|
if (joinrel->fdw_private)
|
|
|
|
return;
|
|
|
|
|
Split create_foreignscan_path() into three functions.
Up to now postgres_fdw has been using create_foreignscan_path() to
generate not only base-relation paths, but also paths for foreign joins
and foreign upperrels. This is wrong, because create_foreignscan_path()
calls get_baserel_parampathinfo() which will only do the right thing for
baserels. It accidentally fails to fail for unparameterized paths, which
are the only ones postgres_fdw (thought it) was handling, but we really
need different APIs for the baserel and join cases.
In HEAD, the best thing to do seems to be to split up the baserel,
joinrel, and upperrel cases into three functions so that they can
have different APIs. I haven't actually given create_foreign_join_path
a different API in this commit: we should spend a bit of time thinking
about just what we want to do there, since perhaps FDWs would want to
do something different from the build-up-a-join-pairwise approach that
get_joinrel_parampathinfo expects. In the meantime, since postgres_fdw
isn't prepared to generate parameterized joins anyway, just give it a
defense against trying to plan joins with lateral refs.
In addition (and this is what triggered this whole mess) fix bug #15613
from Srinivasan S A, by teaching file_fdw and postgres_fdw that plain
baserel foreign paths still have outer refs if the relation has
lateral_relids. Add some assertions in relnode.c to catch future
occurrences of the same error --- in particular, to catch other FDWs
doing that, but also as backstop against core-code mistakes like the
one fixed by commit bdd9a99aa.
Bug #15613 also needs to be fixed in the back branches, but the
appropriate fix will look quite a bit different there, since we don't
want to assume that existing FDWs get the word right away.
Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org
2019-02-07 18:59:47 +01:00
|
|
|
/*
|
|
|
|
* This code does not work for joins with lateral references, since those
|
|
|
|
* must have parameterized paths, which we don't generate yet.
|
|
|
|
*/
|
|
|
|
if (!bms_is_empty(joinrel->lateral_relids))
|
|
|
|
return;
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
/*
|
|
|
|
* Create unfinished PgFdwRelationInfo entry which is used to indicate
|
|
|
|
* that the join relation is already considered, so that we won't waste
|
|
|
|
* time in judging safety of join pushdown and adding the same paths again
|
|
|
|
* if found safe. Once we know that this join can be pushed down, we fill
|
|
|
|
* the entry.
|
|
|
|
*/
|
|
|
|
fpinfo = (PgFdwRelationInfo *) palloc0(sizeof(PgFdwRelationInfo));
|
|
|
|
fpinfo->pushdown_safe = false;
|
|
|
|
joinrel->fdw_private = fpinfo;
|
|
|
|
/* attrs_used is only for base relations. */
|
|
|
|
fpinfo->attrs_used = NULL;
|
|
|
|
|
|
|
|
/*
|
2016-06-17 19:01:14 +02:00
|
|
|
* If there is a possibility that EvalPlanQual will be executed, we need
|
|
|
|
* to be able to reconstruct the row using scans of the base relations.
|
|
|
|
* GetExistingLocalJoinPath will find a suitable path for this purpose in
|
|
|
|
* the path list of the joinrel, if one exists. We must be careful to
|
|
|
|
* call it before adding any ForeignPath, since the ForeignPath might
|
|
|
|
* dominate the only suitable local path available. We also do it before
|
|
|
|
* calling foreign_join_ok(), since that function updates fpinfo and marks
|
|
|
|
* it as pushable if the join is found to be pushable.
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
|
|
|
if (root->parse->commandType == CMD_DELETE ||
|
|
|
|
root->parse->commandType == CMD_UPDATE ||
|
|
|
|
root->rowMarks)
|
|
|
|
{
|
|
|
|
epq_path = GetExistingLocalJoinPath(joinrel);
|
|
|
|
if (!epq_path)
|
|
|
|
{
|
|
|
|
elog(DEBUG3, "could not push down foreign join because a local path suitable for EPQ checks was not found");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
epq_path = NULL;
|
|
|
|
|
|
|
|
if (!foreign_join_ok(root, joinrel, jointype, outerrel, innerrel, extra))
|
|
|
|
{
|
|
|
|
/* Free path required for EPQ if we copied one; we don't need it now */
|
|
|
|
if (epq_path)
|
|
|
|
pfree(epq_path);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Compute the selectivity and cost of the local_conds, so we don't have
|
|
|
|
* to do it over again for each path. The best we can do for these
|
|
|
|
* conditions is to estimate selectivity on the basis of local statistics.
|
|
|
|
* The local conditions are applied after the join has been computed on
|
|
|
|
* the remote side like quals in WHERE clause, so pass jointype as
|
|
|
|
* JOIN_INNER.
|
|
|
|
*/
|
|
|
|
fpinfo->local_conds_sel = clauselist_selectivity(root,
|
|
|
|
fpinfo->local_conds,
|
|
|
|
0,
|
|
|
|
JOIN_INNER,
|
|
|
|
NULL);
|
|
|
|
cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root);
|
|
|
|
|
|
|
|
/*
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
* If we are going to estimate costs locally, estimate the join clause
|
|
|
|
* selectivity here while we have special join info.
|
2016-02-09 20:00:50 +01:00
|
|
|
*/
|
Avoid invalidating all foreign-join cached plans when user mappings change.
We must not push down a foreign join when the foreign tables involved
should be accessed under different user mappings. Previously we tried
to enforce that rule literally during planning, but that meant that the
resulting plans were dependent on the current contents of the
pg_user_mapping catalog, and we had to blow away all cached plans
containing any remote join when anything at all changed in pg_user_mapping.
This could have been improved somewhat, but the fact that a syscache inval
callback has very limited info about what changed made it hard to do better
within that design. Instead, let's change the planner to not consider user
mappings per se, but to allow a foreign join if both RTEs have the same
checkAsUser value. If they do, then they necessarily will use the same
user mapping at runtime, and we don't need to know specifically which one
that is. Post-plan-time changes in pg_user_mapping no longer require any
plan invalidation.
This rule does give up some optimization ability, to wit where two foreign
table references come from views with different owners or one's from a view
and one's directly in the query, but nonetheless the same user mapping
would have applied. We'll sacrifice the first case, but to not regress
more than we have to in the second case, allow a foreign join involving
both zero and nonzero checkAsUser values if the nonzero one is the same as
the prevailing effective userID. In that case, mark the plan as only
runnable by that userID.
The plancache code already had a notion of plans being userID-specific,
in order to support RLS. It was a little confused though, in particular
lacking clarity of thought as to whether it was the rewritten query or just
the finished plan that's dependent on the userID. Rearrange that code so
that it's clearer what depends on which, and so that the same logic applies
to both RLS-injected role dependency and foreign-join-injected role
dependency.
Note that this patch doesn't remove the other issue mentioned in the
original complaint, which is that while we'll reliably stop using a foreign
join if it's disallowed in a new context, we might fail to start using a
foreign join if it's now allowed, but we previously created a generic
cached plan that didn't use one. It was agreed that the chance of winning
that way was not high enough to justify the much larger number of plan
invalidations that would have to occur if we tried to cause it to happen.
In passing, clean up randomly-varying spelling of EXPLAIN commands in
postgres_fdw.sql, and fix a COSTS ON example that had been allowed to
leak into the committed tests.
This reverts most of commits fbe5a3fb7 and 5d4171d1c, which were the
previous attempt at ensuring we wouldn't push down foreign joins that
span permissions contexts.
Etsuro Fujita and Tom Lane
Discussion: <d49c1e5b-f059-20f4-c132-e9752ee0113e@lab.ntt.co.jp>
2016-07-15 23:22:56 +02:00
|
|
|
if (!fpinfo->use_remote_estimate)
|
2016-02-09 20:00:50 +01:00
|
|
|
fpinfo->joinclause_sel = clauselist_selectivity(root, fpinfo->joinclauses,
|
|
|
|
0, fpinfo->jointype,
|
2017-04-07 01:10:51 +02:00
|
|
|
extra->sjinfo);
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
/* Estimate costs for bare join relation */
|
2019-04-02 12:20:30 +02:00
|
|
|
estimate_path_cost_size(root, joinrel, NIL, NIL, NULL,
|
|
|
|
&rows, &width, &startup_cost, &total_cost);
|
2016-02-09 20:00:50 +01:00
|
|
|
/* Now update this information in the joinrel */
|
|
|
|
joinrel->rows = rows;
|
2016-03-14 21:59:59 +01:00
|
|
|
joinrel->reltarget->width = width;
|
2016-02-09 20:00:50 +01:00
|
|
|
fpinfo->rows = rows;
|
|
|
|
fpinfo->width = width;
|
|
|
|
fpinfo->startup_cost = startup_cost;
|
|
|
|
fpinfo->total_cost = total_cost;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create a new join path and add it to the joinrel which represents a
|
|
|
|
* join between foreign tables.
|
|
|
|
*/
|
Split create_foreignscan_path() into three functions.
Up to now postgres_fdw has been using create_foreignscan_path() to
generate not only base-relation paths, but also paths for foreign joins
and foreign upperrels. This is wrong, because create_foreignscan_path()
calls get_baserel_parampathinfo() which will only do the right thing for
baserels. It accidentally fails to fail for unparameterized paths, which
are the only ones postgres_fdw (thought it) was handling, but we really
need different APIs for the baserel and join cases.
In HEAD, the best thing to do seems to be to split up the baserel,
joinrel, and upperrel cases into three functions so that they can
have different APIs. I haven't actually given create_foreign_join_path
a different API in this commit: we should spend a bit of time thinking
about just what we want to do there, since perhaps FDWs would want to
do something different from the build-up-a-join-pairwise approach that
get_joinrel_parampathinfo expects. In the meantime, since postgres_fdw
isn't prepared to generate parameterized joins anyway, just give it a
defense against trying to plan joins with lateral refs.
In addition (and this is what triggered this whole mess) fix bug #15613
from Srinivasan S A, by teaching file_fdw and postgres_fdw that plain
baserel foreign paths still have outer refs if the relation has
lateral_relids. Add some assertions in relnode.c to catch future
occurrences of the same error --- in particular, to catch other FDWs
doing that, but also as backstop against core-code mistakes like the
one fixed by commit bdd9a99aa.
Bug #15613 also needs to be fixed in the back branches, but the
appropriate fix will look quite a bit different there, since we don't
want to assume that existing FDWs get the word right away.
Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org
2019-02-07 18:59:47 +01:00
|
|
|
joinpath = create_foreign_join_path(root,
|
|
|
|
joinrel,
|
|
|
|
NULL, /* default pathtarget */
|
|
|
|
rows,
|
|
|
|
startup_cost,
|
|
|
|
total_cost,
|
|
|
|
NIL, /* no pathkeys */
|
|
|
|
joinrel->lateral_relids,
|
|
|
|
epq_path,
|
|
|
|
NIL); /* no fdw_private */
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
/* Add generated path into joinrel by add_path(). */
|
|
|
|
add_path(joinrel, (Path *) joinpath);
|
|
|
|
|
2016-03-09 16:51:49 +01:00
|
|
|
/* Consider pathkeys for the join relation */
|
|
|
|
add_paths_with_pathkeys_for_rel(root, joinrel, epq_path);
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
/* XXX Consider parameterized paths for the join relation */
|
|
|
|
}
|
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
/*
|
|
|
|
* Assess whether the aggregation, grouping and having operations can be pushed
|
|
|
|
* down to the foreign server. As a side effect, save information we obtain in
|
|
|
|
* this function to PgFdwRelationInfo of the input relation.
|
|
|
|
*/
|
|
|
|
static bool
|
2018-04-02 16:51:50 +02:00
|
|
|
foreign_grouping_ok(PlannerInfo *root, RelOptInfo *grouped_rel,
|
|
|
|
Node *havingQual)
|
2016-10-21 15:54:29 +02:00
|
|
|
{
|
|
|
|
Query *query = root->parse;
|
|
|
|
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) grouped_rel->fdw_private;
|
2018-04-02 16:51:50 +02:00
|
|
|
PathTarget *grouping_target = grouped_rel->reltarget;
|
2016-10-21 15:54:29 +02:00
|
|
|
PgFdwRelationInfo *ofpinfo;
|
|
|
|
ListCell *lc;
|
|
|
|
int i;
|
|
|
|
List *tlist = NIL;
|
|
|
|
|
2018-01-12 22:52:49 +01:00
|
|
|
/* We currently don't support pushing Grouping Sets. */
|
2016-10-21 15:54:29 +02:00
|
|
|
if (query->groupingSets)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Get the fpinfo of the underlying scan relation. */
|
|
|
|
ofpinfo = (PgFdwRelationInfo *) fpinfo->outerrel->fdw_private;
|
|
|
|
|
|
|
|
/*
|
2018-01-12 22:52:49 +01:00
|
|
|
* If underlying scan relation has any local conditions, those conditions
|
2016-10-21 15:54:29 +02:00
|
|
|
* are required to be applied before performing aggregation. Hence the
|
|
|
|
* aggregate cannot be pushed down.
|
|
|
|
*/
|
|
|
|
if (ofpinfo->local_conds)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
2018-01-12 22:52:49 +01:00
|
|
|
* Examine grouping expressions, as well as other expressions we'd need to
|
|
|
|
* compute, and check whether they are safe to push down to the foreign
|
|
|
|
* server. All GROUP BY expressions will be part of the grouping target
|
|
|
|
* and thus there is no need to search for them separately. Add grouping
|
|
|
|
* expressions into target list which will be passed to foreign server.
|
Avoid postgres_fdw crash for a targetlist entry that's just a Param.
foreign_grouping_ok() is willing to put fairly arbitrary expressions into
the targetlist of a remote SELECT that's doing grouping or aggregation on
the remote side, including expressions that have no foreign component to
them at all. This is possibly a bit dubious from an efficiency standpoint;
but it rises to the level of a crash-causing bug if the expression is just
a Param or non-foreign Var. In that case, the expression will necessarily
also appear in the fdw_exprs list of values we need to send to the remote
server, and then setrefs.c's set_foreignscan_references will mistakenly
replace the fdw_exprs entry with a Var referencing the targetlist result.
The root cause of this problem is bad design in commit e7cb7ee14: it put
logic into set_foreignscan_references that IMV is postgres_fdw-specific,
and yet this bug shows that it isn't postgres_fdw-specific enough. The
transformation being done on fdw_exprs assumes that fdw_exprs is to be
evaluated with the fdw_scan_tlist as input, which is not how postgres_fdw
uses it; yet it could be the right thing for some other FDW. (In the
bigger picture, setrefs.c has no business assuming this for the other
expression fields of a ForeignScan either.)
The right fix therefore would be to expand the FDW API so that the
FDW could inform setrefs.c how it intends to evaluate these various
expressions. We can't change that in the back branches though, and we
also can't just summarily change setrefs.c's behavior there, or we're
likely to break external FDWs.
As a stopgap, therefore, hack up postgres_fdw so that it won't attempt
to send targetlist entries that look exactly like the fdw_exprs entries
they'd produce. In most cases this actually produces a superior plan,
IMO, with less data needing to be transmitted and returned; so we probably
ought to think harder about whether we should ship tlist expressions at
all when they don't contain any foreign Vars or Aggs. But that's an
optimization not a bug fix so I left it for later. One case where this
produces an inferior plan is where the expression in question is actually
a GROUP BY expression: then the restriction prevents us from using remote
grouping. It might be possible to work around that (since that would
reduce to group-by-a-constant on the remote side); but it seems like a
pretty unlikely corner case, so I'm not sure it's worth expending effort
solely to improve that. In any case the right long-term answer is to fix
the API as sketched above, and then revert this hack.
Per bug #15781 from Sean Johnston. Back-patch to v10 where the problem
was introduced.
Discussion: https://postgr.es/m/15781-2601b1002bad087c@postgresql.org
2019-04-27 19:15:54 +02:00
|
|
|
*
|
|
|
|
* A tricky fine point is that we must not put any expression into the
|
|
|
|
* target list that is just a foreign param (that is, something that
|
|
|
|
* deparse.c would conclude has to be sent to the foreign server). If we
|
|
|
|
* do, the expression will also appear in the fdw_exprs list of the plan
|
|
|
|
* node, and setrefs.c will get confused and decide that the fdw_exprs
|
|
|
|
* entry is actually a reference to the fdw_scan_tlist entry, resulting in
|
|
|
|
* a broken plan. Somewhat oddly, it's OK if the expression contains such
|
|
|
|
* a node, as long as it's not at top level; then no match is possible.
|
2016-10-21 15:54:29 +02:00
|
|
|
*/
|
|
|
|
i = 0;
|
|
|
|
foreach(lc, grouping_target->exprs)
|
|
|
|
{
|
|
|
|
Expr *expr = (Expr *) lfirst(lc);
|
|
|
|
Index sgref = get_pathtarget_sortgroupref(grouping_target, i);
|
|
|
|
ListCell *l;
|
|
|
|
|
|
|
|
/* Check whether this expression is part of GROUP BY clause */
|
|
|
|
if (sgref && get_sortgroupref_clause_noerr(sgref, query->groupClause))
|
|
|
|
{
|
2018-01-12 22:52:49 +01:00
|
|
|
TargetEntry *tle;
|
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
/*
|
2018-01-12 22:52:49 +01:00
|
|
|
* If any GROUP BY expression is not shippable, then we cannot
|
2016-10-21 15:54:29 +02:00
|
|
|
* push down aggregation to the foreign server.
|
|
|
|
*/
|
|
|
|
if (!is_foreign_expr(root, grouped_rel, expr))
|
|
|
|
return false;
|
|
|
|
|
Avoid postgres_fdw crash for a targetlist entry that's just a Param.
foreign_grouping_ok() is willing to put fairly arbitrary expressions into
the targetlist of a remote SELECT that's doing grouping or aggregation on
the remote side, including expressions that have no foreign component to
them at all. This is possibly a bit dubious from an efficiency standpoint;
but it rises to the level of a crash-causing bug if the expression is just
a Param or non-foreign Var. In that case, the expression will necessarily
also appear in the fdw_exprs list of values we need to send to the remote
server, and then setrefs.c's set_foreignscan_references will mistakenly
replace the fdw_exprs entry with a Var referencing the targetlist result.
The root cause of this problem is bad design in commit e7cb7ee14: it put
logic into set_foreignscan_references that IMV is postgres_fdw-specific,
and yet this bug shows that it isn't postgres_fdw-specific enough. The
transformation being done on fdw_exprs assumes that fdw_exprs is to be
evaluated with the fdw_scan_tlist as input, which is not how postgres_fdw
uses it; yet it could be the right thing for some other FDW. (In the
bigger picture, setrefs.c has no business assuming this for the other
expression fields of a ForeignScan either.)
The right fix therefore would be to expand the FDW API so that the
FDW could inform setrefs.c how it intends to evaluate these various
expressions. We can't change that in the back branches though, and we
also can't just summarily change setrefs.c's behavior there, or we're
likely to break external FDWs.
As a stopgap, therefore, hack up postgres_fdw so that it won't attempt
to send targetlist entries that look exactly like the fdw_exprs entries
they'd produce. In most cases this actually produces a superior plan,
IMO, with less data needing to be transmitted and returned; so we probably
ought to think harder about whether we should ship tlist expressions at
all when they don't contain any foreign Vars or Aggs. But that's an
optimization not a bug fix so I left it for later. One case where this
produces an inferior plan is where the expression in question is actually
a GROUP BY expression: then the restriction prevents us from using remote
grouping. It might be possible to work around that (since that would
reduce to group-by-a-constant on the remote side); but it seems like a
pretty unlikely corner case, so I'm not sure it's worth expending effort
solely to improve that. In any case the right long-term answer is to fix
the API as sketched above, and then revert this hack.
Per bug #15781 from Sean Johnston. Back-patch to v10 where the problem
was introduced.
Discussion: https://postgr.es/m/15781-2601b1002bad087c@postgresql.org
2019-04-27 19:15:54 +02:00
|
|
|
/*
|
|
|
|
* If it would be a foreign param, we can't put it into the tlist,
|
|
|
|
* so we have to fail.
|
|
|
|
*/
|
|
|
|
if (is_foreign_param(root, grouped_rel, expr))
|
|
|
|
return false;
|
|
|
|
|
2018-01-12 22:52:49 +01:00
|
|
|
/*
|
|
|
|
* Pushable, so add to tlist. We need to create a TLE for this
|
|
|
|
* expression and apply the sortgroupref to it. We cannot use
|
|
|
|
* add_to_flat_tlist() here because that avoids making duplicate
|
|
|
|
* entries in the tlist. If there are duplicate entries with
|
|
|
|
* distinct sortgrouprefs, we have to duplicate that situation in
|
|
|
|
* the output tlist.
|
|
|
|
*/
|
|
|
|
tle = makeTargetEntry(expr, list_length(tlist) + 1, NULL, false);
|
|
|
|
tle->ressortgroupref = sgref;
|
|
|
|
tlist = lappend(tlist, tle);
|
2016-10-21 15:54:29 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2018-01-12 22:52:49 +01:00
|
|
|
/*
|
Avoid postgres_fdw crash for a targetlist entry that's just a Param.
foreign_grouping_ok() is willing to put fairly arbitrary expressions into
the targetlist of a remote SELECT that's doing grouping or aggregation on
the remote side, including expressions that have no foreign component to
them at all. This is possibly a bit dubious from an efficiency standpoint;
but it rises to the level of a crash-causing bug if the expression is just
a Param or non-foreign Var. In that case, the expression will necessarily
also appear in the fdw_exprs list of values we need to send to the remote
server, and then setrefs.c's set_foreignscan_references will mistakenly
replace the fdw_exprs entry with a Var referencing the targetlist result.
The root cause of this problem is bad design in commit e7cb7ee14: it put
logic into set_foreignscan_references that IMV is postgres_fdw-specific,
and yet this bug shows that it isn't postgres_fdw-specific enough. The
transformation being done on fdw_exprs assumes that fdw_exprs is to be
evaluated with the fdw_scan_tlist as input, which is not how postgres_fdw
uses it; yet it could be the right thing for some other FDW. (In the
bigger picture, setrefs.c has no business assuming this for the other
expression fields of a ForeignScan either.)
The right fix therefore would be to expand the FDW API so that the
FDW could inform setrefs.c how it intends to evaluate these various
expressions. We can't change that in the back branches though, and we
also can't just summarily change setrefs.c's behavior there, or we're
likely to break external FDWs.
As a stopgap, therefore, hack up postgres_fdw so that it won't attempt
to send targetlist entries that look exactly like the fdw_exprs entries
they'd produce. In most cases this actually produces a superior plan,
IMO, with less data needing to be transmitted and returned; so we probably
ought to think harder about whether we should ship tlist expressions at
all when they don't contain any foreign Vars or Aggs. But that's an
optimization not a bug fix so I left it for later. One case where this
produces an inferior plan is where the expression in question is actually
a GROUP BY expression: then the restriction prevents us from using remote
grouping. It might be possible to work around that (since that would
reduce to group-by-a-constant on the remote side); but it seems like a
pretty unlikely corner case, so I'm not sure it's worth expending effort
solely to improve that. In any case the right long-term answer is to fix
the API as sketched above, and then revert this hack.
Per bug #15781 from Sean Johnston. Back-patch to v10 where the problem
was introduced.
Discussion: https://postgr.es/m/15781-2601b1002bad087c@postgresql.org
2019-04-27 19:15:54 +02:00
|
|
|
* Non-grouping expression we need to compute. Can we ship it
|
|
|
|
* as-is to the foreign server?
|
2018-01-12 22:52:49 +01:00
|
|
|
*/
|
Avoid postgres_fdw crash for a targetlist entry that's just a Param.
foreign_grouping_ok() is willing to put fairly arbitrary expressions into
the targetlist of a remote SELECT that's doing grouping or aggregation on
the remote side, including expressions that have no foreign component to
them at all. This is possibly a bit dubious from an efficiency standpoint;
but it rises to the level of a crash-causing bug if the expression is just
a Param or non-foreign Var. In that case, the expression will necessarily
also appear in the fdw_exprs list of values we need to send to the remote
server, and then setrefs.c's set_foreignscan_references will mistakenly
replace the fdw_exprs entry with a Var referencing the targetlist result.
The root cause of this problem is bad design in commit e7cb7ee14: it put
logic into set_foreignscan_references that IMV is postgres_fdw-specific,
and yet this bug shows that it isn't postgres_fdw-specific enough. The
transformation being done on fdw_exprs assumes that fdw_exprs is to be
evaluated with the fdw_scan_tlist as input, which is not how postgres_fdw
uses it; yet it could be the right thing for some other FDW. (In the
bigger picture, setrefs.c has no business assuming this for the other
expression fields of a ForeignScan either.)
The right fix therefore would be to expand the FDW API so that the
FDW could inform setrefs.c how it intends to evaluate these various
expressions. We can't change that in the back branches though, and we
also can't just summarily change setrefs.c's behavior there, or we're
likely to break external FDWs.
As a stopgap, therefore, hack up postgres_fdw so that it won't attempt
to send targetlist entries that look exactly like the fdw_exprs entries
they'd produce. In most cases this actually produces a superior plan,
IMO, with less data needing to be transmitted and returned; so we probably
ought to think harder about whether we should ship tlist expressions at
all when they don't contain any foreign Vars or Aggs. But that's an
optimization not a bug fix so I left it for later. One case where this
produces an inferior plan is where the expression in question is actually
a GROUP BY expression: then the restriction prevents us from using remote
grouping. It might be possible to work around that (since that would
reduce to group-by-a-constant on the remote side); but it seems like a
pretty unlikely corner case, so I'm not sure it's worth expending effort
solely to improve that. In any case the right long-term answer is to fix
the API as sketched above, and then revert this hack.
Per bug #15781 from Sean Johnston. Back-patch to v10 where the problem
was introduced.
Discussion: https://postgr.es/m/15781-2601b1002bad087c@postgresql.org
2019-04-27 19:15:54 +02:00
|
|
|
if (is_foreign_expr(root, grouped_rel, expr) &&
|
|
|
|
!is_foreign_param(root, grouped_rel, expr))
|
2016-10-21 15:54:29 +02:00
|
|
|
{
|
2018-01-12 22:52:49 +01:00
|
|
|
/* Yes, so add to tlist as-is; OK to suppress duplicates */
|
2016-10-21 15:54:29 +02:00
|
|
|
tlist = add_to_flat_tlist(tlist, list_make1(expr));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2018-01-12 22:52:49 +01:00
|
|
|
/* Not pushable as a whole; extract its Vars and aggregates */
|
Avoid postgres_fdw crash for a targetlist entry that's just a Param.
foreign_grouping_ok() is willing to put fairly arbitrary expressions into
the targetlist of a remote SELECT that's doing grouping or aggregation on
the remote side, including expressions that have no foreign component to
them at all. This is possibly a bit dubious from an efficiency standpoint;
but it rises to the level of a crash-causing bug if the expression is just
a Param or non-foreign Var. In that case, the expression will necessarily
also appear in the fdw_exprs list of values we need to send to the remote
server, and then setrefs.c's set_foreignscan_references will mistakenly
replace the fdw_exprs entry with a Var referencing the targetlist result.
The root cause of this problem is bad design in commit e7cb7ee14: it put
logic into set_foreignscan_references that IMV is postgres_fdw-specific,
and yet this bug shows that it isn't postgres_fdw-specific enough. The
transformation being done on fdw_exprs assumes that fdw_exprs is to be
evaluated with the fdw_scan_tlist as input, which is not how postgres_fdw
uses it; yet it could be the right thing for some other FDW. (In the
bigger picture, setrefs.c has no business assuming this for the other
expression fields of a ForeignScan either.)
The right fix therefore would be to expand the FDW API so that the
FDW could inform setrefs.c how it intends to evaluate these various
expressions. We can't change that in the back branches though, and we
also can't just summarily change setrefs.c's behavior there, or we're
likely to break external FDWs.
As a stopgap, therefore, hack up postgres_fdw so that it won't attempt
to send targetlist entries that look exactly like the fdw_exprs entries
they'd produce. In most cases this actually produces a superior plan,
IMO, with less data needing to be transmitted and returned; so we probably
ought to think harder about whether we should ship tlist expressions at
all when they don't contain any foreign Vars or Aggs. But that's an
optimization not a bug fix so I left it for later. One case where this
produces an inferior plan is where the expression in question is actually
a GROUP BY expression: then the restriction prevents us from using remote
grouping. It might be possible to work around that (since that would
reduce to group-by-a-constant on the remote side); but it seems like a
pretty unlikely corner case, so I'm not sure it's worth expending effort
solely to improve that. In any case the right long-term answer is to fix
the API as sketched above, and then revert this hack.
Per bug #15781 from Sean Johnston. Back-patch to v10 where the problem
was introduced.
Discussion: https://postgr.es/m/15781-2601b1002bad087c@postgresql.org
2019-04-27 19:15:54 +02:00
|
|
|
List *aggvars;
|
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
aggvars = pull_var_clause((Node *) expr,
|
|
|
|
PVC_INCLUDE_AGGREGATES);
|
|
|
|
|
2018-01-12 22:52:49 +01:00
|
|
|
/*
|
|
|
|
* If any aggregate expression is not shippable, then we
|
Avoid postgres_fdw crash for a targetlist entry that's just a Param.
foreign_grouping_ok() is willing to put fairly arbitrary expressions into
the targetlist of a remote SELECT that's doing grouping or aggregation on
the remote side, including expressions that have no foreign component to
them at all. This is possibly a bit dubious from an efficiency standpoint;
but it rises to the level of a crash-causing bug if the expression is just
a Param or non-foreign Var. In that case, the expression will necessarily
also appear in the fdw_exprs list of values we need to send to the remote
server, and then setrefs.c's set_foreignscan_references will mistakenly
replace the fdw_exprs entry with a Var referencing the targetlist result.
The root cause of this problem is bad design in commit e7cb7ee14: it put
logic into set_foreignscan_references that IMV is postgres_fdw-specific,
and yet this bug shows that it isn't postgres_fdw-specific enough. The
transformation being done on fdw_exprs assumes that fdw_exprs is to be
evaluated with the fdw_scan_tlist as input, which is not how postgres_fdw
uses it; yet it could be the right thing for some other FDW. (In the
bigger picture, setrefs.c has no business assuming this for the other
expression fields of a ForeignScan either.)
The right fix therefore would be to expand the FDW API so that the
FDW could inform setrefs.c how it intends to evaluate these various
expressions. We can't change that in the back branches though, and we
also can't just summarily change setrefs.c's behavior there, or we're
likely to break external FDWs.
As a stopgap, therefore, hack up postgres_fdw so that it won't attempt
to send targetlist entries that look exactly like the fdw_exprs entries
they'd produce. In most cases this actually produces a superior plan,
IMO, with less data needing to be transmitted and returned; so we probably
ought to think harder about whether we should ship tlist expressions at
all when they don't contain any foreign Vars or Aggs. But that's an
optimization not a bug fix so I left it for later. One case where this
produces an inferior plan is where the expression in question is actually
a GROUP BY expression: then the restriction prevents us from using remote
grouping. It might be possible to work around that (since that would
reduce to group-by-a-constant on the remote side); but it seems like a
pretty unlikely corner case, so I'm not sure it's worth expending effort
solely to improve that. In any case the right long-term answer is to fix
the API as sketched above, and then revert this hack.
Per bug #15781 from Sean Johnston. Back-patch to v10 where the problem
was introduced.
Discussion: https://postgr.es/m/15781-2601b1002bad087c@postgresql.org
2019-04-27 19:15:54 +02:00
|
|
|
* cannot push down aggregation to the foreign server. (We
|
|
|
|
* don't have to check is_foreign_param, since that certainly
|
|
|
|
* won't return true for any such expression.)
|
2018-01-12 22:52:49 +01:00
|
|
|
*/
|
2016-10-21 15:54:29 +02:00
|
|
|
if (!is_foreign_expr(root, grouped_rel, (Expr *) aggvars))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
2018-01-12 22:52:49 +01:00
|
|
|
* Add aggregates, if any, into the targetlist. Plain Vars
|
|
|
|
* outside an aggregate can be ignored, because they should be
|
|
|
|
* either same as some GROUP BY column or part of some GROUP
|
|
|
|
* BY expression. In either case, they are already part of
|
2016-10-21 15:54:29 +02:00
|
|
|
* the targetlist and thus no need to add them again. In fact
|
2018-01-12 22:52:49 +01:00
|
|
|
* including plain Vars in the tlist when they do not match a
|
|
|
|
* GROUP BY column would cause the foreign server to complain
|
|
|
|
* that the shipped query is invalid.
|
2016-10-21 15:54:29 +02:00
|
|
|
*/
|
|
|
|
foreach(l, aggvars)
|
|
|
|
{
|
|
|
|
Expr *expr = (Expr *) lfirst(l);
|
|
|
|
|
|
|
|
if (IsA(expr, Aggref))
|
|
|
|
tlist = add_to_flat_tlist(tlist, list_make1(expr));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2018-01-12 22:52:49 +01:00
|
|
|
* Classify the pushable and non-pushable HAVING clauses and save them in
|
2016-10-21 15:54:29 +02:00
|
|
|
* remote_conds and local_conds of the grouped rel's fpinfo.
|
|
|
|
*/
|
2018-04-02 16:51:50 +02:00
|
|
|
if (havingQual)
|
2016-10-21 15:54:29 +02:00
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
2018-04-02 16:51:50 +02:00
|
|
|
foreach(lc, (List *) havingQual)
|
2016-10-21 15:54:29 +02:00
|
|
|
{
|
|
|
|
Expr *expr = (Expr *) lfirst(lc);
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
RestrictInfo *rinfo;
|
2016-10-21 15:54:29 +02:00
|
|
|
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
/*
|
|
|
|
* Currently, the core code doesn't wrap havingQuals in
|
|
|
|
* RestrictInfos, so we must make our own.
|
|
|
|
*/
|
|
|
|
Assert(!IsA(expr, RestrictInfo));
|
Fix pull_varnos' miscomputation of relids set for a PlaceHolderVar.
Previously, pull_varnos() took the relids of a PlaceHolderVar as being
equal to the relids in its contents, but that fails to account for the
possibility that we have to postpone evaluation of the PHV due to outer
joins. This could result in a malformed plan. The known cases end up
triggering the "failed to assign all NestLoopParams to plan nodes"
sanity check in createplan.c, but other symptoms may be possible.
The right value to use is the join level we actually intend to evaluate
the PHV at. We can get that from the ph_eval_at field of the associated
PlaceHolderInfo. However, there are some places that call pull_varnos()
before the PlaceHolderInfos have been created; in that case, fall back
to the conservative assumption that the PHV will be evaluated at its
syntactic level. (In principle this might result in missing some legal
optimization, but I'm not aware of any cases where it's an issue in
practice.) Things are also a bit ticklish for calls occurring during
deconstruct_jointree(), but AFAICS the ph_eval_at fields should have
reached their final values by the time we need them.
The main problem in making this work is that pull_varnos() has no
way to get at the PlaceHolderInfos. We can fix that easily, if a
bit tediously, in HEAD by passing it the planner "root" pointer.
In the back branches that'd cause an unacceptable API/ABI break for
extensions, so leave the existing entry points alone and add new ones
with the additional parameter. (If an old entry point is called and
encounters a PHV, it'll fall back to using the syntactic level,
again possibly missing some valid optimization.)
Back-patch to v12. The computation is surely also wrong before that,
but it appears that we cannot reach a bad plan thanks to join order
restrictions imposed on the subquery that the PlaceHolderVar came from.
The error only became reachable when commit 4be058fe9 allowed trivial
subqueries to be collapsed out completely, eliminating their join order
restrictions.
Per report from Stephan Springl.
Discussion: https://postgr.es/m/171041.1610849523@sss.pgh.pa.us
2021-01-21 21:37:23 +01:00
|
|
|
rinfo = make_restrictinfo(root,
|
|
|
|
expr,
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
true,
|
|
|
|
false,
|
|
|
|
false,
|
|
|
|
root->qual_security_level,
|
|
|
|
grouped_rel->relids,
|
|
|
|
NULL,
|
|
|
|
NULL);
|
|
|
|
if (is_foreign_expr(root, grouped_rel, expr))
|
|
|
|
fpinfo->remote_conds = lappend(fpinfo->remote_conds, rinfo);
|
2016-10-21 15:54:29 +02:00
|
|
|
else
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
fpinfo->local_conds = lappend(fpinfo->local_conds, rinfo);
|
2016-10-21 15:54:29 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there are any local conditions, pull Vars and aggregates from it and
|
|
|
|
* check whether they are safe to pushdown or not.
|
|
|
|
*/
|
|
|
|
if (fpinfo->local_conds)
|
|
|
|
{
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
List *aggvars = NIL;
|
2016-10-21 15:54:29 +02:00
|
|
|
ListCell *lc;
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
|
|
|
|
foreach(lc, fpinfo->local_conds)
|
|
|
|
{
|
|
|
|
RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc);
|
|
|
|
|
|
|
|
aggvars = list_concat(aggvars,
|
|
|
|
pull_var_clause((Node *) rinfo->clause,
|
|
|
|
PVC_INCLUDE_AGGREGATES));
|
|
|
|
}
|
2016-10-21 15:54:29 +02:00
|
|
|
|
|
|
|
foreach(lc, aggvars)
|
|
|
|
{
|
|
|
|
Expr *expr = (Expr *) lfirst(lc);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If aggregates within local conditions are not safe to push
|
|
|
|
* down, then we cannot push down the query. Vars are already
|
|
|
|
* part of GROUP BY clause which are checked above, so no need to
|
Avoid postgres_fdw crash for a targetlist entry that's just a Param.
foreign_grouping_ok() is willing to put fairly arbitrary expressions into
the targetlist of a remote SELECT that's doing grouping or aggregation on
the remote side, including expressions that have no foreign component to
them at all. This is possibly a bit dubious from an efficiency standpoint;
but it rises to the level of a crash-causing bug if the expression is just
a Param or non-foreign Var. In that case, the expression will necessarily
also appear in the fdw_exprs list of values we need to send to the remote
server, and then setrefs.c's set_foreignscan_references will mistakenly
replace the fdw_exprs entry with a Var referencing the targetlist result.
The root cause of this problem is bad design in commit e7cb7ee14: it put
logic into set_foreignscan_references that IMV is postgres_fdw-specific,
and yet this bug shows that it isn't postgres_fdw-specific enough. The
transformation being done on fdw_exprs assumes that fdw_exprs is to be
evaluated with the fdw_scan_tlist as input, which is not how postgres_fdw
uses it; yet it could be the right thing for some other FDW. (In the
bigger picture, setrefs.c has no business assuming this for the other
expression fields of a ForeignScan either.)
The right fix therefore would be to expand the FDW API so that the
FDW could inform setrefs.c how it intends to evaluate these various
expressions. We can't change that in the back branches though, and we
also can't just summarily change setrefs.c's behavior there, or we're
likely to break external FDWs.
As a stopgap, therefore, hack up postgres_fdw so that it won't attempt
to send targetlist entries that look exactly like the fdw_exprs entries
they'd produce. In most cases this actually produces a superior plan,
IMO, with less data needing to be transmitted and returned; so we probably
ought to think harder about whether we should ship tlist expressions at
all when they don't contain any foreign Vars or Aggs. But that's an
optimization not a bug fix so I left it for later. One case where this
produces an inferior plan is where the expression in question is actually
a GROUP BY expression: then the restriction prevents us from using remote
grouping. It might be possible to work around that (since that would
reduce to group-by-a-constant on the remote side); but it seems like a
pretty unlikely corner case, so I'm not sure it's worth expending effort
solely to improve that. In any case the right long-term answer is to fix
the API as sketched above, and then revert this hack.
Per bug #15781 from Sean Johnston. Back-patch to v10 where the problem
was introduced.
Discussion: https://postgr.es/m/15781-2601b1002bad087c@postgresql.org
2019-04-27 19:15:54 +02:00
|
|
|
* access them again here. Again, we need not check
|
|
|
|
* is_foreign_param for a foreign aggregate.
|
2016-10-21 15:54:29 +02:00
|
|
|
*/
|
|
|
|
if (IsA(expr, Aggref))
|
|
|
|
{
|
|
|
|
if (!is_foreign_expr(root, grouped_rel, expr))
|
|
|
|
return false;
|
|
|
|
|
Handle restriction clause lists more uniformly in postgres_fdw.
Clauses in the lists retained by postgres_fdw during planning were
sometimes bare boolean clauses, sometimes RestrictInfos, and sometimes
a mixture of the two in the same list. The comment about that situation
didn't come close to telling the full truth, either. Aside from being
confusing, this had a couple of bad practical consequences:
* waste of planning cycles due to inability to cache per-clause selectivity
and cost estimates;
* sometimes, RestrictInfos would sneak into the fdw_private list of a
finished Plan node, causing failures if, for example, we tried to ship
the Plan tree to a parallel worker.
(It may well be that it's a bug in the parallel-query logic that we
would ever try to ship such a plan to a parallel worker, but in any
case this deserves to be cleaned up.)
To fix, rearrange so that clause lists in PgFdwRelationInfo are always
lists of RestrictInfos, and then strip the RestrictInfos at the last
minute when making a Plan node. In passing do a bit of refactoring and
comment cleanup in postgresGetForeignPlan and foreign_join_ok.
Although the messiness here dates back at least to 9.6, there's no evidence
that it causes anything worse than wasted planning cycles in 9.6, so no
back-patch for now.
Per fuzz testing by Andreas Seltenreich.
Tom Lane and Ashutosh Bapat
Discussion: https://postgr.es/m/87tw5x4vcu.fsf@credativ.de
2017-04-11 17:58:59 +02:00
|
|
|
tlist = add_to_flat_tlist(tlist, list_make1(expr));
|
2016-10-21 15:54:29 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Store generated targetlist */
|
|
|
|
fpinfo->grouped_tlist = tlist;
|
|
|
|
|
|
|
|
/* Safe to pushdown */
|
|
|
|
fpinfo->pushdown_safe = true;
|
|
|
|
|
|
|
|
/*
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
* Set # of retrieved rows and cached relation costs to some negative
|
|
|
|
* value, so that we can detect when they are set to some sensible values,
|
|
|
|
* during one (usually the first) of the calls to estimate_path_cost_size.
|
2016-10-21 15:54:29 +02:00
|
|
|
*/
|
postgres_fdw: Fix costing of pre-sorted foreign paths with local stats.
Commit aa09cd242 modified estimate_path_cost_size() so that it reuses
cached costs of a basic foreign path for a given foreign-base/join
relation when costing pre-sorted foreign paths for that relation, but it
incorrectly re-computed retrieved_rows, an estimated number of rows
fetched from the remote side, which is needed for costing both the basic
and pre-sorted foreign paths. To fix, handle retrieved_rows the same way
as the cached costs: store in that relation's fpinfo the retrieved_rows
estimate computed for costing the basic foreign path, and reuse it when
costing the pre-sorted foreign paths. Also, reuse the rows/width
estimates stored in that relation's fpinfo when costing the pre-sorted
foreign paths, to make the code consistent.
In commit ffab494a4, to extend the costing mentioned above to the
foreign-grouping case, I made a change to add_foreign_grouping_paths() to
store in a given foreign-grouped relation's RelOptInfo the rows estimate
for that relation for reuse, but this patch makes that change unnecessary
since we already store the row estimate in that relation's fpinfo, which
this patch reuses when costing a foreign path for that relation with the
sortClause ordering; remove that change.
In passing, fix thinko in commit 7012b132d: in estimate_path_cost_size(),
the width estimate for a given foreign-grouped relation to be stored in
that relation's fpinfo was reset incorrectly when costing a basic foreign
path for that relation with local stats.
Apply the patch to HEAD only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/CAPmGK17jaJLPDEkgnP2VmkOg=5wT8YQ1CqssU8JRpZ_NSE+dqQ@mail.gmail.com
2019-06-14 13:49:59 +02:00
|
|
|
fpinfo->retrieved_rows = -1;
|
2016-10-21 15:54:29 +02:00
|
|
|
fpinfo->rel_startup_cost = -1;
|
|
|
|
fpinfo->rel_total_cost = -1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set the string describing this grouped relation to be used in EXPLAIN
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
* output of corresponding ForeignScan. Note that the decoration we add
|
|
|
|
* to the base relation name mustn't include any digits, or it'll confuse
|
|
|
|
* postgresExplainForeignScan.
|
2016-10-21 15:54:29 +02:00
|
|
|
*/
|
Make postgres_fdw's "Relations" output agree with the rest of EXPLAIN.
The relation aliases shown in the "Relations" line for a foreign scan
didn't always agree with those used in the rest of EXPLAIN's output.
The regression test result changes appearing here provide examples.
It's really impossible for postgres_fdw to duplicate EXPLAIN's alias
assignment logic during postgresGetForeignRelSize(), because of the
de-duplication that EXPLAIN does on a global basis --- and anyway,
trying to duplicate that would be unmaintainable. Instead, just put
numeric rangetable indexes into the string, and convert those to
table names/aliases in postgresExplainForeignScan, which does have
access to the results of ruleutils.c's alias assignment logic.
Aside from being more reliable, this shifts some work from planning
to EXPLAIN, which is a good tradeoff for performance. (I also
changed from using StringInfo to using psprintf, which makes the
code slightly simpler and reduces its memory consumption.)
A kluge required by this solution is that we have to reverse-engineer
the rtoffset applied by setrefs.c. If that logic ever fails
(presumably because the member tables of a join got offset by
different amounts), we'll need some more cooperation with setrefs.c
to keep things straight. But for now, there's no need for that.
Arguably this is a back-patchable bug fix, but since this is a mostly
cosmetic issue and there have been no field complaints, I'll refrain
for now.
Discussion: https://postgr.es/m/12424.1575168015@sss.pgh.pa.us
2019-12-02 22:31:03 +01:00
|
|
|
fpinfo->relation_name = psprintf("Aggregate on (%s)",
|
|
|
|
ofpinfo->relation_name);
|
2016-10-21 15:54:29 +02:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresGetForeignUpperPaths
|
|
|
|
* Add paths for post-join operations like aggregation, grouping etc. if
|
|
|
|
* corresponding operations are safe to push down.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresGetForeignUpperPaths(PlannerInfo *root, UpperRelationKind stage,
|
2018-04-02 16:51:50 +02:00
|
|
|
RelOptInfo *input_rel, RelOptInfo *output_rel,
|
|
|
|
void *extra)
|
2016-10-21 15:54:29 +02:00
|
|
|
{
|
|
|
|
PgFdwRelationInfo *fpinfo;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If input rel is not safe to pushdown, then simply return as we cannot
|
|
|
|
* perform any post-join operations on the foreign server.
|
|
|
|
*/
|
|
|
|
if (!input_rel->fdw_private ||
|
|
|
|
!((PgFdwRelationInfo *) input_rel->fdw_private)->pushdown_safe)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Ignore stages we don't support; and skip any duplicate calls. */
|
2019-04-02 12:20:30 +02:00
|
|
|
if ((stage != UPPERREL_GROUP_AGG &&
|
2019-04-02 13:30:45 +02:00
|
|
|
stage != UPPERREL_ORDERED &&
|
|
|
|
stage != UPPERREL_FINAL) ||
|
2019-04-02 12:20:30 +02:00
|
|
|
output_rel->fdw_private)
|
2016-10-21 15:54:29 +02:00
|
|
|
return;
|
|
|
|
|
|
|
|
fpinfo = (PgFdwRelationInfo *) palloc0(sizeof(PgFdwRelationInfo));
|
|
|
|
fpinfo->pushdown_safe = false;
|
2019-04-02 12:20:30 +02:00
|
|
|
fpinfo->stage = stage;
|
2016-10-21 15:54:29 +02:00
|
|
|
output_rel->fdw_private = fpinfo;
|
|
|
|
|
2019-04-02 12:20:30 +02:00
|
|
|
switch (stage)
|
|
|
|
{
|
|
|
|
case UPPERREL_GROUP_AGG:
|
|
|
|
add_foreign_grouping_paths(root, input_rel, output_rel,
|
|
|
|
(GroupPathExtraData *) extra);
|
|
|
|
break;
|
|
|
|
case UPPERREL_ORDERED:
|
|
|
|
add_foreign_ordered_paths(root, input_rel, output_rel);
|
|
|
|
break;
|
2019-04-02 13:30:45 +02:00
|
|
|
case UPPERREL_FINAL:
|
|
|
|
add_foreign_final_paths(root, input_rel, output_rel,
|
|
|
|
(FinalPathExtraData *) extra);
|
|
|
|
break;
|
2019-04-02 12:20:30 +02:00
|
|
|
default:
|
|
|
|
elog(ERROR, "unexpected upper relation: %d", (int) stage);
|
|
|
|
break;
|
|
|
|
}
|
2016-10-21 15:54:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* add_foreign_grouping_paths
|
|
|
|
* Add foreign path for grouping and/or aggregation.
|
|
|
|
*
|
|
|
|
* Given input_rel represents the underlying scan. The paths are added to the
|
|
|
|
* given grouped_rel.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
add_foreign_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel,
|
2018-04-02 16:51:50 +02:00
|
|
|
RelOptInfo *grouped_rel,
|
|
|
|
GroupPathExtraData *extra)
|
2016-10-21 15:54:29 +02:00
|
|
|
{
|
|
|
|
Query *parse = root->parse;
|
|
|
|
PgFdwRelationInfo *ifpinfo = input_rel->fdw_private;
|
|
|
|
PgFdwRelationInfo *fpinfo = grouped_rel->fdw_private;
|
|
|
|
ForeignPath *grouppath;
|
|
|
|
double rows;
|
|
|
|
int width;
|
|
|
|
Cost startup_cost;
|
|
|
|
Cost total_cost;
|
|
|
|
|
|
|
|
/* Nothing to be done, if there is no grouping or aggregation required. */
|
|
|
|
if (!parse->groupClause && !parse->groupingSets && !parse->hasAggs &&
|
|
|
|
!root->hasHavingQual)
|
|
|
|
return;
|
|
|
|
|
2018-04-02 16:51:50 +02:00
|
|
|
Assert(extra->patype == PARTITIONWISE_AGGREGATE_NONE ||
|
|
|
|
extra->patype == PARTITIONWISE_AGGREGATE_FULL);
|
2016-10-21 15:54:29 +02:00
|
|
|
|
|
|
|
/* save the input_rel as outerrel in fpinfo */
|
|
|
|
fpinfo->outerrel = input_rel;
|
|
|
|
|
|
|
|
/*
|
2017-04-25 04:50:07 +02:00
|
|
|
* Copy foreign table, foreign server, user mapping, FDW options etc.
|
|
|
|
* details from the input relation's fpinfo.
|
2016-10-21 15:54:29 +02:00
|
|
|
*/
|
|
|
|
fpinfo->table = ifpinfo->table;
|
|
|
|
fpinfo->server = ifpinfo->server;
|
|
|
|
fpinfo->user = ifpinfo->user;
|
2017-04-25 04:50:07 +02:00
|
|
|
merge_fdw_options(fpinfo, ifpinfo, NULL);
|
2016-10-21 15:54:29 +02:00
|
|
|
|
2018-04-02 16:51:50 +02:00
|
|
|
/*
|
|
|
|
* Assess if it is safe to push down aggregation and grouping.
|
|
|
|
*
|
|
|
|
* Use HAVING qual from extra. In case of child partition, it will have
|
|
|
|
* translated Vars.
|
|
|
|
*/
|
|
|
|
if (!foreign_grouping_ok(root, grouped_rel, extra->havingQual))
|
2016-10-21 15:54:29 +02:00
|
|
|
return;
|
|
|
|
|
postgres_fdw: Improve cost and size estimation for aggregate pushdown.
In commit 7012b132d07c2b4ea15b0b3cb1ea9f3278801d98, which added aggregate
pushdown to postgres_fdw, we didn't account for the evaluation cost and the
selectivity of HAVING quals attached to ForeignPaths performing aggregate
pushdown, as core had never accounted for that for AggPaths and GroupPaths.
And we didn't set these values of the locally-checked quals (ie, fpinfo's
local_conds_cost and local_conds_sel), which were initialized to zeros, but
since estimate_path_cost_size factors in these to estimate the result size
and the evaluation cost of such a ForeignPath when the use_remote_estimate
option is enabled, this caused it to produce underestimated results in that
case.
By commit 7b6c07547190f056b0464098bb5a2247129d7aa2 core was changed so that
it accounts for the evaluation cost and the selectivity of HAVING quals in
aggregation paths, so change the postgres_fdw's aggregate pushdown code as
well as such. This not only fixes the underestimation issue mentioned
above, but improves the estimation using local statistics in that function
when that option is disabled.
This would be a bug fix rather than an improvement, but apply it to HEAD
only to avoid destabilizing existing plan choices.
Author: Etsuro Fujita
Discussion: https://postgr.es/m/5BFD3EAD.2060301%40lab.ntt.co.jp
2018-12-04 09:18:58 +01:00
|
|
|
/*
|
|
|
|
* Compute the selectivity and cost of the local_conds, so we don't have
|
|
|
|
* to do it over again for each path. (Currently we create just a single
|
|
|
|
* path here, but in future it would be possible that we build more paths
|
|
|
|
* such as pre-sorted paths as in postgresGetForeignPaths and
|
|
|
|
* postgresGetForeignJoinPaths.) The best we can do for these conditions
|
|
|
|
* is to estimate selectivity on the basis of local statistics.
|
|
|
|
*/
|
|
|
|
fpinfo->local_conds_sel = clauselist_selectivity(root,
|
|
|
|
fpinfo->local_conds,
|
|
|
|
0,
|
|
|
|
JOIN_INNER,
|
|
|
|
NULL);
|
|
|
|
|
|
|
|
cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root);
|
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
/* Estimate the cost of push down */
|
2019-04-02 12:20:30 +02:00
|
|
|
estimate_path_cost_size(root, grouped_rel, NIL, NIL, NULL,
|
|
|
|
&rows, &width, &startup_cost, &total_cost);
|
2016-10-21 15:54:29 +02:00
|
|
|
|
|
|
|
/* Now update this information in the fpinfo */
|
|
|
|
fpinfo->rows = rows;
|
|
|
|
fpinfo->width = width;
|
|
|
|
fpinfo->startup_cost = startup_cost;
|
|
|
|
fpinfo->total_cost = total_cost;
|
|
|
|
|
|
|
|
/* Create and add foreign path to the grouping relation. */
|
Split create_foreignscan_path() into three functions.
Up to now postgres_fdw has been using create_foreignscan_path() to
generate not only base-relation paths, but also paths for foreign joins
and foreign upperrels. This is wrong, because create_foreignscan_path()
calls get_baserel_parampathinfo() which will only do the right thing for
baserels. It accidentally fails to fail for unparameterized paths, which
are the only ones postgres_fdw (thought it) was handling, but we really
need different APIs for the baserel and join cases.
In HEAD, the best thing to do seems to be to split up the baserel,
joinrel, and upperrel cases into three functions so that they can
have different APIs. I haven't actually given create_foreign_join_path
a different API in this commit: we should spend a bit of time thinking
about just what we want to do there, since perhaps FDWs would want to
do something different from the build-up-a-join-pairwise approach that
get_joinrel_parampathinfo expects. In the meantime, since postgres_fdw
isn't prepared to generate parameterized joins anyway, just give it a
defense against trying to plan joins with lateral refs.
In addition (and this is what triggered this whole mess) fix bug #15613
from Srinivasan S A, by teaching file_fdw and postgres_fdw that plain
baserel foreign paths still have outer refs if the relation has
lateral_relids. Add some assertions in relnode.c to catch future
occurrences of the same error --- in particular, to catch other FDWs
doing that, but also as backstop against core-code mistakes like the
one fixed by commit bdd9a99aa.
Bug #15613 also needs to be fixed in the back branches, but the
appropriate fix will look quite a bit different there, since we don't
want to assume that existing FDWs get the word right away.
Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org
2019-02-07 18:59:47 +01:00
|
|
|
grouppath = create_foreign_upper_path(root,
|
|
|
|
grouped_rel,
|
|
|
|
grouped_rel->reltarget,
|
|
|
|
rows,
|
|
|
|
startup_cost,
|
|
|
|
total_cost,
|
|
|
|
NIL, /* no pathkeys */
|
|
|
|
NULL,
|
|
|
|
NIL); /* no fdw_private */
|
2016-10-21 15:54:29 +02:00
|
|
|
|
|
|
|
/* Add generated path into grouped_rel by add_path(). */
|
|
|
|
add_path(grouped_rel, (Path *) grouppath);
|
|
|
|
}
|
|
|
|
|
2019-04-02 12:20:30 +02:00
|
|
|
/*
|
|
|
|
* add_foreign_ordered_paths
|
|
|
|
* Add foreign paths for performing the final sort remotely.
|
|
|
|
*
|
|
|
|
* Given input_rel contains the source-data Paths. The paths are added to the
|
|
|
|
* given ordered_rel.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
add_foreign_ordered_paths(PlannerInfo *root, RelOptInfo *input_rel,
|
|
|
|
RelOptInfo *ordered_rel)
|
|
|
|
{
|
|
|
|
Query *parse = root->parse;
|
|
|
|
PgFdwRelationInfo *ifpinfo = input_rel->fdw_private;
|
|
|
|
PgFdwRelationInfo *fpinfo = ordered_rel->fdw_private;
|
|
|
|
PgFdwPathExtraData *fpextra;
|
|
|
|
double rows;
|
|
|
|
int width;
|
|
|
|
Cost startup_cost;
|
|
|
|
Cost total_cost;
|
|
|
|
List *fdw_private;
|
|
|
|
ForeignPath *ordered_path;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
/* Shouldn't get here unless the query has ORDER BY */
|
|
|
|
Assert(parse->sortClause);
|
|
|
|
|
|
|
|
/* We don't support cases where there are any SRFs in the targetlist */
|
|
|
|
if (parse->hasTargetSRFs)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Save the input_rel as outerrel in fpinfo */
|
|
|
|
fpinfo->outerrel = input_rel;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy foreign table, foreign server, user mapping, FDW options etc.
|
|
|
|
* details from the input relation's fpinfo.
|
|
|
|
*/
|
|
|
|
fpinfo->table = ifpinfo->table;
|
|
|
|
fpinfo->server = ifpinfo->server;
|
|
|
|
fpinfo->user = ifpinfo->user;
|
|
|
|
merge_fdw_options(fpinfo, ifpinfo, NULL);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the input_rel is a base or join relation, we would already have
|
|
|
|
* considered pushing down the final sort to the remote server when
|
|
|
|
* creating pre-sorted foreign paths for that relation, because the
|
|
|
|
* query_pathkeys is set to the root->sort_pathkeys in that case (see
|
|
|
|
* standard_qp_callback()).
|
|
|
|
*/
|
|
|
|
if (input_rel->reloptkind == RELOPT_BASEREL ||
|
|
|
|
input_rel->reloptkind == RELOPT_JOINREL)
|
|
|
|
{
|
|
|
|
Assert(root->query_pathkeys == root->sort_pathkeys);
|
|
|
|
|
|
|
|
/* Safe to push down if the query_pathkeys is safe to push down */
|
|
|
|
fpinfo->pushdown_safe = ifpinfo->qp_is_pushdown_safe;
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* The input_rel should be a grouping relation */
|
|
|
|
Assert(input_rel->reloptkind == RELOPT_UPPER_REL &&
|
|
|
|
ifpinfo->stage == UPPERREL_GROUP_AGG);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We try to create a path below by extending a simple foreign path for
|
|
|
|
* the underlying grouping relation to perform the final sort remotely,
|
|
|
|
* which is stored into the fdw_private list of the resulting path.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* Assess if it is safe to push down the final sort */
|
|
|
|
foreach(lc, root->sort_pathkeys)
|
|
|
|
{
|
|
|
|
PathKey *pathkey = (PathKey *) lfirst(lc);
|
|
|
|
EquivalenceClass *pathkey_ec = pathkey->pk_eclass;
|
|
|
|
Expr *sort_expr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* is_foreign_expr would detect volatile expressions as well, but
|
|
|
|
* checking ec_has_volatile here saves some cycles.
|
|
|
|
*/
|
|
|
|
if (pathkey_ec->ec_has_volatile)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Get the sort expression for the pathkey_ec */
|
|
|
|
sort_expr = find_em_expr_for_input_target(root,
|
|
|
|
pathkey_ec,
|
|
|
|
input_rel->reltarget);
|
|
|
|
|
|
|
|
/* If it's unsafe to remote, we cannot push down the final sort */
|
|
|
|
if (!is_foreign_expr(root, input_rel, sort_expr))
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Safe to push down */
|
|
|
|
fpinfo->pushdown_safe = true;
|
|
|
|
|
|
|
|
/* Construct PgFdwPathExtraData */
|
|
|
|
fpextra = (PgFdwPathExtraData *) palloc0(sizeof(PgFdwPathExtraData));
|
|
|
|
fpextra->target = root->upper_targets[UPPERREL_ORDERED];
|
|
|
|
fpextra->has_final_sort = true;
|
|
|
|
|
|
|
|
/* Estimate the costs of performing the final sort remotely */
|
|
|
|
estimate_path_cost_size(root, input_rel, NIL, root->sort_pathkeys, fpextra,
|
|
|
|
&rows, &width, &startup_cost, &total_cost);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Build the fdw_private list that will be used by postgresGetForeignPlan.
|
|
|
|
* Items in the list must match order in enum FdwPathPrivateIndex.
|
|
|
|
*/
|
2019-04-02 13:30:45 +02:00
|
|
|
fdw_private = list_make2(makeInteger(true), makeInteger(false));
|
2019-04-02 12:20:30 +02:00
|
|
|
|
|
|
|
/* Create foreign ordering path */
|
|
|
|
ordered_path = create_foreign_upper_path(root,
|
|
|
|
input_rel,
|
|
|
|
root->upper_targets[UPPERREL_ORDERED],
|
|
|
|
rows,
|
|
|
|
startup_cost,
|
|
|
|
total_cost,
|
|
|
|
root->sort_pathkeys,
|
|
|
|
NULL, /* no extra plan */
|
|
|
|
fdw_private);
|
|
|
|
|
|
|
|
/* and add it to the ordered_rel */
|
|
|
|
add_path(ordered_rel, (Path *) ordered_path);
|
|
|
|
}
|
|
|
|
|
2019-04-02 13:30:45 +02:00
|
|
|
/*
|
|
|
|
* add_foreign_final_paths
|
|
|
|
* Add foreign paths for performing the final processing remotely.
|
|
|
|
*
|
|
|
|
* Given input_rel contains the source-data Paths. The paths are added to the
|
|
|
|
* given final_rel.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
add_foreign_final_paths(PlannerInfo *root, RelOptInfo *input_rel,
|
|
|
|
RelOptInfo *final_rel,
|
|
|
|
FinalPathExtraData *extra)
|
|
|
|
{
|
|
|
|
Query *parse = root->parse;
|
|
|
|
PgFdwRelationInfo *ifpinfo = (PgFdwRelationInfo *) input_rel->fdw_private;
|
|
|
|
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) final_rel->fdw_private;
|
|
|
|
bool has_final_sort = false;
|
|
|
|
List *pathkeys = NIL;
|
|
|
|
PgFdwPathExtraData *fpextra;
|
|
|
|
bool save_use_remote_estimate = false;
|
|
|
|
double rows;
|
|
|
|
int width;
|
|
|
|
Cost startup_cost;
|
|
|
|
Cost total_cost;
|
|
|
|
List *fdw_private;
|
|
|
|
ForeignPath *final_path;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Currently, we only support this for SELECT commands
|
|
|
|
*/
|
|
|
|
if (parse->commandType != CMD_SELECT)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No work if there is no FOR UPDATE/SHARE clause and if there is no need
|
|
|
|
* to add a LIMIT node
|
|
|
|
*/
|
|
|
|
if (!parse->rowMarks && !extra->limit_needed)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* We don't support cases where there are any SRFs in the targetlist */
|
|
|
|
if (parse->hasTargetSRFs)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Save the input_rel as outerrel in fpinfo */
|
|
|
|
fpinfo->outerrel = input_rel;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy foreign table, foreign server, user mapping, FDW options etc.
|
|
|
|
* details from the input relation's fpinfo.
|
|
|
|
*/
|
|
|
|
fpinfo->table = ifpinfo->table;
|
|
|
|
fpinfo->server = ifpinfo->server;
|
|
|
|
fpinfo->user = ifpinfo->user;
|
|
|
|
merge_fdw_options(fpinfo, ifpinfo, NULL);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there is no need to add a LIMIT node, there might be a ForeignPath
|
|
|
|
* in the input_rel's pathlist that implements all behavior of the query.
|
|
|
|
* Note: we would already have accounted for the query's FOR UPDATE/SHARE
|
|
|
|
* (if any) before we get here.
|
|
|
|
*/
|
|
|
|
if (!extra->limit_needed)
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
Assert(parse->rowMarks);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Grouping and aggregation are not supported with FOR UPDATE/SHARE,
|
|
|
|
* so the input_rel should be a base, join, or ordered relation; and
|
|
|
|
* if it's an ordered relation, its input relation should be a base or
|
|
|
|
* join relation.
|
|
|
|
*/
|
|
|
|
Assert(input_rel->reloptkind == RELOPT_BASEREL ||
|
|
|
|
input_rel->reloptkind == RELOPT_JOINREL ||
|
|
|
|
(input_rel->reloptkind == RELOPT_UPPER_REL &&
|
|
|
|
ifpinfo->stage == UPPERREL_ORDERED &&
|
|
|
|
(ifpinfo->outerrel->reloptkind == RELOPT_BASEREL ||
|
|
|
|
ifpinfo->outerrel->reloptkind == RELOPT_JOINREL)));
|
|
|
|
|
|
|
|
foreach(lc, input_rel->pathlist)
|
|
|
|
{
|
|
|
|
Path *path = (Path *) lfirst(lc);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* apply_scanjoin_target_to_paths() uses create_projection_path()
|
|
|
|
* to adjust each of its input paths if needed, whereas
|
|
|
|
* create_ordered_paths() uses apply_projection_to_path() to do
|
|
|
|
* that. So the former might have put a ProjectionPath on top of
|
|
|
|
* the ForeignPath; look through ProjectionPath and see if the
|
|
|
|
* path underneath it is ForeignPath.
|
|
|
|
*/
|
|
|
|
if (IsA(path, ForeignPath) ||
|
|
|
|
(IsA(path, ProjectionPath) &&
|
|
|
|
IsA(((ProjectionPath *) path)->subpath, ForeignPath)))
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Create foreign final path; this gets rid of a
|
|
|
|
* no-longer-needed outer plan (if any), which makes the
|
|
|
|
* EXPLAIN output look cleaner
|
|
|
|
*/
|
|
|
|
final_path = create_foreign_upper_path(root,
|
|
|
|
path->parent,
|
|
|
|
path->pathtarget,
|
|
|
|
path->rows,
|
|
|
|
path->startup_cost,
|
|
|
|
path->total_cost,
|
|
|
|
path->pathkeys,
|
|
|
|
NULL, /* no extra plan */
|
|
|
|
NULL); /* no fdw_private */
|
|
|
|
|
|
|
|
/* and add it to the final_rel */
|
|
|
|
add_path(final_rel, (Path *) final_path);
|
|
|
|
|
|
|
|
/* Safe to push down */
|
|
|
|
fpinfo->pushdown_safe = true;
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we get here it means no ForeignPaths; since we would already
|
|
|
|
* have considered pushing down all operations for the query to the
|
|
|
|
* remote server, give up on it.
|
|
|
|
*/
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
Assert(extra->limit_needed);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the input_rel is an ordered relation, replace the input_rel with its
|
|
|
|
* input relation
|
|
|
|
*/
|
|
|
|
if (input_rel->reloptkind == RELOPT_UPPER_REL &&
|
|
|
|
ifpinfo->stage == UPPERREL_ORDERED)
|
|
|
|
{
|
|
|
|
input_rel = ifpinfo->outerrel;
|
|
|
|
ifpinfo = (PgFdwRelationInfo *) input_rel->fdw_private;
|
|
|
|
has_final_sort = true;
|
|
|
|
pathkeys = root->sort_pathkeys;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* The input_rel should be a base, join, or grouping relation */
|
|
|
|
Assert(input_rel->reloptkind == RELOPT_BASEREL ||
|
|
|
|
input_rel->reloptkind == RELOPT_JOINREL ||
|
|
|
|
(input_rel->reloptkind == RELOPT_UPPER_REL &&
|
|
|
|
ifpinfo->stage == UPPERREL_GROUP_AGG));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We try to create a path below by extending a simple foreign path for
|
|
|
|
* the underlying base, join, or grouping relation to perform the final
|
|
|
|
* sort (if has_final_sort) and the LIMIT restriction remotely, which is
|
|
|
|
* stored into the fdw_private list of the resulting path. (We
|
|
|
|
* re-estimate the costs of sorting the underlying relation, if
|
|
|
|
* has_final_sort.)
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Assess if it is safe to push down the LIMIT and OFFSET to the remote
|
|
|
|
* server
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the underlying relation has any local conditions, the LIMIT/OFFSET
|
|
|
|
* cannot be pushed down.
|
|
|
|
*/
|
|
|
|
if (ifpinfo->local_conds)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Also, the LIMIT/OFFSET cannot be pushed down, if their expressions are
|
|
|
|
* not safe to remote.
|
|
|
|
*/
|
|
|
|
if (!is_foreign_expr(root, input_rel, (Expr *) parse->limitOffset) ||
|
|
|
|
!is_foreign_expr(root, input_rel, (Expr *) parse->limitCount))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Safe to push down */
|
|
|
|
fpinfo->pushdown_safe = true;
|
|
|
|
|
|
|
|
/* Construct PgFdwPathExtraData */
|
|
|
|
fpextra = (PgFdwPathExtraData *) palloc0(sizeof(PgFdwPathExtraData));
|
|
|
|
fpextra->target = root->upper_targets[UPPERREL_FINAL];
|
|
|
|
fpextra->has_final_sort = has_final_sort;
|
|
|
|
fpextra->has_limit = extra->limit_needed;
|
|
|
|
fpextra->limit_tuples = extra->limit_tuples;
|
|
|
|
fpextra->count_est = extra->count_est;
|
|
|
|
fpextra->offset_est = extra->offset_est;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Estimate the costs of performing the final sort and the LIMIT
|
|
|
|
* restriction remotely. If has_final_sort is false, we wouldn't need to
|
|
|
|
* execute EXPLAIN anymore if use_remote_estimate, since the costs can be
|
|
|
|
* roughly estimated using the costs we already have for the underlying
|
|
|
|
* relation, in the same way as when use_remote_estimate is false. Since
|
|
|
|
* it's pretty expensive to execute EXPLAIN, force use_remote_estimate to
|
|
|
|
* false in that case.
|
|
|
|
*/
|
|
|
|
if (!fpextra->has_final_sort)
|
|
|
|
{
|
|
|
|
save_use_remote_estimate = ifpinfo->use_remote_estimate;
|
|
|
|
ifpinfo->use_remote_estimate = false;
|
|
|
|
}
|
|
|
|
estimate_path_cost_size(root, input_rel, NIL, pathkeys, fpextra,
|
|
|
|
&rows, &width, &startup_cost, &total_cost);
|
|
|
|
if (!fpextra->has_final_sort)
|
|
|
|
ifpinfo->use_remote_estimate = save_use_remote_estimate;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Build the fdw_private list that will be used by postgresGetForeignPlan.
|
|
|
|
* Items in the list must match order in enum FdwPathPrivateIndex.
|
|
|
|
*/
|
|
|
|
fdw_private = list_make2(makeInteger(has_final_sort),
|
|
|
|
makeInteger(extra->limit_needed));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create foreign final path; this gets rid of a no-longer-needed outer
|
|
|
|
* plan (if any), which makes the EXPLAIN output look cleaner
|
|
|
|
*/
|
|
|
|
final_path = create_foreign_upper_path(root,
|
|
|
|
input_rel,
|
|
|
|
root->upper_targets[UPPERREL_FINAL],
|
|
|
|
rows,
|
|
|
|
startup_cost,
|
|
|
|
total_cost,
|
|
|
|
pathkeys,
|
|
|
|
NULL, /* no extra plan */
|
|
|
|
fdw_private);
|
|
|
|
|
|
|
|
/* and add it to the final_rel */
|
|
|
|
add_path(final_rel, (Path *) final_path);
|
|
|
|
}
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
/*
|
|
|
|
* postgresIsForeignPathAsyncCapable
|
|
|
|
* Check whether a given ForeignPath node is async-capable.
|
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
postgresIsForeignPathAsyncCapable(ForeignPath *path)
|
|
|
|
{
|
|
|
|
RelOptInfo *rel = ((Path *) path)->parent;
|
|
|
|
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private;
|
|
|
|
|
|
|
|
return fpinfo->async_capable;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresForeignAsyncRequest
|
|
|
|
* Asynchronously request next tuple from a foreign PostgreSQL table.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresForeignAsyncRequest(AsyncRequest *areq)
|
|
|
|
{
|
|
|
|
produce_tuple_asynchronously(areq, true);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresForeignAsyncConfigureWait
|
|
|
|
* Configure a file descriptor event for which we wish to wait.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresForeignAsyncConfigureWait(AsyncRequest *areq)
|
|
|
|
{
|
|
|
|
ForeignScanState *node = (ForeignScanState *) areq->requestee;
|
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
|
|
|
AsyncRequest *pendingAreq = fsstate->conn_state->pendingAreq;
|
|
|
|
AppendState *requestor = (AppendState *) areq->requestor;
|
|
|
|
WaitEventSet *set = requestor->as_eventset;
|
|
|
|
|
|
|
|
/* This should not be called unless callback_pending */
|
|
|
|
Assert(areq->callback_pending);
|
|
|
|
|
|
|
|
/* The core code would have registered postmaster death event */
|
|
|
|
Assert(GetNumRegisteredWaitEvents(set) >= 1);
|
|
|
|
|
|
|
|
/* Begin an asynchronous data fetch if not already done */
|
|
|
|
if (!pendingAreq)
|
|
|
|
fetch_more_data_begin(areq);
|
|
|
|
else if (pendingAreq->requestor != areq->requestor)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* This is the case when the in-process request was made by another
|
|
|
|
* Append. Note that it might be useless to process the request,
|
|
|
|
* because the query might not need tuples from that Append anymore.
|
|
|
|
* Skip the given request if there are any configured events other
|
|
|
|
* than the postmaster death event; otherwise process the request,
|
|
|
|
* then begin a fetch to configure the event below, because otherwise
|
|
|
|
* we might end up with no configured events other than the postmaster
|
|
|
|
* death event.
|
|
|
|
*/
|
|
|
|
if (GetNumRegisteredWaitEvents(set) > 1)
|
|
|
|
return;
|
|
|
|
process_pending_request(pendingAreq);
|
|
|
|
fetch_more_data_begin(areq);
|
|
|
|
}
|
|
|
|
else if (pendingAreq->requestee != areq->requestee)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* This is the case when the in-process request was made by the same
|
|
|
|
* parent but for a different child. Since we configure only the
|
|
|
|
* event for the request made for that child, skip the given request.
|
|
|
|
*/
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
Assert(pendingAreq == areq);
|
|
|
|
|
|
|
|
AddWaitEventToSet(set, WL_SOCKET_READABLE, PQsocket(fsstate->conn),
|
|
|
|
NULL, areq);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* postgresForeignAsyncNotify
|
|
|
|
* Fetch some more tuples from a file descriptor that becomes ready,
|
|
|
|
* requesting next tuple.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
postgresForeignAsyncNotify(AsyncRequest *areq)
|
|
|
|
{
|
|
|
|
ForeignScanState *node = (ForeignScanState *) areq->requestee;
|
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
|
|
|
|
|
|
|
/* The request should be currently in-process */
|
|
|
|
Assert(fsstate->conn_state->pendingAreq == areq);
|
|
|
|
|
|
|
|
/* The core code would have initialized the callback_pending flag */
|
|
|
|
Assert(!areq->callback_pending);
|
|
|
|
|
|
|
|
/* On error, report the original query, not the FETCH. */
|
|
|
|
if (!PQconsumeInput(fsstate->conn))
|
|
|
|
pgfdw_report_error(ERROR, NULL, fsstate->conn, false, fsstate->query);
|
|
|
|
|
|
|
|
fetch_more_data(node);
|
|
|
|
|
|
|
|
produce_tuple_asynchronously(areq, true);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Asynchronously produce next tuple from a foreign PostgreSQL table.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
produce_tuple_asynchronously(AsyncRequest *areq, bool fetch)
|
|
|
|
{
|
|
|
|
ForeignScanState *node = (ForeignScanState *) areq->requestee;
|
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
|
|
|
AsyncRequest *pendingAreq = fsstate->conn_state->pendingAreq;
|
|
|
|
TupleTableSlot *result;
|
|
|
|
|
|
|
|
/* This should not be called if the request is currently in-process */
|
|
|
|
Assert(areq != pendingAreq);
|
|
|
|
|
|
|
|
/* Fetch some more tuples, if we've run out */
|
|
|
|
if (fsstate->next_tuple >= fsstate->num_tuples)
|
|
|
|
{
|
|
|
|
/* No point in another fetch if we already detected EOF, though */
|
|
|
|
if (!fsstate->eof_reached)
|
|
|
|
{
|
|
|
|
/* Mark the request as pending for a callback */
|
|
|
|
ExecAsyncRequestPending(areq);
|
|
|
|
/* Begin another fetch if requested and if no pending request */
|
|
|
|
if (fetch && !pendingAreq)
|
|
|
|
fetch_more_data_begin(areq);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* There's nothing more to do; just return a NULL pointer */
|
|
|
|
result = NULL;
|
|
|
|
/* Mark the request as complete */
|
|
|
|
ExecAsyncRequestDone(areq, result);
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Get a tuple from the ForeignScan node */
|
2021-05-12 07:00:00 +02:00
|
|
|
result = areq->requestee->ExecProcNodeReal(areq->requestee);
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
if (!TupIsNull(result))
|
|
|
|
{
|
|
|
|
/* Mark the request as complete */
|
|
|
|
ExecAsyncRequestDone(areq, result);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
Assert(fsstate->next_tuple >= fsstate->num_tuples);
|
|
|
|
|
|
|
|
/* Fetch some more tuples, if we've not detected EOF yet */
|
|
|
|
if (!fsstate->eof_reached)
|
|
|
|
{
|
|
|
|
/* Mark the request as pending for a callback */
|
|
|
|
ExecAsyncRequestPending(areq);
|
|
|
|
/* Begin another fetch if requested and if no pending request */
|
|
|
|
if (fetch && !pendingAreq)
|
|
|
|
fetch_more_data_begin(areq);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* There's nothing more to do; just return a NULL pointer */
|
|
|
|
result = NULL;
|
|
|
|
/* Mark the request as complete */
|
|
|
|
ExecAsyncRequestDone(areq, result);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Begin an asynchronous data fetch.
|
|
|
|
*
|
2021-04-23 05:00:00 +02:00
|
|
|
* Note: this function assumes there is no currently-in-progress asynchronous
|
|
|
|
* data fetch.
|
|
|
|
*
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
* Note: fetch_more_data must be called to fetch the result.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
fetch_more_data_begin(AsyncRequest *areq)
|
|
|
|
{
|
|
|
|
ForeignScanState *node = (ForeignScanState *) areq->requestee;
|
|
|
|
PgFdwScanState *fsstate = (PgFdwScanState *) node->fdw_state;
|
|
|
|
char sql[64];
|
|
|
|
|
|
|
|
Assert(!fsstate->conn_state->pendingAreq);
|
|
|
|
|
|
|
|
/* Create the cursor synchronously. */
|
|
|
|
if (!fsstate->cursor_exists)
|
|
|
|
create_cursor(node);
|
|
|
|
|
|
|
|
/* We will send this query, but not wait for the response. */
|
|
|
|
snprintf(sql, sizeof(sql), "FETCH %d FROM c%u",
|
|
|
|
fsstate->fetch_size, fsstate->cursor_number);
|
|
|
|
|
|
|
|
if (PQsendQuery(fsstate->conn, sql) < 0)
|
|
|
|
pgfdw_report_error(ERROR, NULL, fsstate->conn, false, fsstate->query);
|
|
|
|
|
|
|
|
/* Remember that the request is in process */
|
|
|
|
fsstate->conn_state->pendingAreq = areq;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Process a pending asynchronous request.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
process_pending_request(AsyncRequest *areq)
|
|
|
|
{
|
|
|
|
ForeignScanState *node = (ForeignScanState *) areq->requestee;
|
2021-03-31 22:50:45 +02:00
|
|
|
PgFdwScanState *fsstate PG_USED_FOR_ASSERTS_ONLY = (PgFdwScanState *) node->fdw_state;
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
EState *estate = node->ss.ps.state;
|
|
|
|
MemoryContext oldcontext;
|
|
|
|
|
|
|
|
/* The request should be currently in-process */
|
|
|
|
Assert(fsstate->conn_state->pendingAreq == areq);
|
|
|
|
|
|
|
|
oldcontext = MemoryContextSwitchTo(estate->es_query_cxt);
|
|
|
|
|
|
|
|
/* The request would have been pending for a callback */
|
|
|
|
Assert(areq->callback_pending);
|
|
|
|
|
|
|
|
/* Unlike AsyncNotify, we unset callback_pending ourselves */
|
|
|
|
areq->callback_pending = false;
|
|
|
|
|
|
|
|
fetch_more_data(node);
|
|
|
|
|
|
|
|
/* We need to send a new query afterwards; don't fetch */
|
|
|
|
produce_tuple_asynchronously(areq, false);
|
|
|
|
|
|
|
|
/* Unlike AsyncNotify, we call ExecAsyncResponse ourselves */
|
|
|
|
ExecAsyncResponse(areq);
|
|
|
|
|
2021-05-12 07:00:00 +02:00
|
|
|
/* Also, we do instrumentation ourselves, if required */
|
|
|
|
if (areq->requestee->instrument)
|
|
|
|
InstrUpdateTupleCount(areq->requestee->instrument,
|
|
|
|
TupIsNull(areq->result) ? 0.0 : 1.0);
|
|
|
|
|
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a
non-parallel-aware Append concurrently rather than serially to improve
performance when possible. Currently, the only node type that can be
run concurrently is a ForeignScan that is an immediate child of such an
Append. In the case where such ForeignScans access data on different
remote servers, this would run those ForeignScans concurrently, and
overlap the remote operations to be performed simultaneously, so it'll
improve the performance especially when the operations involve
time-consuming ones such as remote join and remote aggregation.
We may extend this to other node types such as joins or aggregates over
ForeignScans in the future.
This also adds the support for postgres_fdw, which is enabled by the
table-level/server-level option "async_capable". The default is false.
Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit
is mostly based on the patch proposed by Robert Haas, but also uses
stuff from the patch proposed by Kyotaro Horiguchi and from the patch
proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin
Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and
others.
Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com
Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com
Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
2021-03-31 11:45:00 +02:00
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
}
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/*
|
|
|
|
* Create a tuple from the specified row of the PGresult.
|
|
|
|
*
|
|
|
|
* rel is the local representation of the foreign table, attinmeta is
|
2013-03-22 05:31:11 +01:00
|
|
|
* conversion data for the rel's tupdesc, and retrieved_attrs is an
|
|
|
|
* integer list of the table column numbers present in the PGresult.
|
|
|
|
* temp_context is a working context that can be reset after each tuple.
|
2013-02-21 11:26:23 +01:00
|
|
|
*/
|
|
|
|
static HeapTuple
|
|
|
|
make_tuple_from_result_row(PGresult *res,
|
|
|
|
int row,
|
|
|
|
Relation rel,
|
|
|
|
AttInMetadata *attinmeta,
|
2013-03-22 05:31:11 +01:00
|
|
|
List *retrieved_attrs,
|
2016-02-09 20:00:50 +01:00
|
|
|
ForeignScanState *fsstate,
|
2013-02-21 11:26:23 +01:00
|
|
|
MemoryContext temp_context)
|
|
|
|
{
|
|
|
|
HeapTuple tuple;
|
2016-02-09 20:00:50 +01:00
|
|
|
TupleDesc tupdesc;
|
2013-02-21 11:26:23 +01:00
|
|
|
Datum *values;
|
|
|
|
bool *nulls;
|
2013-03-10 19:14:53 +01:00
|
|
|
ItemPointer ctid = NULL;
|
2013-02-21 11:26:23 +01:00
|
|
|
ConversionLocation errpos;
|
|
|
|
ErrorContextCallback errcallback;
|
|
|
|
MemoryContext oldcontext;
|
2013-03-22 05:31:11 +01:00
|
|
|
ListCell *lc;
|
2013-02-21 11:26:23 +01:00
|
|
|
int j;
|
|
|
|
|
|
|
|
Assert(row < PQntuples(res));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Do the following work in a temp context that we reset after each tuple.
|
|
|
|
* This cleans up not only the data we have direct access to, but any
|
|
|
|
* cruft the I/O functions might leak.
|
|
|
|
*/
|
|
|
|
oldcontext = MemoryContextSwitchTo(temp_context);
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
if (rel)
|
|
|
|
tupdesc = RelationGetDescr(rel);
|
|
|
|
else
|
|
|
|
{
|
|
|
|
Assert(fsstate);
|
postgres_fdw: Push down UPDATE/DELETE joins to remote servers.
Commit 0bf3ae88af330496517722e391e7c975e6bad219 allowed direct
foreign table modification; instead of fetching each row, updating
it locally, and then pushing the modification back to the remote
side, we would instead do all the work on the remote server via a
single remote UPDATE or DELETE command. However, that commit only
enabled this optimization when join tree consisted only of the
target table.
This change allows the same optimization when an UPDATE statement
has a FROM clause or a DELETE statement has a USING clause. This
works much like ordinary foreign join pushdown, in that the tables
must be on the same remote server, relevant parts of the query
must be pushdown-safe, and so forth.
Etsuro Fujita, reviewed by Ashutosh Bapat, Rushabh Lathia, and me.
Some formatting corrections by me.
Discussion: http://postgr.es/m/5A57193A.2080003@lab.ntt.co.jp
Discussion: http://postgr.es/m/b9cee735-62f8-6c07-7528-6364ce9347d0@lab.ntt.co.jp
2018-02-07 21:34:30 +01:00
|
|
|
tupdesc = fsstate->ss.ss_ScanTupleSlot->tts_tupleDescriptor;
|
2016-02-09 20:00:50 +01:00
|
|
|
}
|
|
|
|
|
2013-03-22 05:31:11 +01:00
|
|
|
values = (Datum *) palloc0(tupdesc->natts * sizeof(Datum));
|
2013-02-21 11:26:23 +01:00
|
|
|
nulls = (bool *) palloc(tupdesc->natts * sizeof(bool));
|
2013-03-22 05:31:11 +01:00
|
|
|
/* Initialize to nulls for any columns not present in result */
|
|
|
|
memset(nulls, true, tupdesc->natts * sizeof(bool));
|
2013-02-21 11:26:23 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Set up and install callback to report where conversion error occurs.
|
|
|
|
*/
|
|
|
|
errpos.rel = rel;
|
|
|
|
errpos.cur_attno = 0;
|
2016-02-09 20:00:50 +01:00
|
|
|
errpos.fsstate = fsstate;
|
2013-02-21 11:26:23 +01:00
|
|
|
errcallback.callback = conversion_error_callback;
|
|
|
|
errcallback.arg = (void *) &errpos;
|
|
|
|
errcallback.previous = error_context_stack;
|
|
|
|
error_context_stack = &errcallback;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* i indexes columns in the relation, j indexes columns in the PGresult.
|
|
|
|
*/
|
2013-03-22 05:31:11 +01:00
|
|
|
j = 0;
|
|
|
|
foreach(lc, retrieved_attrs)
|
2013-02-21 11:26:23 +01:00
|
|
|
{
|
2013-03-22 05:31:11 +01:00
|
|
|
int i = lfirst_int(lc);
|
2013-02-21 11:26:23 +01:00
|
|
|
char *valstr;
|
|
|
|
|
2013-03-22 05:31:11 +01:00
|
|
|
/* fetch next column's textual value */
|
2013-02-21 11:26:23 +01:00
|
|
|
if (PQgetisnull(res, row, j))
|
|
|
|
valstr = NULL;
|
|
|
|
else
|
|
|
|
valstr = PQgetvalue(res, row, j);
|
|
|
|
|
2016-08-26 15:33:57 +02:00
|
|
|
/*
|
|
|
|
* convert value to internal representation
|
|
|
|
*
|
|
|
|
* Note: we ignore system columns other than ctid and oid in result
|
|
|
|
*/
|
2016-03-15 21:51:56 +01:00
|
|
|
errpos.cur_attno = i;
|
2013-03-22 05:31:11 +01:00
|
|
|
if (i > 0)
|
|
|
|
{
|
|
|
|
/* ordinary column */
|
|
|
|
Assert(i <= tupdesc->natts);
|
|
|
|
nulls[i - 1] = (valstr == NULL);
|
|
|
|
/* Apply the input function even to nulls, to support domains */
|
|
|
|
values[i - 1] = InputFunctionCall(&attinmeta->attinfuncs[i - 1],
|
|
|
|
valstr,
|
|
|
|
attinmeta->attioparams[i - 1],
|
|
|
|
attinmeta->atttypmods[i - 1]);
|
|
|
|
}
|
|
|
|
else if (i == SelfItemPointerAttributeNumber)
|
|
|
|
{
|
2016-08-26 15:33:57 +02:00
|
|
|
/* ctid */
|
2013-03-22 05:31:11 +01:00
|
|
|
if (valstr != NULL)
|
|
|
|
{
|
|
|
|
Datum datum;
|
2013-02-21 11:26:23 +01:00
|
|
|
|
2013-03-22 05:31:11 +01:00
|
|
|
datum = DirectFunctionCall1(tidin, CStringGetDatum(valstr));
|
|
|
|
ctid = (ItemPointer) DatumGetPointer(datum);
|
|
|
|
}
|
|
|
|
}
|
2016-03-15 21:51:56 +01:00
|
|
|
errpos.cur_attno = 0;
|
2013-03-10 19:14:53 +01:00
|
|
|
|
|
|
|
j++;
|
|
|
|
}
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/* Uninstall error context callback. */
|
|
|
|
error_context_stack = errcallback.previous;
|
|
|
|
|
2013-03-22 05:31:11 +01:00
|
|
|
/*
|
|
|
|
* Check we got the expected number of columns. Note: j == 0 and
|
|
|
|
* PQnfields == 1 is expected, since deparse emits a NULL if no columns.
|
|
|
|
*/
|
|
|
|
if (j > 0 && j != PQnfields(res))
|
2013-02-21 11:26:23 +01:00
|
|
|
elog(ERROR, "remote query result does not match the foreign table");
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Build the result tuple in caller's memory context.
|
|
|
|
*/
|
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
|
|
|
|
tuple = heap_form_tuple(tupdesc, values, nulls);
|
|
|
|
|
Fix postgres_fdw to return the right ctid value in EvalPlanQual cases.
If a postgres_fdw foreign table is a non-locked source relation in an
UPDATE, DELETE, or SELECT FOR UPDATE/SHARE, and the query selects its
ctid column, the wrong value would be returned if an EvalPlanQual
recheck occurred. This happened because the foreign table's result row
was copied via the ROW_MARK_COPY code path, and EvalPlanQualFetchRowMarks
just unconditionally set the reconstructed tuple's t_self to "invalid".
To fix that, we can have EvalPlanQualFetchRowMarks copy the composite
datum's t_ctid field, and be sure to initialize that along with t_self
when postgres_fdw constructs a tuple to return.
If we just did that much then EvalPlanQualFetchRowMarks would start
returning "(0,0)" as ctid for all other ROW_MARK_COPY cases, which perhaps
does not matter much, but then again maybe it might. The cause of that is
that heap_form_tuple, which is the ultimate source of all composite datums,
simply leaves t_ctid as zeroes in newly constructed tuples. That seems
like a bad idea on general principles: a field that's really not been
initialized shouldn't appear to have a valid value. So let's eat the
trivial additional overhead of doing "ItemPointerSetInvalid(&(td->t_ctid))"
in heap_form_tuple.
This closes out our handling of Etsuro Fujita's report that tableoid and
ctid weren't correctly set in postgres_fdw EvalPlanQual cases. Along the
way we did a great deal of work to improve FDWs' ability to control row
locking behavior; which was not wasted effort by any means, but it didn't
end up being a fix for this problem because that feature would be too
expensive for postgres_fdw to use all the time.
Although the fix for the tableoid misbehavior was back-patched, I'm
hesitant to do so here; it seems far less likely that people would care
about remote ctid than tableoid, and even such a minor behavioral change
as this in heap_form_tuple is perhaps best not back-patched. So commit
to HEAD only, at least for the moment.
Etsuro Fujita, with some adjustments by me
2015-05-13 20:05:17 +02:00
|
|
|
/*
|
|
|
|
* If we have a CTID to return, install it in both t_self and t_ctid.
|
|
|
|
* t_self is the normal place, but if the tuple is converted to a
|
|
|
|
* composite Datum, t_self will be lost; setting t_ctid allows CTID to be
|
|
|
|
* preserved during EvalPlanQual re-evaluations (see ROW_MARK_COPY code).
|
|
|
|
*/
|
2013-03-10 19:14:53 +01:00
|
|
|
if (ctid)
|
Fix postgres_fdw to return the right ctid value in EvalPlanQual cases.
If a postgres_fdw foreign table is a non-locked source relation in an
UPDATE, DELETE, or SELECT FOR UPDATE/SHARE, and the query selects its
ctid column, the wrong value would be returned if an EvalPlanQual
recheck occurred. This happened because the foreign table's result row
was copied via the ROW_MARK_COPY code path, and EvalPlanQualFetchRowMarks
just unconditionally set the reconstructed tuple's t_self to "invalid".
To fix that, we can have EvalPlanQualFetchRowMarks copy the composite
datum's t_ctid field, and be sure to initialize that along with t_self
when postgres_fdw constructs a tuple to return.
If we just did that much then EvalPlanQualFetchRowMarks would start
returning "(0,0)" as ctid for all other ROW_MARK_COPY cases, which perhaps
does not matter much, but then again maybe it might. The cause of that is
that heap_form_tuple, which is the ultimate source of all composite datums,
simply leaves t_ctid as zeroes in newly constructed tuples. That seems
like a bad idea on general principles: a field that's really not been
initialized shouldn't appear to have a valid value. So let's eat the
trivial additional overhead of doing "ItemPointerSetInvalid(&(td->t_ctid))"
in heap_form_tuple.
This closes out our handling of Etsuro Fujita's report that tableoid and
ctid weren't correctly set in postgres_fdw EvalPlanQual cases. Along the
way we did a great deal of work to improve FDWs' ability to control row
locking behavior; which was not wasted effort by any means, but it didn't
end up being a fix for this problem because that feature would be too
expensive for postgres_fdw to use all the time.
Although the fix for the tableoid misbehavior was back-patched, I'm
hesitant to do so here; it seems far less likely that people would care
about remote ctid than tableoid, and even such a minor behavioral change
as this in heap_form_tuple is perhaps best not back-patched. So commit
to HEAD only, at least for the moment.
Etsuro Fujita, with some adjustments by me
2015-05-13 20:05:17 +02:00
|
|
|
tuple->t_self = tuple->t_data->t_ctid = *ctid;
|
2013-03-10 19:14:53 +01:00
|
|
|
|
2016-04-15 17:58:56 +02:00
|
|
|
/*
|
|
|
|
* Stomp on the xmin, xmax, and cmin fields from the tuple created by
|
|
|
|
* heap_form_tuple. heap_form_tuple actually creates the tuple with
|
|
|
|
* DatumTupleFields, not HeapTupleFields, but the executor expects
|
|
|
|
* HeapTupleFields and will happily extract system columns on that
|
|
|
|
* assumption. If we don't do this then, for example, the tuple length
|
|
|
|
* ends up in the xmin field, which isn't what we want.
|
|
|
|
*/
|
|
|
|
HeapTupleHeaderSetXmax(tuple->t_data, InvalidTransactionId);
|
|
|
|
HeapTupleHeaderSetXmin(tuple->t_data, InvalidTransactionId);
|
|
|
|
HeapTupleHeaderSetCmin(tuple->t_data, InvalidTransactionId);
|
|
|
|
|
2013-02-21 11:26:23 +01:00
|
|
|
/* Clean up */
|
|
|
|
MemoryContextReset(temp_context);
|
|
|
|
|
|
|
|
return tuple;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Callback function which is called when error occurs during column value
|
|
|
|
* conversion. Print names of column and relation.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
conversion_error_callback(void *arg)
|
|
|
|
{
|
2016-02-09 20:00:50 +01:00
|
|
|
const char *attname = NULL;
|
|
|
|
const char *relname = NULL;
|
2016-07-01 17:29:25 +02:00
|
|
|
bool is_wholerow = false;
|
2013-02-21 11:26:23 +01:00
|
|
|
ConversionLocation *errpos = (ConversionLocation *) arg;
|
|
|
|
|
2016-02-09 20:00:50 +01:00
|
|
|
if (errpos->rel)
|
|
|
|
{
|
|
|
|
/* error occurred in a scan against a foreign table */
|
|
|
|
TupleDesc tupdesc = RelationGetDescr(errpos->rel);
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupdesc, errpos->cur_attno - 1);
|
2016-02-09 20:00:50 +01:00
|
|
|
|
|
|
|
if (errpos->cur_attno > 0 && errpos->cur_attno <= tupdesc->natts)
|
2017-08-20 20:19:07 +02:00
|
|
|
attname = NameStr(attr->attname);
|
2016-02-09 20:00:50 +01:00
|
|
|
else if (errpos->cur_attno == SelfItemPointerAttributeNumber)
|
|
|
|
attname = "ctid";
|
|
|
|
|
|
|
|
relname = RelationGetRelationName(errpos->rel);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* error occurred in a scan against a foreign join */
|
|
|
|
ForeignScanState *fsstate = errpos->fsstate;
|
2017-01-27 01:47:03 +01:00
|
|
|
ForeignScan *fsplan = castNode(ForeignScan, fsstate->ss.ps.plan);
|
2016-02-09 20:00:50 +01:00
|
|
|
EState *estate = fsstate->ss.ps.state;
|
|
|
|
TargetEntry *tle;
|
|
|
|
|
Improve castNode notation by introducing list-extraction-specific variants.
This extends the castNode() notation introduced by commit 5bcab1114 to
provide, in one step, extraction of a list cell's pointer and coercion to
a concrete node type. For example, "lfirst_node(Foo, lc)" is the same
as "castNode(Foo, lfirst(lc))". Almost half of the uses of castNode
that have appeared so far include a list extraction call, so this is
pretty widely useful, and it saves a few more keystrokes compared to the
old way.
As with the previous patch, back-patch the addition of these macros to
pg_list.h, so that the notation will be available when back-patching.
Patch by me, after an idea of Andrew Gierth's.
Discussion: https://postgr.es/m/14197.1491841216@sss.pgh.pa.us
2017-04-10 19:51:29 +02:00
|
|
|
tle = list_nth_node(TargetEntry, fsplan->fdw_scan_tlist,
|
|
|
|
errpos->cur_attno - 1);
|
2016-02-09 20:00:50 +01:00
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
/*
|
|
|
|
* Target list can have Vars and expressions. For Vars, we can get
|
2017-12-27 20:01:47 +01:00
|
|
|
* its relation, however for expressions we can't. Thus for
|
2016-10-21 15:54:29 +02:00
|
|
|
* expressions, just show generic context message.
|
|
|
|
*/
|
|
|
|
if (IsA(tle->expr, Var))
|
|
|
|
{
|
|
|
|
RangeTblEntry *rte;
|
|
|
|
Var *var = (Var *) tle->expr;
|
2016-07-01 17:29:25 +02:00
|
|
|
|
2018-10-04 21:48:17 +02:00
|
|
|
rte = exec_rt_fetch(var->varno, estate);
|
2016-10-21 15:54:29 +02:00
|
|
|
|
|
|
|
if (var->varattno == 0)
|
|
|
|
is_wholerow = true;
|
|
|
|
else
|
2018-02-12 23:30:30 +01:00
|
|
|
attname = get_attname(rte->relid, var->varattno, false);
|
2016-07-01 17:29:25 +02:00
|
|
|
|
2016-10-21 15:54:29 +02:00
|
|
|
relname = get_rel_name(rte->relid);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
errcontext("processing expression at position %d in select list",
|
|
|
|
errpos->cur_attno);
|
2016-02-09 20:00:50 +01:00
|
|
|
}
|
|
|
|
|
2016-07-01 17:29:25 +02:00
|
|
|
if (relname)
|
|
|
|
{
|
|
|
|
if (is_wholerow)
|
|
|
|
errcontext("whole-row reference to foreign table \"%s\"", relname);
|
|
|
|
else if (attname)
|
|
|
|
errcontext("column \"%s\" of foreign table \"%s\"", attname, relname);
|
|
|
|
}
|
2013-02-21 11:26:23 +01:00
|
|
|
}
|
2015-11-03 18:46:06 +01:00
|
|
|
|
2019-04-02 12:20:30 +02:00
|
|
|
/*
|
|
|
|
* Find an equivalence class member expression to be computed as a sort column
|
|
|
|
* in the given target.
|
|
|
|
*/
|
|
|
|
Expr *
|
|
|
|
find_em_expr_for_input_target(PlannerInfo *root,
|
|
|
|
EquivalenceClass *ec,
|
|
|
|
PathTarget *target)
|
|
|
|
{
|
|
|
|
ListCell *lc1;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
i = 0;
|
|
|
|
foreach(lc1, target->exprs)
|
|
|
|
{
|
|
|
|
Expr *expr = (Expr *) lfirst(lc1);
|
|
|
|
Index sgref = get_pathtarget_sortgroupref(target, i);
|
|
|
|
ListCell *lc2;
|
|
|
|
|
|
|
|
/* Ignore non-sort expressions */
|
|
|
|
if (sgref == 0 ||
|
|
|
|
get_sortgroupref_clause_noerr(sgref,
|
|
|
|
root->parse->sortClause) == NULL)
|
|
|
|
{
|
|
|
|
i++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We ignore binary-compatible relabeling on both ends */
|
|
|
|
while (expr && IsA(expr, RelabelType))
|
|
|
|
expr = ((RelabelType *) expr)->arg;
|
|
|
|
|
|
|
|
/* Locate an EquivalenceClass member matching this expr, if any */
|
|
|
|
foreach(lc2, ec->ec_members)
|
|
|
|
{
|
|
|
|
EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2);
|
|
|
|
Expr *em_expr;
|
|
|
|
|
|
|
|
/* Don't match constants */
|
|
|
|
if (em->em_is_const)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Ignore child members */
|
|
|
|
if (em->em_is_child)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Match if same expression (after stripping relabel) */
|
|
|
|
em_expr = em->em_expr;
|
|
|
|
while (em_expr && IsA(em_expr, RelabelType))
|
|
|
|
em_expr = ((RelabelType *) em_expr)->arg;
|
|
|
|
|
|
|
|
if (equal(em_expr, expr))
|
|
|
|
return em->em_expr;
|
|
|
|
}
|
|
|
|
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
|
|
|
|
elog(ERROR, "could not find pathkey item to sort");
|
|
|
|
return NULL; /* keep compiler quiet */
|
|
|
|
}
|
2021-01-20 23:05:46 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Determine batch size for a given foreign table. The option specified for
|
|
|
|
* a table has precedence.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
get_batch_size_option(Relation rel)
|
|
|
|
{
|
|
|
|
Oid foreigntableid = RelationGetRelid(rel);
|
|
|
|
ForeignTable *table;
|
|
|
|
ForeignServer *server;
|
|
|
|
List *options;
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
/* we use 1 by default, which means "no batching" */
|
|
|
|
int batch_size = 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Load options for table and server. We append server options after table
|
|
|
|
* options, because table options take precedence.
|
|
|
|
*/
|
|
|
|
table = GetForeignTable(foreigntableid);
|
|
|
|
server = GetForeignServer(table->serverid);
|
|
|
|
|
|
|
|
options = NIL;
|
|
|
|
options = list_concat(options, table->options);
|
|
|
|
options = list_concat(options, server->options);
|
|
|
|
|
|
|
|
/* See if either table or server specifies batch_size. */
|
|
|
|
foreach(lc, options)
|
|
|
|
{
|
|
|
|
DefElem *def = (DefElem *) lfirst(lc);
|
|
|
|
|
|
|
|
if (strcmp(def->defname, "batch_size") == 0)
|
|
|
|
{
|
|
|
|
batch_size = strtol(defGetString(def), NULL, 10);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return batch_size;
|
|
|
|
}
|