From 69f4b9c85f168ae006929eec44fc44d569e846b9 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 18 Jan 2017 12:46:50 -0800 Subject: [PATCH] Move targetlist SRF handling from expression evaluation to new executor node. Evaluation of set returning functions (SRFs_ in the targetlist (like SELECT generate_series(1,5)) so far was done in the expression evaluation (i.e. ExecEvalExpr()) and projection (i.e. ExecProject/ExecTargetList) code. This meant that most executor nodes performing projection, and most expression evaluation functions, had to deal with the possibility that an evaluated expression could return a set of return values. That's bad because it leads to repeated code in a lot of places. It also, and that's my (Andres's) motivation, made it a lot harder to implement a more efficient way of doing expression evaluation. To fix this, introduce a new executor node (ProjectSet) that can evaluate targetlists containing one or more SRFs. To avoid the complexity of the old way of handling nested expressions returning sets (e.g. having to pass up ExprDoneCond, and dealing with arguments to functions returning sets etc.), those SRFs can only be at the top level of the node's targetlist. The planner makes sure (via split_pathtarget_at_srfs()) that SRF evaluation is only necessary in ProjectSet nodes and that SRFs are only present at the top level of the node's targetlist. If there are nested SRFs the planner creates multiple stacked ProjectSet nodes. The ProjectSet nodes always get input from an underlying node. We also discussed and prototyped evaluating targetlist SRFs using ROWS FROM(), but that turned out to be more complicated than we'd hoped. While moving SRF evaluation to ProjectSet would allow to retain the old "least common multiple" behavior when multiple SRFs are present in one targetlist (i.e. continue returning rows until all SRFs are at the end of their input at the same time), we decided to instead only return rows till all SRFs are exhausted, returning NULL for already exhausted ones. We deemed the previous behavior to be too confusing, unexpected and actually not particularly useful. As a side effect, the previously prohibited case of multiple set returning arguments to a function, is now allowed. Not because it's particularly desirable, but because it ends up working and there seems to be no argument for adding code to prohibit it. Currently the behavior for COALESCE and CASE containing SRFs has changed, returning multiple rows from the expression, even when the SRF containing "arm" of the expression is not evaluated. That's because the SRFs are evaluated in a separate ProjectSet node. As that's quite confusing, we're likely to instead prohibit SRFs in those places. But that's still being discussed, and the code would reside in places not touched here, so that's a task for later. There's a lot of, now superfluous, code dealing with set return expressions around. But as the changes to get rid of those are verbose largely boring, it seems better for readability to keep the cleanup as a separate commit. Author: Tom Lane and Andres Freund Discussion: https://postgr.es/m/20160822214023.aaxz5l4igypowyri@alap3.anarazel.de --- doc/src/sgml/xfunc.sgml | 52 +++- src/backend/commands/explain.c | 3 + src/backend/executor/Makefile | 5 +- src/backend/executor/execAmi.c | 5 + src/backend/executor/execProcnode.c | 16 +- src/backend/executor/execQual.c | 154 ++++++------ src/backend/executor/nodeProjectSet.c | 300 +++++++++++++++++++++++ src/backend/nodes/copyfuncs.c | 19 ++ src/backend/nodes/outfuncs.c | 24 ++ src/backend/nodes/readfuncs.c | 15 ++ src/backend/optimizer/README | 1 + src/backend/optimizer/path/allpaths.c | 4 + src/backend/optimizer/plan/createplan.c | 68 +++++ src/backend/optimizer/plan/planner.c | 219 ++++++++++++++--- src/backend/optimizer/plan/setrefs.c | 3 + src/backend/optimizer/plan/subselect.c | 2 + src/backend/optimizer/util/clauses.c | 104 +------- src/backend/optimizer/util/pathnode.c | 66 +++++ src/backend/optimizer/util/tlist.c | 199 +++++++++++++++ src/include/executor/executor.h | 4 + src/include/executor/nodeProjectSet.h | 24 ++ src/include/nodes/execnodes.h | 20 +- src/include/nodes/nodes.h | 3 + src/include/nodes/plannodes.h | 11 + src/include/nodes/relation.h | 11 + src/include/optimizer/clauses.h | 1 - src/include/optimizer/pathnode.h | 4 + src/include/optimizer/tlist.h | 3 + src/test/regress/expected/aggregates.out | 5 +- src/test/regress/expected/limit.out | 22 +- src/test/regress/expected/portals.out | 16 +- src/test/regress/expected/rangefuncs.out | 10 +- src/test/regress/expected/subselect.out | 43 ++-- src/test/regress/expected/tsrf.out | 19 +- src/test/regress/expected/union.out | 5 +- 35 files changed, 1186 insertions(+), 274 deletions(-) create mode 100644 src/backend/executor/nodeProjectSet.c create mode 100644 src/include/executor/nodeProjectSet.h diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index f2f379870f..09427bbed2 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -962,12 +962,11 @@ SELECT name, child FROM nodes, LATERAL listchildren(name) AS child; - Currently, functions returning sets can also be called in the select list + Functions returning sets can also be called in the select list of a query. For each row that the query - generates by itself, the function returning set is invoked, and an output - row is generated for each element of the function's result set. Note, - however, that this capability is deprecated and might be removed in future - releases. The previous example could also be done with queries like + generates by itself, the set-returning function is invoked, and an output + row is generated for each element of the function's result set. + The previous example could also be done with queries like these: @@ -998,6 +997,33 @@ SELECT name, listchildren(name) FROM nodes; the LATERAL syntax. + + If there is more than one set-returning function in the same select + list, the behavior is similar to what you get from putting the functions + into a single LATERAL ROWS FROM( ... ) FROM-clause + item. For each row from the underlying query, there is an output row + using the first result from each function, then an output row using the + second result, and so on. If some of the set-returning functions + produce fewer outputs than others, null values are substituted for the + missing data, so that the total number of rows emitted for one + underlying row is the same as for the set-returning function that + produced the most outputs. + + + + Set-returning functions can be nested in a select list, although that is + not allowed in FROM-clause items. In such cases, each level + of nesting is treated separately, as though it were + another LATERAL ROWS FROM( ... ) item. For example, in + +SELECT srf1(srf2(x), srf3(y)), srf4(srf5(z)) FROM ... + + the set-returning functions srf2, srf3, + and srf5 would be run in lockstep for each row of the + underlying query, and then srf1 and srf4 would + be applied in lockstep to each row produced by the lower functions. + + If a function's last command is INSERT, UPDATE, @@ -1012,14 +1038,14 @@ SELECT name, listchildren(name) FROM nodes; - The key problem with using set-returning functions in the select list, - rather than the FROM clause, is that putting more than one - set-returning function in the same select list does not behave very - sensibly. (What you actually get if you do so is a number of output - rows equal to the least common multiple of the numbers of rows produced - by each set-returning function.) The LATERAL syntax - produces less surprising results when calling multiple set-returning - functions, and should usually be used instead. + Before PostgreSQL 10, putting more than one + set-returning function in the same select list did not behave very + sensibly unless they always produced equal numbers of rows. Otherwise, + what you got was a number of output rows equal to the least common + multiple of the numbers of rows produced by the set-returning + functions. Furthermore, nested set-returning functions did not work at + all. Use of the LATERAL syntax is recommended when writing + queries that need to work in older PostgreSQL versions. diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index ee7046c47b..f9fb27658f 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -852,6 +852,9 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_Result: pname = sname = "Result"; break; + case T_ProjectSet: + pname = sname = "ProjectSet"; + break; case T_ModifyTable: sname = "ModifyTable"; switch (((ModifyTable *) plan)->operation) diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile index 51edd4c5e7..c51415830a 100644 --- a/src/backend/executor/Makefile +++ b/src/backend/executor/Makefile @@ -17,11 +17,12 @@ OBJS = execAmi.o execCurrent.o execGrouping.o execIndexing.o execJunk.o \ execScan.o execTuples.o \ execUtils.o functions.o instrument.o nodeAppend.o nodeAgg.o \ nodeBitmapAnd.o nodeBitmapOr.o \ - nodeBitmapHeapscan.o nodeBitmapIndexscan.o nodeCustom.o nodeGather.o \ + nodeBitmapHeapscan.o nodeBitmapIndexscan.o \ + nodeCustom.o nodeFunctionscan.o nodeGather.o \ nodeHash.o nodeHashjoin.o nodeIndexscan.o nodeIndexonlyscan.o \ nodeLimit.o nodeLockRows.o \ nodeMaterial.o nodeMergeAppend.o nodeMergejoin.o nodeModifyTable.o \ - nodeNestloop.o nodeFunctionscan.o nodeRecursiveunion.o nodeResult.o \ + nodeNestloop.o nodeProjectSet.o nodeRecursiveunion.o nodeResult.o \ nodeSamplescan.o nodeSeqscan.o nodeSetOp.o nodeSort.o nodeUnique.o \ nodeValuesscan.o nodeCtescan.o nodeWorktablescan.o \ nodeGroup.o nodeSubplan.o nodeSubqueryscan.o nodeTidscan.o \ diff --git a/src/backend/executor/execAmi.c b/src/backend/executor/execAmi.c index 3ea36979b3..b52cfaa41f 100644 --- a/src/backend/executor/execAmi.c +++ b/src/backend/executor/execAmi.c @@ -39,6 +39,7 @@ #include "executor/nodeMergejoin.h" #include "executor/nodeModifyTable.h" #include "executor/nodeNestloop.h" +#include "executor/nodeProjectSet.h" #include "executor/nodeRecursiveunion.h" #include "executor/nodeResult.h" #include "executor/nodeSamplescan.h" @@ -130,6 +131,10 @@ ExecReScan(PlanState *node) ExecReScanResult((ResultState *) node); break; + case T_ProjectSetState: + ExecReScanProjectSet((ProjectSetState *) node); + break; + case T_ModifyTableState: ExecReScanModifyTable((ModifyTableState *) node); break; diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c index b8edd36470..0dd95c6d17 100644 --- a/src/backend/executor/execProcnode.c +++ b/src/backend/executor/execProcnode.c @@ -88,6 +88,7 @@ #include "executor/nodeCustom.h" #include "executor/nodeForeignscan.h" #include "executor/nodeFunctionscan.h" +#include "executor/nodeGather.h" #include "executor/nodeGroup.h" #include "executor/nodeHash.h" #include "executor/nodeHashjoin.h" @@ -100,7 +101,7 @@ #include "executor/nodeMergejoin.h" #include "executor/nodeModifyTable.h" #include "executor/nodeNestloop.h" -#include "executor/nodeGather.h" +#include "executor/nodeProjectSet.h" #include "executor/nodeRecursiveunion.h" #include "executor/nodeResult.h" #include "executor/nodeSamplescan.h" @@ -155,6 +156,11 @@ ExecInitNode(Plan *node, EState *estate, int eflags) estate, eflags); break; + case T_ProjectSet: + result = (PlanState *) ExecInitProjectSet((ProjectSet *) node, + estate, eflags); + break; + case T_ModifyTable: result = (PlanState *) ExecInitModifyTable((ModifyTable *) node, estate, eflags); @@ -392,6 +398,10 @@ ExecProcNode(PlanState *node) result = ExecResult((ResultState *) node); break; + case T_ProjectSetState: + result = ExecProjectSet((ProjectSetState *) node); + break; + case T_ModifyTableState: result = ExecModifyTable((ModifyTableState *) node); break; @@ -634,6 +644,10 @@ ExecEndNode(PlanState *node) ExecEndResult((ResultState *) node); break; + case T_ProjectSetState: + ExecEndProjectSet((ProjectSetState *) node); + break; + case T_ModifyTableState: ExecEndModifyTable((ModifyTableState *) node); break; diff --git a/src/backend/executor/execQual.c b/src/backend/executor/execQual.c index bf007b7efd..eed7e95c75 100644 --- a/src/backend/executor/execQual.c +++ b/src/backend/executor/execQual.c @@ -29,9 +29,9 @@ * instead of doing needless copying. -cim 5/31/91 * * During expression evaluation, we check_stack_depth only in - * ExecMakeFunctionResult (and substitute routines) rather than at every - * single node. This is a compromise that trades off precision of the - * stack limit setting to gain speed. + * ExecMakeFunctionResultSet/ExecMakeFunctionResultNoSets rather than at + * every single node. This is a compromise that trades off precision of + * the stack limit setting to gain speed. */ #include "postgres.h" @@ -92,7 +92,7 @@ static Datum ExecEvalParamExec(ExprState *exprstate, ExprContext *econtext, static Datum ExecEvalParamExtern(ExprState *exprstate, ExprContext *econtext, bool *isNull, ExprDoneCond *isDone); static void init_fcache(Oid foid, Oid input_collation, FuncExprState *fcache, - MemoryContext fcacheCxt, bool needDescForSets); + MemoryContext fcacheCxt, bool allowSRF, bool needDescForSRF); static void ShutdownFuncExpr(Datum arg); static TupleDesc get_cached_rowtype(Oid type_id, int32 typmod, TupleDesc *cache_field, ExprContext *econtext); @@ -104,10 +104,6 @@ static void ExecPrepareTuplestoreResult(FuncExprState *fcache, Tuplestorestate *resultStore, TupleDesc resultDesc); static void tupledesc_match(TupleDesc dst_tupdesc, TupleDesc src_tupdesc); -static Datum ExecMakeFunctionResult(FuncExprState *fcache, - ExprContext *econtext, - bool *isNull, - ExprDoneCond *isDone); static Datum ExecMakeFunctionResultNoSets(FuncExprState *fcache, ExprContext *econtext, bool *isNull, ExprDoneCond *isDone); @@ -1327,7 +1323,7 @@ GetAttributeByName(HeapTupleHeader tuple, const char *attname, bool *isNull) */ static void init_fcache(Oid foid, Oid input_collation, FuncExprState *fcache, - MemoryContext fcacheCxt, bool needDescForSets) + MemoryContext fcacheCxt, bool allowSRF, bool needDescForSRF) { AclResult aclresult; @@ -1360,8 +1356,17 @@ init_fcache(Oid foid, Oid input_collation, FuncExprState *fcache, list_length(fcache->args), input_collation, NULL, NULL); + /* If function returns set, check if that's allowed by caller */ + if (fcache->func.fn_retset && !allowSRF) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + + /* Otherwise, ExecInitExpr should have marked the fcache correctly */ + Assert(fcache->func.fn_retset == fcache->funcReturnsSet); + /* If function returns set, prepare expected tuple descriptor */ - if (fcache->func.fn_retset && needDescForSets) + if (fcache->func.fn_retset && needDescForSRF) { TypeFuncClass functypclass; Oid funcrettype; @@ -1549,7 +1554,7 @@ ExecEvalFuncArgs(FunctionCallInfo fcinfo, /* * ExecPrepareTuplestoreResult * - * Subroutine for ExecMakeFunctionResult: prepare to extract rows from a + * Subroutine for ExecMakeFunctionResultSet: prepare to extract rows from a * tuplestore function result. We must set up a funcResultSlot (unless * already done in a previous call cycle) and verify that the function * returned the expected tuple descriptor. @@ -1673,19 +1678,17 @@ tupledesc_match(TupleDesc dst_tupdesc, TupleDesc src_tupdesc) } /* - * ExecMakeFunctionResult + * ExecMakeFunctionResultSet * - * Evaluate the arguments to a function and then the function itself. - * init_fcache is presumed already run on the FuncExprState. - * - * This function handles the most general case, wherein the function or - * one of its arguments can return a set. + * Evaluate the arguments to a set-returning function and then call the + * function itself. The argument expressions may not contain set-returning + * functions (the planner is supposed to have separated evaluation for those). */ -static Datum -ExecMakeFunctionResult(FuncExprState *fcache, - ExprContext *econtext, - bool *isNull, - ExprDoneCond *isDone) +Datum +ExecMakeFunctionResultSet(FuncExprState *fcache, + ExprContext *econtext, + bool *isNull, + ExprDoneCond *isDone) { List *arguments; Datum result; @@ -1701,6 +1704,31 @@ restart: /* Guard against stack overflow due to overly complex expressions */ check_stack_depth(); + /* + * Initialize function cache if first time through. The expression node + * could be either a FuncExpr or an OpExpr. + */ + if (fcache->func.fn_oid == InvalidOid) + { + if (IsA(fcache->xprstate.expr, FuncExpr)) + { + FuncExpr *func = (FuncExpr *) fcache->xprstate.expr; + + init_fcache(func->funcid, func->inputcollid, fcache, + econtext->ecxt_per_query_memory, true, true); + } + else if (IsA(fcache->xprstate.expr, OpExpr)) + { + OpExpr *op = (OpExpr *) fcache->xprstate.expr; + + init_fcache(op->opfuncid, op->inputcollid, fcache, + econtext->ecxt_per_query_memory, true, true); + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(fcache->xprstate.expr)); + } + /* * If a previous call of the function returned a set result in the form of * a tuplestore, continue reading rows from the tuplestore until it's @@ -1750,19 +1778,11 @@ restart: if (!fcache->setArgsValid) { argDone = ExecEvalFuncArgs(fcinfo, arguments, econtext); - if (argDone == ExprEndResult) - { - /* input is an empty set, so return an empty set. */ - *isNull = true; - if (isDone) - *isDone = ExprEndResult; - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - return (Datum) 0; - } - hasSetArg = (argDone != ExprSingleResult); + if (argDone != ExprSingleResult) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + hasSetArg = false; } else { @@ -1989,8 +2009,8 @@ restart: /* * ExecMakeFunctionResultNoSets * - * Simplified version of ExecMakeFunctionResult that can only handle - * non-set cases. Hand-tuned for speed. + * Evaluate a function or operator node with a non-set-returning function. + * Assumes init_fcache() already done. Hand-tuned for speed. */ static Datum ExecMakeFunctionResultNoSets(FuncExprState *fcache, @@ -2120,7 +2140,7 @@ ExecMakeTableFunctionResult(ExprState *funcexpr, ExprDoneCond argDone; /* - * This path is similar to ExecMakeFunctionResult. + * This path is similar to ExecMakeFunctionResultSet. */ direct_function_call = true; @@ -2132,7 +2152,7 @@ ExecMakeTableFunctionResult(ExprState *funcexpr, FuncExpr *func = (FuncExpr *) fcache->xprstate.expr; init_fcache(func->funcid, func->inputcollid, fcache, - econtext->ecxt_per_query_memory, false); + econtext->ecxt_per_query_memory, true, false); } returnsSet = fcache->func.fn_retset; InitFunctionCallInfoData(fcinfo, &(fcache->func), @@ -2423,24 +2443,11 @@ ExecEvalFunc(FuncExprState *fcache, /* Initialize function lookup info */ init_fcache(func->funcid, func->inputcollid, fcache, - econtext->ecxt_per_query_memory, true); + econtext->ecxt_per_query_memory, false, false); - /* - * We need to invoke ExecMakeFunctionResult if either the function itself - * or any of its input expressions can return a set. Otherwise, invoke - * ExecMakeFunctionResultNoSets. In either case, change the evalfunc - * pointer to go directly there on subsequent uses. - */ - if (fcache->func.fn_retset || expression_returns_set((Node *) func->args)) - { - fcache->xprstate.evalfunc = (ExprStateEvalFunc) ExecMakeFunctionResult; - return ExecMakeFunctionResult(fcache, econtext, isNull, isDone); - } - else - { - fcache->xprstate.evalfunc = (ExprStateEvalFunc) ExecMakeFunctionResultNoSets; - return ExecMakeFunctionResultNoSets(fcache, econtext, isNull, isDone); - } + /* Change the evalfunc pointer to save a few cycles in additional calls */ + fcache->xprstate.evalfunc = (ExprStateEvalFunc) ExecMakeFunctionResultNoSets; + return ExecMakeFunctionResultNoSets(fcache, econtext, isNull, isDone); } /* ---------------------------------------------------------------- @@ -2458,24 +2465,11 @@ ExecEvalOper(FuncExprState *fcache, /* Initialize function lookup info */ init_fcache(op->opfuncid, op->inputcollid, fcache, - econtext->ecxt_per_query_memory, true); + econtext->ecxt_per_query_memory, false, false); - /* - * We need to invoke ExecMakeFunctionResult if either the function itself - * or any of its input expressions can return a set. Otherwise, invoke - * ExecMakeFunctionResultNoSets. In either case, change the evalfunc - * pointer to go directly there on subsequent uses. - */ - if (fcache->func.fn_retset || expression_returns_set((Node *) op->args)) - { - fcache->xprstate.evalfunc = (ExprStateEvalFunc) ExecMakeFunctionResult; - return ExecMakeFunctionResult(fcache, econtext, isNull, isDone); - } - else - { - fcache->xprstate.evalfunc = (ExprStateEvalFunc) ExecMakeFunctionResultNoSets; - return ExecMakeFunctionResultNoSets(fcache, econtext, isNull, isDone); - } + /* Change the evalfunc pointer to save a few cycles in additional calls */ + fcache->xprstate.evalfunc = (ExprStateEvalFunc) ExecMakeFunctionResultNoSets; + return ExecMakeFunctionResultNoSets(fcache, econtext, isNull, isDone); } /* ---------------------------------------------------------------- @@ -2512,8 +2506,7 @@ ExecEvalDistinct(FuncExprState *fcache, DistinctExpr *op = (DistinctExpr *) fcache->xprstate.expr; init_fcache(op->opfuncid, op->inputcollid, fcache, - econtext->ecxt_per_query_memory, true); - Assert(!fcache->func.fn_retset); + econtext->ecxt_per_query_memory, false, false); } /* @@ -2589,8 +2582,7 @@ ExecEvalScalarArrayOp(ScalarArrayOpExprState *sstate, if (sstate->fxprstate.func.fn_oid == InvalidOid) { init_fcache(opexpr->opfuncid, opexpr->inputcollid, &sstate->fxprstate, - econtext->ecxt_per_query_memory, true); - Assert(!sstate->fxprstate.func.fn_retset); + econtext->ecxt_per_query_memory, false, false); } /* @@ -3857,8 +3849,7 @@ ExecEvalNullIf(FuncExprState *nullIfExpr, NullIfExpr *op = (NullIfExpr *) nullIfExpr->xprstate.expr; init_fcache(op->opfuncid, op->inputcollid, nullIfExpr, - econtext->ecxt_per_query_memory, true); - Assert(!nullIfExpr->func.fn_retset); + econtext->ecxt_per_query_memory, false, false); } /* @@ -4739,6 +4730,7 @@ ExecInitExpr(Expr *node, PlanState *parent) fstate->args = (List *) ExecInitExpr((Expr *) funcexpr->args, parent); fstate->func.fn_oid = InvalidOid; /* not initialized */ + fstate->funcReturnsSet = funcexpr->funcretset; state = (ExprState *) fstate; } break; @@ -4751,6 +4743,7 @@ ExecInitExpr(Expr *node, PlanState *parent) fstate->args = (List *) ExecInitExpr((Expr *) opexpr->args, parent); fstate->func.fn_oid = InvalidOid; /* not initialized */ + fstate->funcReturnsSet = opexpr->opretset; state = (ExprState *) fstate; } break; @@ -4763,6 +4756,7 @@ ExecInitExpr(Expr *node, PlanState *parent) fstate->args = (List *) ExecInitExpr((Expr *) distinctexpr->args, parent); fstate->func.fn_oid = InvalidOid; /* not initialized */ + fstate->funcReturnsSet = false; /* not supported */ state = (ExprState *) fstate; } break; @@ -4775,6 +4769,7 @@ ExecInitExpr(Expr *node, PlanState *parent) fstate->args = (List *) ExecInitExpr((Expr *) nullifexpr->args, parent); fstate->func.fn_oid = InvalidOid; /* not initialized */ + fstate->funcReturnsSet = false; /* not supported */ state = (ExprState *) fstate; } break; @@ -4787,6 +4782,7 @@ ExecInitExpr(Expr *node, PlanState *parent) sstate->fxprstate.args = (List *) ExecInitExpr((Expr *) opexpr->args, parent); sstate->fxprstate.func.fn_oid = InvalidOid; /* not initialized */ + sstate->fxprstate.funcReturnsSet = false; /* not supported */ sstate->element_type = InvalidOid; /* ditto */ state = (ExprState *) sstate; } diff --git a/src/backend/executor/nodeProjectSet.c b/src/backend/executor/nodeProjectSet.c new file mode 100644 index 0000000000..391e97ea6f --- /dev/null +++ b/src/backend/executor/nodeProjectSet.c @@ -0,0 +1,300 @@ +/*------------------------------------------------------------------------- + * + * nodeProjectSet.c + * support for evaluating targetlists containing set-returning functions + * + * DESCRIPTION + * + * ProjectSet nodes are inserted by the planner to evaluate set-returning + * functions in the targetlist. It's guaranteed that all set-returning + * functions are directly at the top level of the targetlist, i.e. they + * can't be inside more-complex expressions. If that'd otherwise be + * the case, the planner adds additional ProjectSet nodes. + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/executor/nodeProjectSet.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "executor/executor.h" +#include "executor/nodeProjectSet.h" +#include "utils/memutils.h" + + +static TupleTableSlot *ExecProjectSRF(ProjectSetState *node, bool continuing); + + +/* ---------------------------------------------------------------- + * ExecProjectSet(node) + * + * Return tuples after evaluating the targetlist (which contains set + * returning functions). + * ---------------------------------------------------------------- + */ +TupleTableSlot * +ExecProjectSet(ProjectSetState *node) +{ + TupleTableSlot *outerTupleSlot; + TupleTableSlot *resultSlot; + PlanState *outerPlan; + ExprContext *econtext; + + econtext = node->ps.ps_ExprContext; + + /* + * Check to see if we're still projecting out tuples from a previous scan + * tuple (because there is a function-returning-set in the projection + * expressions). If so, try to project another one. + */ + if (node->pending_srf_tuples) + { + resultSlot = ExecProjectSRF(node, true); + + if (resultSlot != NULL) + return resultSlot; + } + + /* + * Reset per-tuple memory context to free any expression evaluation + * storage allocated in the previous tuple cycle. Note this can't happen + * until we're done projecting out tuples from a scan tuple. + */ + ResetExprContext(econtext); + + /* + * Get another input tuple and project SRFs from it. + */ + for (;;) + { + /* + * Retrieve tuples from the outer plan until there are no more. + */ + outerPlan = outerPlanState(node); + outerTupleSlot = ExecProcNode(outerPlan); + + if (TupIsNull(outerTupleSlot)) + return NULL; + + /* + * Prepare to compute projection expressions, which will expect to + * access the input tuples as varno OUTER. + */ + econtext->ecxt_outertuple = outerTupleSlot; + + /* Evaluate the expressions */ + resultSlot = ExecProjectSRF(node, false); + + /* + * Return the tuple unless the projection produced no rows (due to an + * empty set), in which case we must loop back to see if there are + * more outerPlan tuples. + */ + if (resultSlot) + return resultSlot; + } + + return NULL; +} + +/* ---------------------------------------------------------------- + * ExecProjectSRF + * + * Project a targetlist containing one or more set-returning functions. + * + * 'continuing' indicates whether to continue projecting rows for the + * same input tuple; or whether a new input tuple is being projected. + * + * Returns NULL if no output tuple has been produced. + * + * ---------------------------------------------------------------- + */ +static TupleTableSlot * +ExecProjectSRF(ProjectSetState *node, bool continuing) +{ + TupleTableSlot *resultSlot = node->ps.ps_ResultTupleSlot; + ExprContext *econtext = node->ps.ps_ExprContext; + bool hassrf PG_USED_FOR_ASSERTS_ONLY = false; + bool hasresult; + int argno; + ListCell *lc; + + ExecClearTuple(resultSlot); + + /* + * Assume no further tuples are produced unless an ExprMultipleResult is + * encountered from a set returning function. + */ + node->pending_srf_tuples = false; + + hasresult = false; + argno = 0; + foreach(lc, node->ps.targetlist) + { + GenericExprState *gstate = (GenericExprState *) lfirst(lc); + ExprDoneCond *isdone = &node->elemdone[argno]; + Datum *result = &resultSlot->tts_values[argno]; + bool *isnull = &resultSlot->tts_isnull[argno]; + + if (continuing && *isdone == ExprEndResult) + { + /* + * If we're continuing to project output rows from a source tuple, + * return NULLs once the SRF has been exhausted. + */ + *result = (Datum) 0; + *isnull = true; + hassrf = true; + } + else if (IsA(gstate->arg, FuncExprState) && + ((FuncExprState *) gstate->arg)->funcReturnsSet) + { + /* + * Evaluate SRF - possibly continuing previously started output. + */ + *result = ExecMakeFunctionResultSet((FuncExprState *) gstate->arg, + econtext, isnull, isdone); + + if (*isdone != ExprEndResult) + hasresult = true; + if (*isdone == ExprMultipleResult) + node->pending_srf_tuples = true; + hassrf = true; + } + else + { + /* Non-SRF tlist expression, just evaluate normally. */ + *result = ExecEvalExpr(gstate->arg, econtext, isnull, NULL); + *isdone = ExprSingleResult; + } + + argno++; + } + + /* ProjectSet should not be used if there's no SRFs */ + Assert(hassrf); + + /* + * If all the SRFs returned EndResult, we consider that as no row being + * produced. + */ + if (hasresult) + { + ExecStoreVirtualTuple(resultSlot); + return resultSlot; + } + + return NULL; +} + +/* ---------------------------------------------------------------- + * ExecInitProjectSet + * + * Creates the run-time state information for the ProjectSet node + * produced by the planner and initializes outer relations + * (child nodes). + * ---------------------------------------------------------------- + */ +ProjectSetState * +ExecInitProjectSet(ProjectSet *node, EState *estate, int eflags) +{ + ProjectSetState *state; + + /* check for unsupported flags */ + Assert(!(eflags & (EXEC_FLAG_MARK | EXEC_FLAG_BACKWARD))); + + /* + * create state structure + */ + state = makeNode(ProjectSetState); + state->ps.plan = (Plan *) node; + state->ps.state = estate; + + state->pending_srf_tuples = false; + + /* + * Miscellaneous initialization + * + * create expression context for node + */ + ExecAssignExprContext(estate, &state->ps); + + /* + * tuple table initialization + */ + ExecInitResultTupleSlot(estate, &state->ps); + + /* + * initialize child expressions + */ + state->ps.targetlist = (List *) + ExecInitExpr((Expr *) node->plan.targetlist, + (PlanState *) state); + Assert(node->plan.qual == NIL); + + /* + * initialize child nodes + */ + outerPlanState(state) = ExecInitNode(outerPlan(node), estate, eflags); + + /* + * we don't use inner plan + */ + Assert(innerPlan(node) == NULL); + + /* + * initialize tuple type and projection info + */ + ExecAssignResultTypeFromTL(&state->ps); + + /* Create workspace for per-SRF is-done state */ + state->nelems = list_length(node->plan.targetlist); + state->elemdone = (ExprDoneCond *) + palloc(sizeof(ExprDoneCond) * state->nelems); + + return state; +} + +/* ---------------------------------------------------------------- + * ExecEndProjectSet + * + * frees up storage allocated through C routines + * ---------------------------------------------------------------- + */ +void +ExecEndProjectSet(ProjectSetState *node) +{ + /* + * Free the exprcontext + */ + ExecFreeExprContext(&node->ps); + + /* + * clean out the tuple table + */ + ExecClearTuple(node->ps.ps_ResultTupleSlot); + + /* + * shut down subplans + */ + ExecEndNode(outerPlanState(node)); +} + +void +ExecReScanProjectSet(ProjectSetState *node) +{ + /* Forget any incompletely-evaluated SRFs */ + node->pending_srf_tuples = false; + + /* + * If chgParam of subnode is not null then plan will be re-scanned by + * first ExecProcNode. + */ + if (node->ps.lefttree->chgParam == NULL) + ExecReScan(node->ps.lefttree); +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 9f6a7e6154..f871e9d4bb 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -165,6 +165,22 @@ _copyResult(const Result *from) return newnode; } +/* + * _copyProjectSet + */ +static ProjectSet * +_copyProjectSet(const ProjectSet *from) +{ + ProjectSet *newnode = makeNode(ProjectSet); + + /* + * copy node superclass fields + */ + CopyPlanFields((const Plan *) from, (Plan *) newnode); + + return newnode; +} + /* * _copyModifyTable */ @@ -4415,6 +4431,9 @@ copyObject(const void *from) case T_Result: retval = _copyResult(from); break; + case T_ProjectSet: + retval = _copyProjectSet(from); + break; case T_ModifyTable: retval = _copyModifyTable(from); break; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index c2ba38ecd6..1560ac3989 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -326,6 +326,14 @@ _outResult(StringInfo str, const Result *node) WRITE_NODE_FIELD(resconstantqual); } +static void +_outProjectSet(StringInfo str, const ProjectSet *node) +{ + WRITE_NODE_TYPE("PROJECTSET"); + + _outPlanInfo(str, (const Plan *) node); +} + static void _outModifyTable(StringInfo str, const ModifyTable *node) { @@ -1807,6 +1815,16 @@ _outProjectionPath(StringInfo str, const ProjectionPath *node) WRITE_BOOL_FIELD(dummypp); } +static void +_outProjectSetPath(StringInfo str, const ProjectSetPath *node) +{ + WRITE_NODE_TYPE("PROJECTSETPATH"); + + _outPathInfo(str, (const Path *) node); + + WRITE_NODE_FIELD(subpath); +} + static void _outSortPath(StringInfo str, const SortPath *node) { @@ -3367,6 +3385,9 @@ outNode(StringInfo str, const void *obj) case T_Result: _outResult(str, obj); break; + case T_ProjectSet: + _outProjectSet(str, obj); + break; case T_ModifyTable: _outModifyTable(str, obj); break; @@ -3679,6 +3700,9 @@ outNode(StringInfo str, const void *obj) case T_ProjectionPath: _outProjectionPath(str, obj); break; + case T_ProjectSetPath: + _outProjectSetPath(str, obj); + break; case T_SortPath: _outSortPath(str, obj); break; diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index e02dd94f05..dcfa6ee28d 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1483,6 +1483,19 @@ _readResult(void) READ_DONE(); } +/* + * _readProjectSet + */ +static ProjectSet * +_readProjectSet(void) +{ + READ_LOCALS_NO_FIELDS(ProjectSet); + + ReadCommonPlan(&local_node->plan); + + READ_DONE(); +} + /* * _readModifyTable */ @@ -2450,6 +2463,8 @@ parseNodeString(void) return_value = _readPlan(); else if (MATCH("RESULT", 6)) return_value = _readResult(); + else if (MATCH("PROJECTSET", 10)) + return_value = _readProjectSet(); else if (MATCH("MODIFYTABLE", 11)) return_value = _readModifyTable(); else if (MATCH("APPEND", 6)) diff --git a/src/backend/optimizer/README b/src/backend/optimizer/README index 1998739702..7ae2b74b2c 100644 --- a/src/backend/optimizer/README +++ b/src/backend/optimizer/README @@ -375,6 +375,7 @@ RelOptInfo - a relation or joined relations UniquePath - remove duplicate rows (either by hashing or sorting) GatherPath - collect the results of parallel workers ProjectionPath - a Result plan node with child (used for projection) + ProjectSetPath - a ProjectSet plan node applied to some sub-path SortPath - a Sort plan node applied to some sub-path GroupPath - a Group plan node applied to some sub-path UpperUniquePath - a Unique plan node applied to some sub-path diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 96ca7d3bb0..7c017fe1e4 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3051,6 +3051,10 @@ print_path(PlannerInfo *root, Path *path, int indent) ptype = "Projection"; subpath = ((ProjectionPath *) path)->subpath; break; + case T_ProjectSetPath: + ptype = "ProjectSet"; + subpath = ((ProjectSetPath *) path)->subpath; + break; case T_SortPath: ptype = "Sort"; subpath = ((SortPath *) path)->subpath; diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index c4ada214ed..fae1f67b9c 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -81,6 +81,7 @@ static Plan *create_join_plan(PlannerInfo *root, JoinPath *best_path); static Plan *create_append_plan(PlannerInfo *root, AppendPath *best_path); static Plan *create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path); static Result *create_result_plan(PlannerInfo *root, ResultPath *best_path); +static ProjectSet *create_project_set_plan(PlannerInfo *root, ProjectSetPath *best_path); static Material *create_material_plan(PlannerInfo *root, MaterialPath *best_path, int flags); static Plan *create_unique_plan(PlannerInfo *root, UniquePath *best_path, @@ -264,6 +265,7 @@ static SetOp *make_setop(SetOpCmd cmd, SetOpStrategy strategy, Plan *lefttree, long numGroups); static LockRows *make_lockrows(Plan *lefttree, List *rowMarks, int epqParam); static Result *make_result(List *tlist, Node *resconstantqual, Plan *subplan); +static ProjectSet *make_project_set(List *tlist, Plan *subplan); static ModifyTable *make_modifytable(PlannerInfo *root, CmdType operation, bool canSetTag, Index nominalRelation, @@ -392,6 +394,10 @@ create_plan_recurse(PlannerInfo *root, Path *best_path, int flags) (ResultPath *) best_path); } break; + case T_ProjectSet: + plan = (Plan *) create_project_set_plan(root, + (ProjectSetPath *) best_path); + break; case T_Material: plan = (Plan *) create_material_plan(root, (MaterialPath *) best_path, @@ -1141,6 +1147,31 @@ create_result_plan(PlannerInfo *root, ResultPath *best_path) return plan; } +/* + * create_project_set_plan + * Create a ProjectSet plan for 'best_path'. + * + * Returns a Plan node. + */ +static ProjectSet * +create_project_set_plan(PlannerInfo *root, ProjectSetPath *best_path) +{ + ProjectSet *plan; + Plan *subplan; + List *tlist; + + /* Since we intend to project, we don't need to constrain child tlist */ + subplan = create_plan_recurse(root, best_path->subpath, 0); + + tlist = build_path_tlist(root, &best_path->path); + + plan = make_project_set(tlist, subplan); + + copy_generic_path_info(&plan->plan, (Path *) best_path); + + return plan; +} + /* * create_material_plan * Create a Material plan for 'best_path' and (recursively) plans @@ -6063,6 +6094,25 @@ make_result(List *tlist, return node; } +/* + * make_project_set + * Build a ProjectSet plan node + */ +static ProjectSet * +make_project_set(List *tlist, + Plan *subplan) +{ + ProjectSet *node = makeNode(ProjectSet); + Plan *plan = &node->plan; + + plan->targetlist = tlist; + plan->qual = NIL; + plan->lefttree = subplan; + plan->righttree = NULL; + + return node; +} + /* * make_modifytable * Build a ModifyTable plan node @@ -6229,6 +6279,15 @@ is_projection_capable_path(Path *path) * projection to its dummy path. */ return IS_DUMMY_PATH(path); + case T_ProjectSet: + + /* + * Although ProjectSet certainly projects, say "no" because we + * don't want the planner to randomly replace its tlist with + * something else; the SRFs have to stay at top level. This might + * get relaxed later. + */ + return false; default: break; } @@ -6257,6 +6316,15 @@ is_projection_capable_plan(Plan *plan) case T_MergeAppend: case T_RecursiveUnion: return false; + case T_ProjectSet: + + /* + * Although ProjectSet certainly projects, say "no" because we + * don't want the planner to randomly replace its tlist with + * something else; the SRFs have to stay at top level. This might + * get relaxed later. + */ + return false; default: break; } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 25f2c5a614..4b5902fc3e 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -153,6 +153,8 @@ static List *make_pathkeys_for_window(PlannerInfo *root, WindowClause *wc, static PathTarget *make_sort_input_target(PlannerInfo *root, PathTarget *final_target, bool *have_postponed_srfs); +static void adjust_paths_for_srfs(PlannerInfo *root, RelOptInfo *rel, + List *targets, List *targets_contain_srfs); /***************************************************************************** @@ -1400,8 +1402,9 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, int64 count_est = 0; double limit_tuples = -1.0; bool have_postponed_srfs = false; - double tlist_rows; PathTarget *final_target; + List *final_targets; + List *final_targets_contain_srfs; RelOptInfo *current_rel; RelOptInfo *final_rel; ListCell *lc; @@ -1464,6 +1467,10 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, /* Also extract the PathTarget form of the setop result tlist */ final_target = current_rel->cheapest_total_path->pathtarget; + /* The setop result tlist couldn't contain any SRFs */ + Assert(!parse->hasTargetSRFs); + final_targets = final_targets_contain_srfs = NIL; + /* * Can't handle FOR [KEY] UPDATE/SHARE here (parser should have * checked already, but let's make sure). @@ -1489,8 +1496,14 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, { /* No set operations, do regular planning */ PathTarget *sort_input_target; + List *sort_input_targets; + List *sort_input_targets_contain_srfs; PathTarget *grouping_target; + List *grouping_targets; + List *grouping_targets_contain_srfs; PathTarget *scanjoin_target; + List *scanjoin_targets; + List *scanjoin_targets_contain_srfs; bool have_grouping; AggClauseCosts agg_costs; WindowFuncLists *wflists = NULL; @@ -1735,8 +1748,50 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, scanjoin_target = grouping_target; /* - * Forcibly apply scan/join target to all the Paths for the scan/join - * rel. + * If there are any SRFs in the targetlist, we must separate each of + * these PathTargets into SRF-computing and SRF-free targets. Replace + * each of the named targets with a SRF-free version, and remember the + * list of additional projection steps we need to add afterwards. + */ + if (parse->hasTargetSRFs) + { + /* final_target doesn't recompute any SRFs in sort_input_target */ + split_pathtarget_at_srfs(root, final_target, sort_input_target, + &final_targets, + &final_targets_contain_srfs); + final_target = (PathTarget *) linitial(final_targets); + Assert(!linitial_int(final_targets_contain_srfs)); + /* likewise for sort_input_target vs. grouping_target */ + split_pathtarget_at_srfs(root, sort_input_target, grouping_target, + &sort_input_targets, + &sort_input_targets_contain_srfs); + sort_input_target = (PathTarget *) linitial(sort_input_targets); + Assert(!linitial_int(sort_input_targets_contain_srfs)); + /* likewise for grouping_target vs. scanjoin_target */ + split_pathtarget_at_srfs(root, grouping_target, scanjoin_target, + &grouping_targets, + &grouping_targets_contain_srfs); + grouping_target = (PathTarget *) linitial(grouping_targets); + Assert(!linitial_int(grouping_targets_contain_srfs)); + /* scanjoin_target will not have any SRFs precomputed for it */ + split_pathtarget_at_srfs(root, scanjoin_target, NULL, + &scanjoin_targets, + &scanjoin_targets_contain_srfs); + scanjoin_target = (PathTarget *) linitial(scanjoin_targets); + Assert(!linitial_int(scanjoin_targets_contain_srfs)); + } + else + { + /* initialize lists, just to keep compiler quiet */ + final_targets = final_targets_contain_srfs = NIL; + sort_input_targets = sort_input_targets_contain_srfs = NIL; + grouping_targets = grouping_targets_contain_srfs = NIL; + scanjoin_targets = scanjoin_targets_contain_srfs = NIL; + } + + /* + * Forcibly apply SRF-free scan/join target to all the Paths for the + * scan/join rel. * * In principle we should re-run set_cheapest() here to identify the * cheapest path, but it seems unlikely that adding the same tlist @@ -1807,6 +1862,12 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, current_rel->partial_pathlist = NIL; } + /* Now fix things up if scan/join target contains SRFs */ + if (parse->hasTargetSRFs) + adjust_paths_for_srfs(root, current_rel, + scanjoin_targets, + scanjoin_targets_contain_srfs); + /* * Save the various upper-rel PathTargets we just computed into * root->upper_targets[]. The core code doesn't use this, but it @@ -1831,6 +1892,11 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, &agg_costs, rollup_lists, rollup_groupclauses); + /* Fix things up if grouping_target contains SRFs */ + if (parse->hasTargetSRFs) + adjust_paths_for_srfs(root, current_rel, + grouping_targets, + grouping_targets_contain_srfs); } /* @@ -1846,6 +1912,11 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, tlist, wflists, activeWindows); + /* Fix things up if sort_input_target contains SRFs */ + if (parse->hasTargetSRFs) + adjust_paths_for_srfs(root, current_rel, + sort_input_targets, + sort_input_targets_contain_srfs); } /* @@ -1874,40 +1945,11 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, final_target, have_postponed_srfs ? -1.0 : limit_tuples); - } - - /* - * If there are set-returning functions in the tlist, scale up the output - * rowcounts of all surviving Paths to account for that. Note that if any - * SRFs appear in sorting or grouping columns, we'll have underestimated - * the numbers of rows passing through earlier steps; but that's such a - * weird usage that it doesn't seem worth greatly complicating matters to - * account for it. - */ - if (parse->hasTargetSRFs) - tlist_rows = tlist_returns_set_rows(tlist); - else - tlist_rows = 1; - - if (tlist_rows > 1) - { - foreach(lc, current_rel->pathlist) - { - Path *path = (Path *) lfirst(lc); - - /* - * We assume that execution costs of the tlist as such were - * already accounted for. However, it still seems appropriate to - * charge something more for the executor's general costs of - * processing the added tuples. The cost is probably less than - * cpu_tuple_cost, though, so we arbitrarily use half of that. - */ - path->total_cost += path->rows * (tlist_rows - 1) * - cpu_tuple_cost / 2; - - path->rows *= tlist_rows; - } - /* No need to run set_cheapest; we're keeping all paths anyway. */ + /* Fix things up if final_target contains SRFs */ + if (parse->hasTargetSRFs) + adjust_paths_for_srfs(root, current_rel, + final_targets, + final_targets_contain_srfs); } /* @@ -5101,6 +5143,109 @@ get_cheapest_fractional_path(RelOptInfo *rel, double tuple_fraction) return best_path; } +/* + * adjust_paths_for_srfs + * Fix up the Paths of the given upperrel to handle tSRFs properly. + * + * The executor can only handle set-returning functions that appear at the + * top level of the targetlist of a ProjectSet plan node. If we have any SRFs + * that are not at top level, we need to split up the evaluation into multiple + * plan levels in which each level satisfies this constraint. This function + * modifies each Path of an upperrel that (might) compute any SRFs in its + * output tlist to insert appropriate projection steps. + * + * The given targets and targets_contain_srfs lists are from + * split_pathtarget_at_srfs(). We assume the existing Paths emit the first + * target in targets. + */ +static void +adjust_paths_for_srfs(PlannerInfo *root, RelOptInfo *rel, + List *targets, List *targets_contain_srfs) +{ + ListCell *lc; + + Assert(list_length(targets) == list_length(targets_contain_srfs)); + Assert(!linitial_int(targets_contain_srfs)); + + /* If no SRFs appear at this plan level, nothing to do */ + if (list_length(targets) == 1) + return; + + /* + * Stack SRF-evaluation nodes atop each path for the rel. + * + * In principle we should re-run set_cheapest() here to identify the + * cheapest path, but it seems unlikely that adding the same tlist eval + * costs to all the paths would change that, so we don't bother. Instead, + * just assume that the cheapest-startup and cheapest-total paths remain + * so. (There should be no parameterized paths anymore, so we needn't + * worry about updating cheapest_parameterized_paths.) + */ + foreach(lc, rel->pathlist) + { + Path *subpath = (Path *) lfirst(lc); + Path *newpath = subpath; + ListCell *lc1, + *lc2; + + Assert(subpath->param_info == NULL); + forboth(lc1, targets, lc2, targets_contain_srfs) + { + PathTarget *thistarget = (PathTarget *) lfirst(lc1); + bool contains_srfs = (bool) lfirst_int(lc2); + + /* If this level doesn't contain SRFs, do regular projection */ + if (contains_srfs) + newpath = (Path *) create_set_projection_path(root, + rel, + newpath, + thistarget); + else + newpath = (Path *) apply_projection_to_path(root, + rel, + newpath, + thistarget); + } + lfirst(lc) = newpath; + if (subpath == rel->cheapest_startup_path) + rel->cheapest_startup_path = newpath; + if (subpath == rel->cheapest_total_path) + rel->cheapest_total_path = newpath; + } + + /* Likewise for partial paths, if any */ + foreach(lc, rel->partial_pathlist) + { + Path *subpath = (Path *) lfirst(lc); + Path *newpath = subpath; + ListCell *lc1, + *lc2; + + Assert(subpath->param_info == NULL); + forboth(lc1, targets, lc2, targets_contain_srfs) + { + PathTarget *thistarget = (PathTarget *) lfirst(lc1); + bool contains_srfs = (bool) lfirst_int(lc2); + + /* If this level doesn't contain SRFs, do regular projection */ + if (contains_srfs) + newpath = (Path *) create_set_projection_path(root, + rel, + newpath, + thistarget); + else + { + /* avoid apply_projection_to_path, in case of multiple refs */ + newpath = (Path *) create_projection_path(root, + rel, + newpath, + thistarget); + } + } + lfirst(lc) = newpath; + } +} + /* * expression_planner * Perform planner's transformations on a standalone expression. diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 413a0d9da2..be267b9da7 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -733,6 +733,9 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) fix_scan_expr(root, splan->resconstantqual, rtoffset); } break; + case T_ProjectSet: + set_upper_references(root, plan, rtoffset); + break; case T_ModifyTable: { ModifyTable *splan = (ModifyTable *) plan; diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index aad0b684ed..9fc748973e 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2680,6 +2680,7 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, &context); break; + case T_ProjectSet: case T_Hash: case T_Material: case T_Sort: @@ -2687,6 +2688,7 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, case T_Gather: case T_SetOp: case T_Group: + /* no node-type-specific fields need fixing */ break; default: diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 9e122e383d..85ffa3afc7 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -99,7 +99,6 @@ static bool contain_agg_clause_walker(Node *node, void *context); static bool get_agg_clause_costs_walker(Node *node, get_agg_clause_costs_context *context); static bool find_window_functions_walker(Node *node, WindowFuncLists *lists); -static bool expression_returns_set_rows_walker(Node *node, double *count); static bool contain_subplans_walker(Node *node, void *context); static bool contain_mutable_functions_walker(Node *node, void *context); static bool contain_volatile_functions_walker(Node *node, void *context); @@ -790,114 +789,37 @@ find_window_functions_walker(Node *node, WindowFuncLists *lists) /* * expression_returns_set_rows * Estimate the number of rows returned by a set-returning expression. - * The result is 1 if there are no set-returning functions. + * The result is 1 if it's not a set-returning expression. * - * We use the product of the rowcount estimates of all the functions in - * the given tree (this corresponds to the behavior of ExecMakeFunctionResult - * for nested set-returning functions). + * We should only examine the top-level function or operator; it used to be + * appropriate to recurse, but not anymore. (Even if there are more SRFs in + * the function's inputs, their multipliers are accounted for separately.) * * Note: keep this in sync with expression_returns_set() in nodes/nodeFuncs.c. */ double expression_returns_set_rows(Node *clause) { - double result = 1; - - (void) expression_returns_set_rows_walker(clause, &result); - return clamp_row_est(result); -} - -static bool -expression_returns_set_rows_walker(Node *node, double *count) -{ - if (node == NULL) - return false; - if (IsA(node, FuncExpr)) + if (clause == NULL) + return 1.0; + if (IsA(clause, FuncExpr)) { - FuncExpr *expr = (FuncExpr *) node; + FuncExpr *expr = (FuncExpr *) clause; if (expr->funcretset) - *count *= get_func_rows(expr->funcid); + return clamp_row_est(get_func_rows(expr->funcid)); } - if (IsA(node, OpExpr)) + if (IsA(clause, OpExpr)) { - OpExpr *expr = (OpExpr *) node; + OpExpr *expr = (OpExpr *) clause; if (expr->opretset) { set_opfuncid(expr); - *count *= get_func_rows(expr->opfuncid); + return clamp_row_est(get_func_rows(expr->opfuncid)); } } - - /* Avoid recursion for some cases that can't return a set */ - if (IsA(node, Aggref)) - return false; - if (IsA(node, WindowFunc)) - return false; - if (IsA(node, DistinctExpr)) - return false; - if (IsA(node, NullIfExpr)) - return false; - if (IsA(node, ScalarArrayOpExpr)) - return false; - if (IsA(node, BoolExpr)) - return false; - if (IsA(node, SubLink)) - return false; - if (IsA(node, SubPlan)) - return false; - if (IsA(node, AlternativeSubPlan)) - return false; - if (IsA(node, ArrayExpr)) - return false; - if (IsA(node, RowExpr)) - return false; - if (IsA(node, RowCompareExpr)) - return false; - if (IsA(node, CoalesceExpr)) - return false; - if (IsA(node, MinMaxExpr)) - return false; - if (IsA(node, XmlExpr)) - return false; - - return expression_tree_walker(node, expression_returns_set_rows_walker, - (void *) count); -} - -/* - * tlist_returns_set_rows - * Estimate the number of rows returned by a set-returning targetlist. - * The result is 1 if there are no set-returning functions. - * - * Here, the result is the largest rowcount estimate of any of the tlist's - * expressions, not the product as you would get from naively applying - * expression_returns_set_rows() to the whole tlist. The behavior actually - * implemented by ExecTargetList produces a number of rows equal to the least - * common multiple of the expression rowcounts, so that the product would be - * a worst-case estimate that is typically not realistic. Taking the max as - * we do here is a best-case estimate that might not be realistic either, - * but it's probably closer for typical usages. We don't try to compute the - * actual LCM because we're working with very approximate estimates, so their - * LCM would be unduly noisy. - */ -double -tlist_returns_set_rows(List *tlist) -{ - double result = 1; - ListCell *lc; - - foreach(lc, tlist) - { - TargetEntry *tle = (TargetEntry *) lfirst(lc); - double colresult; - - colresult = expression_returns_set_rows((Node *) tle->expr); - if (result < colresult) - result = colresult; - } - return result; + return 1.0; } diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 3b7c56d3c7..f440875ceb 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -2319,6 +2319,72 @@ apply_projection_to_path(PlannerInfo *root, return path; } +/* + * create_set_projection_path + * Creates a pathnode that represents performing a projection that + * includes set-returning functions. + * + * 'rel' is the parent relation associated with the result + * 'subpath' is the path representing the source of data + * 'target' is the PathTarget to be computed + */ +ProjectSetPath * +create_set_projection_path(PlannerInfo *root, + RelOptInfo *rel, + Path *subpath, + PathTarget *target) +{ + ProjectSetPath *pathnode = makeNode(ProjectSetPath); + double tlist_rows; + ListCell *lc; + + pathnode->path.pathtype = T_ProjectSet; + pathnode->path.parent = rel; + pathnode->path.pathtarget = target; + /* For now, assume we are above any joins, so no parameterization */ + pathnode->path.param_info = NULL; + pathnode->path.parallel_aware = false; + pathnode->path.parallel_safe = rel->consider_parallel && + subpath->parallel_safe && + is_parallel_safe(root, (Node *) target->exprs); + pathnode->path.parallel_workers = subpath->parallel_workers; + /* Projection does not change the sort order XXX? */ + pathnode->path.pathkeys = subpath->pathkeys; + + pathnode->subpath = subpath; + + /* + * Estimate number of rows produced by SRFs for each row of input; if + * there's more than one in this node, use the maximum. + */ + tlist_rows = 1; + foreach(lc, target->exprs) + { + Node *node = (Node *) lfirst(lc); + double itemrows; + + itemrows = expression_returns_set_rows(node); + if (tlist_rows < itemrows) + tlist_rows = itemrows; + } + + /* + * In addition to the cost of evaluating the tlist, charge cpu_tuple_cost + * per input row, and half of cpu_tuple_cost for each added output row. + * This is slightly bizarre maybe, but it's what 9.6 did; we may revisit + * this estimate later. + */ + pathnode->path.rows = subpath->rows * tlist_rows; + pathnode->path.startup_cost = subpath->startup_cost + + target->cost.startup; + pathnode->path.total_cost = subpath->total_cost + + target->cost.startup + + (cpu_tuple_cost + target->cost.per_tuple) * subpath->rows + + (pathnode->path.rows - subpath->rows) * cpu_tuple_cost / 2; + + return pathnode; +} + /* * create_sort_path * Creates a pathnode that represents performing an explicit sort. diff --git a/src/backend/optimizer/util/tlist.c b/src/backend/optimizer/util/tlist.c index 45205a830f..cca5db88e2 100644 --- a/src/backend/optimizer/util/tlist.c +++ b/src/backend/optimizer/util/tlist.c @@ -16,9 +16,20 @@ #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#include "optimizer/cost.h" #include "optimizer/tlist.h" +typedef struct +{ + List *nextlevel_tlist; + bool nextlevel_contains_srfs; +} split_pathtarget_context; + +static bool split_pathtarget_walker(Node *node, + split_pathtarget_context *context); + + /***************************************************************************** * Target list creation and searching utilities *****************************************************************************/ @@ -759,3 +770,191 @@ apply_pathtarget_labeling_to_tlist(List *tlist, PathTarget *target) i++; } } + +/* + * split_pathtarget_at_srfs + * Split given PathTarget into multiple levels to position SRFs safely + * + * The executor can only handle set-returning functions that appear at the + * top level of the targetlist of a ProjectSet plan node. If we have any SRFs + * that are not at top level, we need to split up the evaluation into multiple + * plan levels in which each level satisfies this constraint. This function + * creates appropriate PathTarget(s) for each level. + * + * As an example, consider the tlist expression + * x + srf1(srf2(y + z)) + * This expression should appear as-is in the top PathTarget, but below that + * we must have a PathTarget containing + * x, srf1(srf2(y + z)) + * and below that, another PathTarget containing + * x, srf2(y + z) + * and below that, another PathTarget containing + * x, y, z + * When these tlists are processed by setrefs.c, subexpressions that match + * output expressions of the next lower tlist will be replaced by Vars, + * so that what the executor gets are tlists looking like + * Var1 + Var2 + * Var1, srf1(Var2) + * Var1, srf2(Var2 + Var3) + * x, y, z + * which satisfy the desired property. + * + * In some cases, a SRF has already been evaluated in some previous plan level + * and we shouldn't expand it again (that is, what we see in the target is + * already meant as a reference to a lower subexpression). So, don't expand + * any tlist expressions that appear in input_target, if that's not NULL. + * In principle we might need to consider matching subexpressions to + * input_target, but for now it's not necessary because only ORDER BY and + * GROUP BY expressions are at issue and those will look the same at both + * plan levels. + * + * The outputs of this function are two parallel lists, one a list of + * PathTargets and the other an integer list of bool flags indicating + * whether the corresponding PathTarget contains any top-level SRFs. + * The lists are given in the order they'd need to be evaluated in, with + * the "lowest" PathTarget first. So the last list entry is always the + * originally given PathTarget, and any entries before it indicate evaluation + * levels that must be inserted below it. The first list entry must not + * contain any SRFs, since it will typically be attached to a plan node + * that cannot evaluate SRFs. + * + * Note: using a list for the flags may seem like overkill, since there + * are only a few possible patterns for which levels contain SRFs. + * But this representation decouples callers from that knowledge. + */ +void +split_pathtarget_at_srfs(PlannerInfo *root, + PathTarget *target, PathTarget *input_target, + List **targets, List **targets_contain_srfs) +{ + /* Initialize output lists to empty; we prepend to them within loop */ + *targets = *targets_contain_srfs = NIL; + + /* Loop to consider each level of PathTarget we need */ + for (;;) + { + bool target_contains_srfs = false; + split_pathtarget_context context; + ListCell *lc; + + context.nextlevel_tlist = NIL; + context.nextlevel_contains_srfs = false; + + /* + * Scan the PathTarget looking for SRFs. Top-level SRFs are handled + * in this loop, ones lower down are found by split_pathtarget_walker. + */ + foreach(lc, target->exprs) + { + Node *node = (Node *) lfirst(lc); + + /* + * A tlist item that is just a reference to an expression already + * computed in input_target need not be evaluated here, so just + * make sure it's included in the next PathTarget. + */ + if (input_target && list_member(input_target->exprs, node)) + { + context.nextlevel_tlist = lappend(context.nextlevel_tlist, node); + continue; + } + + /* Else, we need to compute this expression. */ + if (IsA(node, FuncExpr) && + ((FuncExpr *) node)->funcretset) + { + /* Top-level SRF: it can be evaluated here */ + target_contains_srfs = true; + /* Recursively examine SRF's inputs */ + split_pathtarget_walker((Node *) ((FuncExpr *) node)->args, + &context); + } + else if (IsA(node, OpExpr) && + ((OpExpr *) node)->opretset) + { + /* Same as above, but for set-returning operator */ + target_contains_srfs = true; + split_pathtarget_walker((Node *) ((OpExpr *) node)->args, + &context); + } + else + { + /* Not a top-level SRF, so recursively examine expression */ + split_pathtarget_walker(node, &context); + } + } + + /* + * Prepend current target and associated flag to output lists. + */ + *targets = lcons(target, *targets); + *targets_contain_srfs = lcons_int(target_contains_srfs, + *targets_contain_srfs); + + /* + * Done if we found no SRFs anywhere in this target; the tentative + * tlist we built for the next level can be discarded. + */ + if (!target_contains_srfs && !context.nextlevel_contains_srfs) + break; + + /* + * Else build the next PathTarget down, and loop back to process it. + * Copy the subexpressions to make sure PathTargets don't share + * substructure (might be unnecessary, but be safe); and drop any + * duplicate entries in the sub-targetlist. + */ + target = create_empty_pathtarget(); + add_new_columns_to_pathtarget(target, + (List *) copyObject(context.nextlevel_tlist)); + set_pathtarget_cost_width(root, target); + } +} + +/* Recursively examine expressions for split_pathtarget_at_srfs */ +static bool +split_pathtarget_walker(Node *node, split_pathtarget_context *context) +{ + if (node == NULL) + return false; + if (IsA(node, Var) || + IsA(node, PlaceHolderVar) || + IsA(node, Aggref) || + IsA(node, GroupingFunc) || + IsA(node, WindowFunc)) + { + /* + * Pass these items down to the child plan level for evaluation. + * + * We assume that these constructs cannot contain any SRFs (if one + * does, there will be an executor failure from a misplaced SRF). + */ + context->nextlevel_tlist = lappend(context->nextlevel_tlist, node); + + /* Having done that, we need not examine their sub-structure */ + return false; + } + else if ((IsA(node, FuncExpr) && + ((FuncExpr *) node)->funcretset) || + (IsA(node, OpExpr) && + ((OpExpr *) node)->opretset)) + { + /* + * Pass SRFs down to the child plan level for evaluation, and mark + * that it contains SRFs. (We are not at top level of our own tlist, + * else this would have been picked up by split_pathtarget_at_srfs.) + */ + context->nextlevel_tlist = lappend(context->nextlevel_tlist, node); + context->nextlevel_contains_srfs = true; + + /* Inputs to the SRF need not be considered here, so we're done */ + return false; + } + + /* + * Otherwise, the node is evaluatable within the current PathTarget, so + * recurse to examine its inputs. + */ + return expression_tree_walker(node, split_pathtarget_walker, + (void *) context); +} diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index b9c7f72903..d424031676 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -253,6 +253,10 @@ extern Tuplestorestate *ExecMakeTableFunctionResult(ExprState *funcexpr, MemoryContext argContext, TupleDesc expectedDesc, bool randomAccess); +extern Datum ExecMakeFunctionResultSet(FuncExprState *fcache, + ExprContext *econtext, + bool *isNull, + ExprDoneCond *isDone); extern Datum ExecEvalExprSwitchContext(ExprState *expression, ExprContext *econtext, bool *isNull, ExprDoneCond *isDone); extern ExprState *ExecInitExpr(Expr *node, PlanState *parent); diff --git a/src/include/executor/nodeProjectSet.h b/src/include/executor/nodeProjectSet.h new file mode 100644 index 0000000000..30b2b7cec9 --- /dev/null +++ b/src/include/executor/nodeProjectSet.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * nodeProjectSet.h + * + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/executor/nodeProjectSet.h + * + *------------------------------------------------------------------------- + */ +#ifndef NODEPROJECTSET_H +#define NODEPROJECTSET_H + +#include "nodes/execnodes.h" + +extern ProjectSetState *ExecInitProjectSet(ProjectSet *node, EState *estate, int eflags); +extern TupleTableSlot *ExecProjectSet(ProjectSetState *node); +extern void ExecEndProjectSet(ProjectSetState *node); +extern void ExecReScanProjectSet(ProjectSetState *node); + +#endif /* NODEPROJECTSET_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index ce13bf7635..1da1e1f804 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -696,7 +696,7 @@ typedef struct FuncExprState /* * Function manager's lookup info for the target function. If func.fn_oid * is InvalidOid, we haven't initialized it yet (nor any of the following - * fields). + * fields, except funcReturnsSet). */ FmgrInfo func; @@ -716,6 +716,12 @@ typedef struct FuncExprState bool funcReturnsTuple; /* valid when funcResultDesc isn't * NULL */ + /* + * Remember whether the function is declared to return a set. This is set + * by ExecInitExpr, and is valid even before the FmgrInfo is set up. + */ + bool funcReturnsSet; + /* * setArgsValid is true when we are evaluating a set-returning function * that uses value-per-call mode and we are in the middle of a call @@ -1129,6 +1135,18 @@ typedef struct ResultState bool rs_checkqual; /* do we need to check the qual? */ } ResultState; +/* ---------------- + * ProjectSetState information + * ---------------- + */ +typedef struct ProjectSetState +{ + PlanState ps; /* its first field is NodeTag */ + ExprDoneCond *elemdone; /* array of per-SRF is-done states */ + int nelems; /* length of elemdone[] array */ + bool pending_srf_tuples; /* still evaluating srfs in tlist? */ +} ProjectSetState; + /* ---------------- * ModifyTableState information * ---------------- diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 4c4319bcab..d65958153d 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -43,6 +43,7 @@ typedef enum NodeTag */ T_Plan, T_Result, + T_ProjectSet, T_ModifyTable, T_Append, T_MergeAppend, @@ -91,6 +92,7 @@ typedef enum NodeTag */ T_PlanState, T_ResultState, + T_ProjectSetState, T_ModifyTableState, T_AppendState, T_MergeAppendState, @@ -245,6 +247,7 @@ typedef enum NodeTag T_UniquePath, T_GatherPath, T_ProjectionPath, + T_ProjectSetPath, T_SortPath, T_GroupPath, T_UpperUniquePath, diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 6810f8c099..f72f7a8978 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -176,6 +176,17 @@ typedef struct Result Node *resconstantqual; } Result; +/* ---------------- + * ProjectSet node - + * Apply a projection that includes set-returning functions to the + * output tuples of the outer plan. + * ---------------- + */ +typedef struct ProjectSet +{ + Plan plan; +} ProjectSet; + /* ---------------- * ModifyTable node - * Apply rows produced by subplan(s) to result table(s), diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 1e950c4afd..643be54d40 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -1304,6 +1304,17 @@ typedef struct ProjectionPath bool dummypp; /* true if no separate Result is needed */ } ProjectionPath; +/* + * ProjectSetPath represents evaluation of a targetlist that includes + * set-returning function(s), which will need to be implemented by a + * ProjectSet plan node. + */ +typedef struct ProjectSetPath +{ + Path path; + Path *subpath; /* path representing input source */ +} ProjectSetPath; + /* * SortPath represents an explicit sort step * diff --git a/src/include/optimizer/clauses.h b/src/include/optimizer/clauses.h index 6173ef8d75..cc0d7b0a26 100644 --- a/src/include/optimizer/clauses.h +++ b/src/include/optimizer/clauses.h @@ -54,7 +54,6 @@ extern bool contain_window_function(Node *clause); extern WindowFuncLists *find_window_functions(Node *clause, Index maxWinRef); extern double expression_returns_set_rows(Node *clause); -extern double tlist_returns_set_rows(List *tlist); extern bool contain_subplans(Node *clause); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index d16f879fc1..7b41317621 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -144,6 +144,10 @@ extern Path *apply_projection_to_path(PlannerInfo *root, RelOptInfo *rel, Path *path, PathTarget *target); +extern ProjectSetPath *create_set_projection_path(PlannerInfo *root, + RelOptInfo *rel, + Path *subpath, + PathTarget *target); extern SortPath *create_sort_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, diff --git a/src/include/optimizer/tlist.h b/src/include/optimizer/tlist.h index f80b31a673..976024a164 100644 --- a/src/include/optimizer/tlist.h +++ b/src/include/optimizer/tlist.h @@ -61,6 +61,9 @@ extern void add_column_to_pathtarget(PathTarget *target, extern void add_new_column_to_pathtarget(PathTarget *target, Expr *expr); extern void add_new_columns_to_pathtarget(PathTarget *target, List *exprs); extern void apply_pathtarget_labeling_to_tlist(List *tlist, PathTarget *target); +extern void split_pathtarget_at_srfs(PlannerInfo *root, + PathTarget *target, PathTarget *input_target, + List **targets, List **targets_contain_srfs); /* Convenience macro to get a PathTarget with valid cost/width fields */ #define create_pathtarget(root, tlist) \ diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index fa1f5e7879..0ff80620cc 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -822,8 +822,9 @@ explain (costs off) -> Limit -> Index Only Scan Backward using tenk1_unique2 on tenk1 Index Cond: (unique2 IS NOT NULL) - -> Result -(7 rows) + -> ProjectSet + -> Result +(8 rows) select max(unique2), generate_series(1,3) as g from tenk1 order by g desc; max | g diff --git a/src/test/regress/expected/limit.out b/src/test/regress/expected/limit.out index 9c3eecfc3b..65c8c44a9a 100644 --- a/src/test/regress/expected/limit.out +++ b/src/test/regress/expected/limit.out @@ -208,13 +208,15 @@ select currval('testseq'); explain (verbose, costs off) select unique1, unique2, generate_series(1,10) from tenk1 order by unique2 limit 7; - QUERY PLAN ----------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit Output: unique1, unique2, (generate_series(1, 10)) - -> Index Scan using tenk1_unique2 on public.tenk1 + -> ProjectSet Output: unique1, unique2, generate_series(1, 10) -(4 rows) + -> Index Scan using tenk1_unique2 on public.tenk1 + Output: unique1, unique2, two, four, ten, twenty, hundred, thousand, twothousand, fivethous, tenthous, odd, even, stringu1, stringu2, string4 +(6 rows) select unique1, unique2, generate_series(1,10) from tenk1 order by unique2 limit 7; @@ -236,7 +238,7 @@ select unique1, unique2, generate_series(1,10) -------------------------------------------------------------------- Limit Output: unique1, unique2, (generate_series(1, 10)), tenthous - -> Result + -> ProjectSet Output: unique1, unique2, generate_series(1, 10), tenthous -> Sort Output: unique1, unique2, tenthous @@ -263,9 +265,10 @@ explain (verbose, costs off) select generate_series(0,2) as s1, generate_series((random()*.1)::int,2) as s2; QUERY PLAN ------------------------------------------------------------------------------------------------------ - Result + ProjectSet Output: generate_series(0, 2), generate_series(((random() * '0.1'::double precision))::integer, 2) -(2 rows) + -> Result +(3 rows) select generate_series(0,2) as s1, generate_series((random()*.1)::int,2) as s2; s1 | s2 @@ -283,9 +286,10 @@ order by s2 desc; Sort Output: (generate_series(0, 2)), (generate_series(((random() * '0.1'::double precision))::integer, 2)) Sort Key: (generate_series(((random() * '0.1'::double precision))::integer, 2)) DESC - -> Result + -> ProjectSet Output: generate_series(0, 2), generate_series(((random() * '0.1'::double precision))::integer, 2) -(5 rows) + -> Result +(6 rows) select generate_series(0,2) as s1, generate_series((random()*.1)::int,2) as s2 order by s2 desc; diff --git a/src/test/regress/expected/portals.out b/src/test/regress/expected/portals.out index 3ae918a63c..1b8f7b69d1 100644 --- a/src/test/regress/expected/portals.out +++ b/src/test/regress/expected/portals.out @@ -1320,18 +1320,20 @@ fetch backward all in c1; rollback; begin; explain (costs off) declare c2 cursor for select generate_series(1,3) as g; - QUERY PLAN ------------- - Result -(1 row) - -explain (costs off) declare c2 scroll cursor for select generate_series(1,3) as g; QUERY PLAN -------------- - Materialize + ProjectSet -> Result (2 rows) +explain (costs off) declare c2 scroll cursor for select generate_series(1,3) as g; + QUERY PLAN +-------------------- + Materialize + -> ProjectSet + -> Result +(3 rows) + declare c2 scroll cursor for select generate_series(1,3) as g; fetch all in c2; g diff --git a/src/test/regress/expected/rangefuncs.out b/src/test/regress/expected/rangefuncs.out index 275b66204a..56481de5c3 100644 --- a/src/test/regress/expected/rangefuncs.out +++ b/src/test/regress/expected/rangefuncs.out @@ -1995,12 +1995,10 @@ SELECT *, END) FROM (VALUES (1,''), (2,'0000000049404'), (3,'FROM 10000000876')) v(id, str); - id | str | lower -----+------------------+------------------ - 1 | | - 2 | 0000000049404 | 49404 - 3 | FROM 10000000876 | from 10000000876 -(3 rows) + id | str | lower +----+---------------+------- + 2 | 0000000049404 | 49404 +(1 row) -- check whole-row-Var handling in nested lateral functions (bug #11703) create function extractq2(t int8_tbl) returns int8 as $$ diff --git a/src/test/regress/expected/subselect.out b/src/test/regress/expected/subselect.out index eda319d24b..abd3217e86 100644 --- a/src/test/regress/expected/subselect.out +++ b/src/test/regress/expected/subselect.out @@ -807,24 +807,28 @@ select * from int4_tbl where explain (verbose, costs off) select * from int4_tbl o where (f1, f1) in (select f1, generate_series(1,2) / 10 g from int4_tbl i group by f1); - QUERY PLAN ----------------------------------------------------------------- - Hash Semi Join + QUERY PLAN +------------------------------------------------------------------- + Nested Loop Semi Join Output: o.f1 - Hash Cond: (o.f1 = "ANY_subquery".f1) + Join Filter: (o.f1 = "ANY_subquery".f1) -> Seq Scan on public.int4_tbl o Output: o.f1 - -> Hash + -> Materialize Output: "ANY_subquery".f1, "ANY_subquery".g -> Subquery Scan on "ANY_subquery" Output: "ANY_subquery".f1, "ANY_subquery".g Filter: ("ANY_subquery".f1 = "ANY_subquery".g) - -> HashAggregate - Output: i.f1, (generate_series(1, 2) / 10) - Group Key: i.f1 - -> Seq Scan on public.int4_tbl i - Output: i.f1 -(15 rows) + -> Result + Output: i.f1, ((generate_series(1, 2)) / 10) + -> ProjectSet + Output: i.f1, generate_series(1, 2) + -> HashAggregate + Output: i.f1 + Group Key: i.f1 + -> Seq Scan on public.int4_tbl i + Output: i.f1 +(19 rows) select * from int4_tbl o where (f1, f1) in (select f1, generate_series(1,2) / 10 g from int4_tbl i group by f1); @@ -899,9 +903,10 @@ select * from Subquery Scan on ss Output: x, u Filter: tattle(ss.x, 8) - -> Result + -> ProjectSet Output: 9, unnest('{1,2,3,11,12,13}'::integer[]) -(5 rows) + -> Result +(6 rows) select * from (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss @@ -930,10 +935,11 @@ select * from where tattle(x, 8); QUERY PLAN ---------------------------------------------------- - Result + ProjectSet Output: 9, unnest('{1,2,3,11,12,13}'::integer[]) - One-Time Filter: tattle(9, 8) -(3 rows) + -> Result + One-Time Filter: tattle(9, 8) +(4 rows) select * from (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss @@ -959,9 +965,10 @@ select * from Subquery Scan on ss Output: x, u Filter: tattle(ss.x, ss.u) - -> Result + -> ProjectSet Output: 9, unnest('{1,2,3,11,12,13}'::integer[]) -(5 rows) + -> Result +(6 rows) select * from (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss diff --git a/src/test/regress/expected/tsrf.out b/src/test/regress/expected/tsrf.out index 7bb6d17fcb..8c47f0f668 100644 --- a/src/test/regress/expected/tsrf.out +++ b/src/test/regress/expected/tsrf.out @@ -25,8 +25,8 @@ SELECT generate_series(1, 2), generate_series(1,4); -----------------+----------------- 1 | 1 2 | 2 - 1 | 3 - 2 | 4 + | 3 + | 4 (4 rows) -- srf, with SRF argument @@ -43,7 +43,16 @@ SELECT generate_series(1, generate_series(1, 3)); -- srf, with two SRF arguments SELECT generate_series(generate_series(1,3), generate_series(2, 4)); -ERROR: functions and operators can take at most one set argument + generate_series +----------------- + 1 + 2 + 2 + 3 + 3 + 4 +(6 rows) + CREATE TABLE few(id int, dataa text, datab text); INSERT INTO few VALUES(1, 'a', 'foo'),(2, 'a', 'bar'),(3, 'b', 'bar'); -- SRF output order of sorting is maintained, if SRF is not referenced @@ -118,15 +127,15 @@ SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa, unnest('{1,1,3}'::int[]); dataa | count | min | max | unnest -------+-------+-----+-----+-------- - a | 2 | 1 | 1 | 1 a | 1 | 1 | 1 | 3 + a | 2 | 1 | 1 | 1 (2 rows) SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa, 5; dataa | count | min | max | unnest -------+-------+-----+-----+-------- - a | 2 | 1 | 1 | 1 a | 1 | 1 | 1 | 3 + a | 2 | 1 | 1 | 1 (2 rows) -- check HAVING works when GROUP BY does [not] reference SRF output diff --git a/src/test/regress/expected/union.out b/src/test/regress/expected/union.out index 67f5fc4361..d22db69c7d 100644 --- a/src/test/regress/expected/union.out +++ b/src/test/regress/expected/union.out @@ -636,9 +636,10 @@ ORDER BY x; -> HashAggregate Group Key: (1), (generate_series(1, 10)) -> Append + -> ProjectSet + -> Result -> Result - -> Result -(9 rows) +(10 rows) SELECT * FROM (SELECT 1 AS t, generate_series(1,10) AS x