Add an at-least-marginally-plausible method of estimating the number
of groups produced by GROUP BY. This improves the accuracy of planning estimates for grouped subselects, and is needed to check whether a hashed aggregation plan risks memory overflow.
This commit is contained in:
Tom Lane
parent
54cb1db6cf
commit
b60be3f2f8
|
|
@ -45,7 +45,7 @@
|
|||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $Header: /cvsroot/pgsql/src/backend/executor/nodeAgg.c,v 1.95 2002/11/13 00:39:47 momjian Exp $
|
||||
* $Header: /cvsroot/pgsql/src/backend/executor/nodeAgg.c,v 1.96 2002/11/19 23:21:57 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -619,6 +619,9 @@ lookup_hash_entry(Agg *node, TupleTableSlot *slot)
|
|||
Datum attr;
|
||||
bool isNull;
|
||||
|
||||
/* rotate hashkey left 1 bit at each step */
|
||||
hashkey = (hashkey << 1) | ((hashkey & 0x80000000) ? 1 : 0);
|
||||
|
||||
attr = heap_getattr(tuple, att, tupdesc, &isNull);
|
||||
if (isNull)
|
||||
continue; /* treat nulls as having hash key 0 */
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $Header: /cvsroot/pgsql/src/backend/nodes/copyfuncs.c,v 1.218 2002/11/15 02:50:06 momjian Exp $
|
||||
* $Header: /cvsroot/pgsql/src/backend/nodes/copyfuncs.c,v 1.219 2002/11/19 23:21:58 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -1865,8 +1865,8 @@ _copyQuery(Query *from)
|
|||
|
||||
/*
|
||||
* We do not copy the planner internal fields: base_rel_list,
|
||||
* other_rel_list, join_rel_list, equi_key_list, query_pathkeys. Not
|
||||
* entirely clear if this is right?
|
||||
* other_rel_list, join_rel_list, equi_key_list, query_pathkeys,
|
||||
* hasJoinRTEs. Not entirely clear if this is right?
|
||||
*/
|
||||
|
||||
return newnode;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $Header: /cvsroot/pgsql/src/backend/nodes/equalfuncs.c,v 1.164 2002/11/15 02:50:06 momjian Exp $
|
||||
* $Header: /cvsroot/pgsql/src/backend/nodes/equalfuncs.c,v 1.165 2002/11/19 23:21:58 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -628,9 +628,9 @@ _equalQuery(Query *a, Query *b)
|
|||
|
||||
/*
|
||||
* We do not check the internal-to-the-planner fields: base_rel_list,
|
||||
* other_rel_list, join_rel_list, equi_key_list, query_pathkeys. They
|
||||
* might not be set yet, and in any case they should be derivable from
|
||||
* the other fields.
|
||||
* other_rel_list, join_rel_list, equi_key_list, query_pathkeys,
|
||||
* hasJoinRTEs. They might not be set yet, and in any case they should
|
||||
* be derivable from the other fields.
|
||||
*/
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
*
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $Header: /cvsroot/pgsql/src/backend/optimizer/plan/createplan.c,v 1.122 2002/11/15 02:36:53 tgl Exp $
|
||||
* $Header: /cvsroot/pgsql/src/backend/optimizer/plan/createplan.c,v 1.123 2002/11/19 23:21:58 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -1684,7 +1684,8 @@ make_material(List *tlist, Plan *lefttree)
|
|||
|
||||
Agg *
|
||||
make_agg(List *tlist, List *qual, AggStrategy aggstrategy,
|
||||
int ngrp, AttrNumber *grpColIdx, Plan *lefttree)
|
||||
int ngrp, AttrNumber *grpColIdx, long numGroups, int numAggs,
|
||||
Plan *lefttree)
|
||||
{
|
||||
Agg *node = makeNode(Agg);
|
||||
Plan *plan = &node->plan;
|
||||
|
|
@ -1692,6 +1693,7 @@ make_agg(List *tlist, List *qual, AggStrategy aggstrategy,
|
|||
node->aggstrategy = aggstrategy;
|
||||
node->numCols = ngrp;
|
||||
node->grpColIdx = grpColIdx;
|
||||
node->numGroups = numGroups;
|
||||
|
||||
copy_plan_costsize(plan, lefttree);
|
||||
|
||||
|
|
@ -1699,15 +1701,11 @@ make_agg(List *tlist, List *qual, AggStrategy aggstrategy,
|
|||
* Charge one cpu_operator_cost per aggregate function per input
|
||||
* tuple.
|
||||
*/
|
||||
plan->total_cost += cpu_operator_cost * plan->plan_rows *
|
||||
(length(pull_agg_clause((Node *) tlist)) +
|
||||
length(pull_agg_clause((Node *) qual)));
|
||||
plan->total_cost += cpu_operator_cost * plan->plan_rows * numAggs;
|
||||
|
||||
/*
|
||||
* We will produce a single output tuple if not grouping,
|
||||
* and a tuple per group otherwise. For now, estimate the number of
|
||||
* groups as 10% of the number of tuples --- bogus, but how to do
|
||||
* better?
|
||||
* and a tuple per group otherwise.
|
||||
*/
|
||||
if (aggstrategy == AGG_PLAIN)
|
||||
{
|
||||
|
|
@ -1716,10 +1714,7 @@ make_agg(List *tlist, List *qual, AggStrategy aggstrategy,
|
|||
}
|
||||
else
|
||||
{
|
||||
plan->plan_rows *= 0.1;
|
||||
if (plan->plan_rows < 1)
|
||||
plan->plan_rows = 1;
|
||||
node->numGroups = (long) plan->plan_rows;
|
||||
plan->plan_rows = numGroups;
|
||||
}
|
||||
|
||||
plan->state = (EState *) NULL;
|
||||
|
|
@ -1735,6 +1730,7 @@ Group *
|
|||
make_group(List *tlist,
|
||||
int ngrp,
|
||||
AttrNumber *grpColIdx,
|
||||
double numGroups,
|
||||
Plan *lefttree)
|
||||
{
|
||||
Group *node = makeNode(Group);
|
||||
|
|
@ -1748,13 +1744,8 @@ make_group(List *tlist,
|
|||
*/
|
||||
plan->total_cost += cpu_operator_cost * plan->plan_rows * ngrp;
|
||||
|
||||
/*
|
||||
* Estimate the number of groups as 10% of the number of tuples
|
||||
* --- bogus, but how to do better?
|
||||
*/
|
||||
plan->plan_rows *= 0.1;
|
||||
if (plan->plan_rows < 1)
|
||||
plan->plan_rows = 1;
|
||||
/* One output tuple per estimated result group */
|
||||
plan->plan_rows = numGroups;
|
||||
|
||||
plan->state = (EState *) NULL;
|
||||
plan->qual = NULL;
|
||||
|
|
@ -1786,17 +1777,16 @@ make_unique(List *tlist, Plan *lefttree, List *distinctList)
|
|||
|
||||
/*
|
||||
* Charge one cpu_operator_cost per comparison per input tuple. We
|
||||
* assume all columns get compared at most of the tuples.
|
||||
* assume all columns get compared at most of the tuples. (XXX probably
|
||||
* this is an overestimate.)
|
||||
*/
|
||||
plan->total_cost += cpu_operator_cost * plan->plan_rows * numCols;
|
||||
|
||||
/*
|
||||
* As for Group, we make the unsupported assumption that there will be
|
||||
* 10% as many tuples out as in.
|
||||
* plan->plan_rows is left as a copy of the input subplan's plan_rows;
|
||||
* ie, we assume the filter removes nothing. The caller must alter this
|
||||
* if he has a better idea.
|
||||
*/
|
||||
plan->plan_rows *= 0.1;
|
||||
if (plan->plan_rows < 1)
|
||||
plan->plan_rows = 1;
|
||||
|
||||
plan->state = (EState *) NULL;
|
||||
plan->targetlist = tlist;
|
||||
|
|
@ -1850,8 +1840,8 @@ make_setop(SetOpCmd cmd, List *tlist, Plan *lefttree,
|
|||
plan->total_cost += cpu_operator_cost * plan->plan_rows * numCols;
|
||||
|
||||
/*
|
||||
* As for Group, we make the unsupported assumption that there will be
|
||||
* 10% as many tuples out as in.
|
||||
* We make the unsupported assumption that there will be 10% as many
|
||||
* tuples out as in. Any way to do better?
|
||||
*/
|
||||
plan->plan_rows *= 0.1;
|
||||
if (plan->plan_rows < 1)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
*
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $Header: /cvsroot/pgsql/src/backend/optimizer/plan/initsplan.c,v 1.75 2002/09/04 20:31:21 momjian Exp $
|
||||
* $Header: /cvsroot/pgsql/src/backend/optimizer/plan/initsplan.c,v 1.76 2002/11/19 23:21:58 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -784,6 +784,71 @@ process_implied_equality(Query *root, Node *item1, Node *item2,
|
|||
pull_varnos((Node *) clause));
|
||||
}
|
||||
|
||||
/*
|
||||
* vars_known_equal
|
||||
* Detect whether two Vars are known equal due to equijoin clauses.
|
||||
*
|
||||
* This is not completely accurate since we avoid adding redundant restriction
|
||||
* clauses to individual base rels (see qual_is_redundant). However, after
|
||||
* the implied-equality-deduction phase, it is complete for Vars of different
|
||||
* rels; that's sufficient for planned uses.
|
||||
*/
|
||||
bool
|
||||
vars_known_equal(Query *root, Var *var1, Var *var2)
|
||||
{
|
||||
Index irel1;
|
||||
Index irel2;
|
||||
RelOptInfo *rel1;
|
||||
List *restrictlist;
|
||||
List *itm;
|
||||
|
||||
/*
|
||||
* Would need more work here if we wanted to check for known equality
|
||||
* of general clauses: there might be multiple base rels involved.
|
||||
*/
|
||||
Assert(IsA(var1, Var));
|
||||
irel1 = var1->varno;
|
||||
Assert(IsA(var2, Var));
|
||||
irel2 = var2->varno;
|
||||
|
||||
/*
|
||||
* If both vars belong to same rel, we need to look at that rel's
|
||||
* baserestrictinfo list. If different rels, each will have a
|
||||
* joininfo node for the other, and we can scan either list.
|
||||
*/
|
||||
rel1 = find_base_rel(root, irel1);
|
||||
if (irel1 == irel2)
|
||||
restrictlist = rel1->baserestrictinfo;
|
||||
else
|
||||
{
|
||||
JoinInfo *joininfo = find_joininfo_node(rel1,
|
||||
makeListi1(irel2));
|
||||
|
||||
restrictlist = joininfo->jinfo_restrictinfo;
|
||||
}
|
||||
|
||||
/*
|
||||
* Scan to see if equality is known.
|
||||
*/
|
||||
foreach(itm, restrictlist)
|
||||
{
|
||||
RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(itm);
|
||||
Node *left,
|
||||
*right;
|
||||
|
||||
if (restrictinfo->mergejoinoperator == InvalidOid)
|
||||
continue; /* ignore non-mergejoinable clauses */
|
||||
/* We now know the restrictinfo clause is a binary opclause */
|
||||
left = (Node *) get_leftop(restrictinfo->clause);
|
||||
right = (Node *) get_rightop(restrictinfo->clause);
|
||||
if ((equal(var1, left) && equal(var2, right)) ||
|
||||
(equal(var2, left) && equal(var1, right)))
|
||||
return true; /* found a matching clause */
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* qual_is_redundant
|
||||
* Detect whether an implied-equality qual that turns out to be a
|
||||
|
|
|
|||
|
|
@ -8,14 +8,17 @@
|
|||
*
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $Header: /cvsroot/pgsql/src/backend/optimizer/plan/planner.c,v 1.128 2002/11/14 19:00:36 tgl Exp $
|
||||
* $Header: /cvsroot/pgsql/src/backend/optimizer/plan/planner.c,v 1.129 2002/11/19 23:21:59 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
#include "postgres.h"
|
||||
|
||||
#include <limits.h>
|
||||
|
||||
#include "catalog/pg_type.h"
|
||||
#include "miscadmin.h"
|
||||
#include "nodes/makefuncs.h"
|
||||
#ifdef OPTIMIZER_DEBUG
|
||||
#include "nodes/print.h"
|
||||
|
|
@ -35,6 +38,7 @@
|
|||
#include "parser/parse_expr.h"
|
||||
#include "rewrite/rewriteManip.h"
|
||||
#include "utils/lsyscache.h"
|
||||
#include "utils/selfuncs.h"
|
||||
|
||||
|
||||
/* Expression kind codes for preprocess_expression */
|
||||
|
|
@ -160,6 +164,23 @@ subquery_planner(Query *parse, double tuple_fraction)
|
|||
parse->jointree = (FromExpr *)
|
||||
preprocess_jointree(parse, (Node *) parse->jointree);
|
||||
|
||||
/*
|
||||
* Detect whether any rangetable entries are RTE_JOIN kind; if not,
|
||||
* we can avoid the expense of doing flatten_join_alias_vars().
|
||||
* This must be done after we have done pull_up_subqueries, of course.
|
||||
*/
|
||||
parse->hasJoinRTEs = false;
|
||||
foreach(lst, parse->rtable)
|
||||
{
|
||||
RangeTblEntry *rte = (RangeTblEntry *) lfirst(lst);
|
||||
|
||||
if (rte->rtekind == RTE_JOIN)
|
||||
{
|
||||
parse->hasJoinRTEs = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Do expression preprocessing on targetlist and quals.
|
||||
*/
|
||||
|
|
@ -694,9 +715,6 @@ preprocess_jointree(Query *parse, Node *jtnode)
|
|||
static Node *
|
||||
preprocess_expression(Query *parse, Node *expr, int kind)
|
||||
{
|
||||
bool has_join_rtes;
|
||||
List *rt;
|
||||
|
||||
/*
|
||||
* Simplify constant expressions.
|
||||
*
|
||||
|
|
@ -737,22 +755,8 @@ preprocess_expression(Query *parse, Node *expr, int kind)
|
|||
* with base-relation variables, to allow quals to be pushed down. We
|
||||
* must do this after sublink processing, since it does not recurse
|
||||
* into sublinks.
|
||||
*
|
||||
* The flattening pass is expensive enough that it seems worthwhile to
|
||||
* scan the rangetable to see if we can avoid it.
|
||||
*/
|
||||
has_join_rtes = false;
|
||||
foreach(rt, parse->rtable)
|
||||
{
|
||||
RangeTblEntry *rte = lfirst(rt);
|
||||
|
||||
if (rte->rtekind == RTE_JOIN)
|
||||
{
|
||||
has_join_rtes = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (has_join_rtes)
|
||||
if (parse->hasJoinRTEs)
|
||||
expr = flatten_join_alias_vars(expr, parse->rtable, false);
|
||||
|
||||
return expr;
|
||||
|
|
@ -931,6 +935,9 @@ grouping_planner(Query *parse, double tuple_fraction)
|
|||
AttrNumber *groupColIdx = NULL;
|
||||
Path *cheapest_path;
|
||||
Path *sorted_path;
|
||||
double dNumGroups = 0;
|
||||
long numGroups = 0;
|
||||
int numAggs = 0;
|
||||
bool use_hashed_grouping = false;
|
||||
|
||||
/* Preprocess targetlist in case we are inside an INSERT/UPDATE. */
|
||||
|
|
@ -1006,6 +1013,19 @@ grouping_planner(Query *parse, double tuple_fraction)
|
|||
sort_pathkeys = make_pathkeys_for_sortclauses(parse->sortClause,
|
||||
tlist);
|
||||
|
||||
/*
|
||||
* Will need actual number of aggregates for estimating costs.
|
||||
* Also, it's possible that optimization has eliminated all
|
||||
* aggregates, and we may as well check for that here.
|
||||
*/
|
||||
if (parse->hasAggs)
|
||||
{
|
||||
numAggs = length(pull_agg_clause((Node *) tlist)) +
|
||||
length(pull_agg_clause(parse->havingQual));
|
||||
if (numAggs == 0)
|
||||
parse->hasAggs = false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Figure out whether we need a sorted result from query_planner.
|
||||
*
|
||||
|
|
@ -1215,6 +1235,14 @@ grouping_planner(Query *parse, double tuple_fraction)
|
|||
*/
|
||||
if (parse->groupClause)
|
||||
{
|
||||
/*
|
||||
* Always estimate the number of groups.
|
||||
*/
|
||||
dNumGroups = estimate_num_groups(parse,
|
||||
parse->groupClause,
|
||||
cheapest_path->parent->rows);
|
||||
numGroups = (long) Min(dNumGroups, (double) LONG_MAX);
|
||||
|
||||
/*
|
||||
* Executor doesn't support hashed aggregation with DISTINCT
|
||||
* aggregates. (Doing so would imply storing *all* the input
|
||||
|
|
@ -1226,10 +1254,30 @@ grouping_planner(Query *parse, double tuple_fraction)
|
|||
use_hashed_grouping = false;
|
||||
else
|
||||
{
|
||||
#if 0 /* much more to do here */
|
||||
/* TEMPORARY HOTWIRE FOR TESTING */
|
||||
use_hashed_grouping = true;
|
||||
/*
|
||||
* Use hashed grouping if (a) we think we can fit the
|
||||
* hashtable into SortMem, *and* (b) the estimated cost
|
||||
* is no more than doing it the other way. While avoiding
|
||||
* the need for sorted input is usually a win, the fact
|
||||
* that the output won't be sorted may be a loss; so we
|
||||
* need to do an actual cost comparison.
|
||||
*
|
||||
* In most cases we have no good way to estimate the size of
|
||||
* the transition value needed by an aggregate; arbitrarily
|
||||
* assume it is 100 bytes. Also set the overhead per hashtable
|
||||
* entry at 64 bytes.
|
||||
*/
|
||||
int hashentrysize = cheapest_path->parent->width + 64 +
|
||||
numAggs * 100;
|
||||
|
||||
if (hashentrysize * dNumGroups <= SortMem * 1024L)
|
||||
{
|
||||
/* much more to do here */
|
||||
#if 0
|
||||
/* TEMPORARY HOTWIRE FOR TESTING */
|
||||
use_hashed_grouping = true;
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1319,6 +1367,8 @@ grouping_planner(Query *parse, double tuple_fraction)
|
|||
AGG_HASHED,
|
||||
length(parse->groupClause),
|
||||
groupColIdx,
|
||||
numGroups,
|
||||
numAggs,
|
||||
result_plan);
|
||||
/* Hashed aggregation produces randomly-ordered results */
|
||||
current_pathkeys = NIL;
|
||||
|
|
@ -1356,6 +1406,8 @@ grouping_planner(Query *parse, double tuple_fraction)
|
|||
aggstrategy,
|
||||
length(parse->groupClause),
|
||||
groupColIdx,
|
||||
numGroups,
|
||||
numAggs,
|
||||
result_plan);
|
||||
}
|
||||
else
|
||||
|
|
@ -1387,6 +1439,7 @@ grouping_planner(Query *parse, double tuple_fraction)
|
|||
result_plan = (Plan *) make_group(tlist,
|
||||
length(parse->groupClause),
|
||||
groupColIdx,
|
||||
dNumGroups,
|
||||
result_plan);
|
||||
}
|
||||
}
|
||||
|
|
@ -1410,6 +1463,16 @@ grouping_planner(Query *parse, double tuple_fraction)
|
|||
{
|
||||
result_plan = (Plan *) make_unique(tlist, result_plan,
|
||||
parse->distinctClause);
|
||||
/*
|
||||
* If there was grouping or aggregation, leave plan_rows as-is
|
||||
* (ie, assume the result was already mostly unique). If not,
|
||||
* it's reasonable to assume the UNIQUE filter has effects
|
||||
* comparable to GROUP BY.
|
||||
*/
|
||||
if (!parse->groupClause && !parse->hasAggs)
|
||||
result_plan->plan_rows = estimate_num_groups(parse,
|
||||
parse->distinctClause,
|
||||
result_plan->plan_rows);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
*
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $Header: /cvsroot/pgsql/src/backend/optimizer/plan/setrefs.c,v 1.81 2002/09/04 20:31:21 momjian Exp $
|
||||
* $Header: /cvsroot/pgsql/src/backend/optimizer/plan/setrefs.c,v 1.82 2002/11/19 23:21:59 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -439,7 +439,14 @@ join_references_mutator(Node *node,
|
|||
return (Node *) newvar;
|
||||
}
|
||||
|
||||
/* Perhaps it's a join alias that can be resolved to input vars? */
|
||||
/* Return the Var unmodified, if it's for acceptable_rel */
|
||||
if (var->varno == context->acceptable_rel)
|
||||
return (Node *) copyObject(var);
|
||||
|
||||
/*
|
||||
* Perhaps it's a join alias that can be resolved to input vars?
|
||||
* We try this last since it's relatively slow.
|
||||
*/
|
||||
newnode = flatten_join_alias_vars((Node *) var,
|
||||
context->rtable,
|
||||
true);
|
||||
|
|
@ -450,13 +457,8 @@ join_references_mutator(Node *node,
|
|||
return newnode;
|
||||
}
|
||||
|
||||
/*
|
||||
* No referent found for Var --- either raise an error, or return
|
||||
* the Var unmodified if it's for acceptable_rel.
|
||||
*/
|
||||
if (var->varno != context->acceptable_rel)
|
||||
elog(ERROR, "join_references: variable not in subplan target lists");
|
||||
return (Node *) copyObject(var);
|
||||
/* No referent found for Var */
|
||||
elog(ERROR, "join_references: variable not in subplan target lists");
|
||||
}
|
||||
return expression_tree_mutator(node,
|
||||
join_references_mutator,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
*
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.120 2002/11/08 20:23:57 momjian Exp $
|
||||
* $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.121 2002/11/19 23:21:59 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -85,7 +85,10 @@
|
|||
#include "optimizer/cost.h"
|
||||
#include "optimizer/pathnode.h"
|
||||
#include "optimizer/plancat.h"
|
||||
#include "optimizer/planmain.h"
|
||||
#include "optimizer/prep.h"
|
||||
#include "optimizer/tlist.h"
|
||||
#include "optimizer/var.h"
|
||||
#include "parser/parse_func.h"
|
||||
#include "parser/parse_oper.h"
|
||||
#include "parser/parsetree.h"
|
||||
|
|
@ -1809,6 +1812,251 @@ mergejoinscansel(Query *root, Node *clause,
|
|||
*rightscan = 1.0;
|
||||
}
|
||||
|
||||
/*
|
||||
* estimate_num_groups - Estimate number of groups in a grouped query
|
||||
*
|
||||
* Given a query having a GROUP BY clause, estimate how many groups there
|
||||
* will be --- ie, the number of distinct combinations of the GROUP BY
|
||||
* expressions.
|
||||
*
|
||||
* This routine is also used to estimate the number of rows emitted by
|
||||
* a DISTINCT filtering step; that is an isomorphic problem. (Note:
|
||||
* actually, we only use it for DISTINCT when there's no grouping or
|
||||
* aggregation ahead of the DISTINCT.)
|
||||
*
|
||||
* Inputs:
|
||||
* root - the query
|
||||
* groupClauses - list of GroupClauses (or SortClauses for the DISTINCT
|
||||
* case, but those are equivalent structs)
|
||||
* input_rows - number of rows estimated to arrive at the group/unique
|
||||
* filter step
|
||||
*
|
||||
* Given the lack of any cross-correlation statistics in the system, it's
|
||||
* impossible to do anything really trustworthy with GROUP BY conditions
|
||||
* involving multiple Vars. We should however avoid assuming the worst
|
||||
* case (all possible cross-product terms actually appear as groups) since
|
||||
* very often the grouped-by Vars are highly correlated. Our current approach
|
||||
* is as follows:
|
||||
* 1. Reduce the given expressions to a list of unique Vars used. For
|
||||
* example, GROUP BY a, a + b is treated the same as GROUP BY a, b.
|
||||
* It is clearly correct not to count the same Var more than once.
|
||||
* It is also reasonable to treat f(x) the same as x: f() cannot
|
||||
* increase the number of distinct values (unless it is volatile,
|
||||
* which we consider unlikely for grouping), but it probably won't
|
||||
* reduce the number of distinct values much either.
|
||||
* 2. If the list contains Vars of different relations that are known equal
|
||||
* due to equijoin clauses, then drop all but one of the Vars from each
|
||||
* known-equal set, keeping the one with smallest estimated # of values
|
||||
* (since the extra values of the others can't appear in joined rows).
|
||||
* Note the reason we only consider Vars of different relations is that
|
||||
* if we considered ones of the same rel, we'd be double-counting the
|
||||
* restriction selectivity of the equality in the next step.
|
||||
* 3. For Vars within a single source rel, we multiply together the numbers
|
||||
* of values, clamp to the number of rows in the rel, and then multiply
|
||||
* by the selectivity of the restriction clauses for that rel. The
|
||||
* initial product is probably too high (it's the worst case) but since
|
||||
* we can clamp to the rel's rows it won't be hugely bad. Multiplying
|
||||
* by the restriction selectivity is effectively assuming that the
|
||||
* restriction clauses are independent of the grouping, which is a crummy
|
||||
* assumption, but it's hard to do better.
|
||||
* 4. If there are Vars from multiple rels, we repeat step 3 for each such
|
||||
* rel, and multiply the results together.
|
||||
* Note that rels not containing grouped Vars are ignored completely, as are
|
||||
* join clauses other than the equijoin clauses used in step 2. Such rels
|
||||
* cannot increase the number of groups, and we assume such clauses do not
|
||||
* reduce the number either (somewhat bogus, but we don't have the info to
|
||||
* do better).
|
||||
*/
|
||||
double
|
||||
estimate_num_groups(Query *root, List *groupClauses, double input_rows)
|
||||
{
|
||||
List *allvars = NIL;
|
||||
List *varinfos = NIL;
|
||||
double numdistinct;
|
||||
List *l;
|
||||
typedef struct { /* varinfos is a List of these */
|
||||
Var *var;
|
||||
double ndistinct;
|
||||
} MyVarInfo;
|
||||
|
||||
/* We should not be called unless query has GROUP BY (or DISTINCT) */
|
||||
Assert(groupClauses != NIL);
|
||||
|
||||
/* Step 1: get the unique Vars used */
|
||||
foreach(l, groupClauses)
|
||||
{
|
||||
GroupClause *grpcl = (GroupClause *) lfirst(l);
|
||||
Node *groupexpr = get_sortgroupclause_expr(grpcl,
|
||||
root->targetList);
|
||||
List *varshere;
|
||||
|
||||
varshere = pull_var_clause(groupexpr, false);
|
||||
/*
|
||||
* Replace any JOIN alias Vars with the underlying Vars. (This
|
||||
* is not really right for FULL JOIN ...)
|
||||
*/
|
||||
if (root->hasJoinRTEs)
|
||||
{
|
||||
varshere = (List *) flatten_join_alias_vars((Node *) varshere,
|
||||
root->rtable,
|
||||
true);
|
||||
varshere = pull_var_clause((Node *) varshere, false);
|
||||
}
|
||||
/*
|
||||
* If we find any variable-free GROUP BY item, then either it is
|
||||
* a constant (and we can ignore it) or it contains a volatile
|
||||
* function; in the latter case we punt and assume that each input
|
||||
* row will yield a distinct group.
|
||||
*/
|
||||
if (varshere == NIL)
|
||||
{
|
||||
if (contain_volatile_functions(groupexpr))
|
||||
return input_rows;
|
||||
continue;
|
||||
}
|
||||
allvars = nconc(allvars, varshere);
|
||||
}
|
||||
|
||||
/* If now no Vars, we must have an all-constant GROUP BY list. */
|
||||
if (allvars == NIL)
|
||||
return 1.0;
|
||||
|
||||
/* Use set_union() to discard duplicates */
|
||||
allvars = set_union(NIL, allvars);
|
||||
|
||||
/*
|
||||
* Step 2: acquire statistical estimate of number of distinct values
|
||||
* of each Var (total in its table, without regard for filtering).
|
||||
* Also, detect known-equal Vars and discard the ones we don't want.
|
||||
*/
|
||||
foreach(l, allvars)
|
||||
{
|
||||
Var *var = (Var *) lfirst(l);
|
||||
Oid relid = getrelid(var->varno, root->rtable);
|
||||
HeapTuple statsTuple = NULL;
|
||||
Form_pg_statistic stats = NULL;
|
||||
double ndistinct;
|
||||
bool keep = true;
|
||||
List *l2;
|
||||
|
||||
if (OidIsValid(relid))
|
||||
{
|
||||
statsTuple = SearchSysCache(STATRELATT,
|
||||
ObjectIdGetDatum(relid),
|
||||
Int16GetDatum(var->varattno),
|
||||
0, 0);
|
||||
if (HeapTupleIsValid(statsTuple))
|
||||
stats = (Form_pg_statistic) GETSTRUCT(statsTuple);
|
||||
}
|
||||
ndistinct = get_att_numdistinct(root, var, stats);
|
||||
if (HeapTupleIsValid(statsTuple))
|
||||
ReleaseSysCache(statsTuple);
|
||||
|
||||
foreach(l2, varinfos)
|
||||
{
|
||||
MyVarInfo *varinfo = (MyVarInfo *) lfirst(l2);
|
||||
|
||||
if (var->varno != varinfo->var->varno &&
|
||||
vars_known_equal(root, var, varinfo->var))
|
||||
{
|
||||
/* Found a match */
|
||||
if (varinfo->ndistinct <= ndistinct)
|
||||
{
|
||||
/* Keep older item, forget new one */
|
||||
keep = false;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
/*
|
||||
* Delete the older item. We assume lremove() will not
|
||||
* break the lnext link of the item...
|
||||
*/
|
||||
varinfos = lremove(varinfo, varinfos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (keep)
|
||||
{
|
||||
MyVarInfo *varinfo = (MyVarInfo *) palloc(sizeof(MyVarInfo));
|
||||
|
||||
varinfo->var = var;
|
||||
varinfo->ndistinct = ndistinct;
|
||||
varinfos = lcons(varinfo, varinfos);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Steps 3/4: group Vars by relation and estimate total numdistinct.
|
||||
*
|
||||
* For each iteration of the outer loop, we process the frontmost
|
||||
* Var in varinfos, plus all other Vars in the same relation. We
|
||||
* remove these Vars from the newvarinfos list for the next iteration.
|
||||
* This is the easiest way to group Vars of same rel together.
|
||||
*/
|
||||
Assert(varinfos != NIL);
|
||||
numdistinct = 1.0;
|
||||
|
||||
do
|
||||
{
|
||||
MyVarInfo *varinfo1 = (MyVarInfo *) lfirst(varinfos);
|
||||
RelOptInfo *rel = find_base_rel(root, varinfo1->var->varno);
|
||||
double reldistinct = varinfo1->ndistinct;
|
||||
List *newvarinfos = NIL;
|
||||
|
||||
/*
|
||||
* Get the largest numdistinct estimate of the Vars for this rel.
|
||||
* Also, construct new varinfos list of remaining Vars.
|
||||
*/
|
||||
foreach(l, lnext(varinfos))
|
||||
{
|
||||
MyVarInfo *varinfo2 = (MyVarInfo *) lfirst(l);
|
||||
|
||||
if (varinfo2->var->varno == varinfo1->var->varno)
|
||||
{
|
||||
reldistinct *= varinfo2->ndistinct;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* not time to process varinfo2 yet */
|
||||
newvarinfos = lcons(varinfo2, newvarinfos);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Clamp to size of rel, multiply by restriction selectivity.
|
||||
*/
|
||||
Assert(rel->reloptkind == RELOPT_BASEREL);
|
||||
if (reldistinct > rel->tuples)
|
||||
reldistinct = rel->tuples;
|
||||
reldistinct *= rel->rows / rel->tuples;
|
||||
|
||||
/*
|
||||
* Update estimate of total distinct groups.
|
||||
*/
|
||||
numdistinct *= reldistinct;
|
||||
|
||||
varinfos = newvarinfos;
|
||||
} while (varinfos != NIL);
|
||||
|
||||
/* Guard against out-of-range answers */
|
||||
if (numdistinct > input_rows)
|
||||
numdistinct = input_rows;
|
||||
if (numdistinct < 1.0)
|
||||
numdistinct = 1.0;
|
||||
|
||||
return numdistinct;
|
||||
}
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
*
|
||||
* Support routines
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
/*
|
||||
* get_var_maximum
|
||||
* Estimate the maximum value of the specified variable.
|
||||
|
|
@ -3271,7 +3519,7 @@ pattern_selectivity(Const *patt, Pattern_Type ptype)
|
|||
|
||||
|
||||
/*
|
||||
* We want test whether the database's LC_COLLATE setting is safe for
|
||||
* We want to test whether the database's LC_COLLATE setting is safe for
|
||||
* LIKE/regexp index optimization.
|
||||
*
|
||||
* The key requirement here is that given a prefix string, say "foo",
|
||||
|
|
@ -3284,7 +3532,7 @@ pattern_selectivity(Const *patt, Pattern_Type ptype)
|
|||
*
|
||||
* (In theory, locales other than C may be LIKE-safe so this function
|
||||
* could be different from lc_collate_is_c(), but in a different
|
||||
* theory, non-C locales are completely unpredicable so it's unlikely
|
||||
* theory, non-C locales are completely unpredictable so it's unlikely
|
||||
* to happen.)
|
||||
*
|
||||
* Be sure to maintain the correspondence with the code in initdb.
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
* Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
|
||||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* $Id: parsenodes.h,v 1.215 2002/11/15 03:09:39 momjian Exp $
|
||||
* $Id: parsenodes.h,v 1.216 2002/11/19 23:21:59 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -102,6 +102,7 @@ typedef struct Query
|
|||
List *equi_key_list; /* list of lists of equijoined
|
||||
* PathKeyItems */
|
||||
List *query_pathkeys; /* desired pathkeys for query_planner() */
|
||||
bool hasJoinRTEs; /* true if any RTEs are RTE_JOIN kind */
|
||||
} Query;
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
* Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
|
||||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* $Id: planmain.h,v 1.61 2002/11/06 00:00:45 tgl Exp $
|
||||
* $Id: planmain.h,v 1.62 2002/11/19 23:22:00 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -35,8 +35,11 @@ extern Sort *make_sort(Query *root, List *tlist,
|
|||
extern Sort *make_sort_from_pathkeys(Query *root, List *tlist,
|
||||
Plan *lefttree, List *pathkeys);
|
||||
extern Agg *make_agg(List *tlist, List *qual, AggStrategy aggstrategy,
|
||||
int ngrp, AttrNumber *grpColIdx, Plan *lefttree);
|
||||
extern Group *make_group(List *tlist, int ngrp, AttrNumber *grpColIdx,
|
||||
int ngrp, AttrNumber *grpColIdx,
|
||||
long numGroups, int numAggs,
|
||||
Plan *lefttree);
|
||||
extern Group *make_group(List *tlist,
|
||||
int ngrp, AttrNumber *grpColIdx, double numGroups,
|
||||
Plan *lefttree);
|
||||
extern Material *make_material(List *tlist, Plan *lefttree);
|
||||
extern Unique *make_unique(List *tlist, Plan *lefttree, List *distinctList);
|
||||
|
|
@ -54,6 +57,7 @@ extern void build_base_rel_tlists(Query *root, List *tlist);
|
|||
extern Relids distribute_quals_to_rels(Query *root, Node *jtnode);
|
||||
extern void process_implied_equality(Query *root, Node *item1, Node *item2,
|
||||
Oid sortop1, Oid sortop2);
|
||||
extern bool vars_known_equal(Query *root, Var *var1, Var *var2);
|
||||
|
||||
/*
|
||||
* prototypes for plan/setrefs.c
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
* Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
|
||||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* $Id: selfuncs.h,v 1.9 2002/10/19 02:56:16 tgl Exp $
|
||||
* $Id: selfuncs.h,v 1.10 2002/11/19 23:22:00 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
|
@ -75,6 +75,9 @@ extern void mergejoinscansel(Query *root, Node *clause,
|
|||
Selectivity *leftscan,
|
||||
Selectivity *rightscan);
|
||||
|
||||
extern double estimate_num_groups(Query *root, List *groupClauses,
|
||||
double input_rows);
|
||||
|
||||
extern Datum btcostestimate(PG_FUNCTION_ARGS);
|
||||
extern Datum rtcostestimate(PG_FUNCTION_ARGS);
|
||||
extern Datum hashcostestimate(PG_FUNCTION_ARGS);
|
||||
|
|
|
|||
Loading…
Reference in New Issue