2015-05-15 04:02:54 +02:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* sampling.c
|
|
|
|
* Relation block sampling routines.
|
|
|
|
*
|
2020-01-01 18:21:45 +01:00
|
|
|
* Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
|
2015-05-15 04:02:54 +02:00
|
|
|
* Portions Copyright (c) 1994, Regents of the University of California
|
|
|
|
*
|
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
|
|
|
* src/backend/utils/misc/sampling.c
|
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include <math.h>
|
|
|
|
|
|
|
|
#include "utils/sampling.h"
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* BlockSampler_Init -- prepare for random sampling of blocknumbers
|
|
|
|
*
|
|
|
|
* BlockSampler provides algorithm for block level sampling of a relation
|
|
|
|
* as discussed on pgsql-hackers 2004-04-02 (subject "Large DB")
|
|
|
|
* It selects a random sample of samplesize blocks out of
|
|
|
|
* the nblocks blocks in the table. If the table has less than
|
|
|
|
* samplesize blocks, all blocks are selected.
|
|
|
|
*
|
|
|
|
* Since we know the total number of blocks in advance, we can use the
|
|
|
|
* straightforward Algorithm S from Knuth 3.4.2, rather than Vitter's
|
|
|
|
* algorithm.
|
2020-01-15 15:02:09 +01:00
|
|
|
*
|
|
|
|
* Returns the number of blocks that BlockSampler_Next will return.
|
2015-05-15 04:02:54 +02:00
|
|
|
*/
|
2020-01-15 15:02:09 +01:00
|
|
|
BlockNumber
|
2015-05-15 04:02:54 +02:00
|
|
|
BlockSampler_Init(BlockSampler bs, BlockNumber nblocks, int samplesize,
|
|
|
|
long randseed)
|
|
|
|
{
|
|
|
|
bs->N = nblocks; /* measured table size */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we decide to reduce samplesize for tables that have less or not much
|
|
|
|
* more than samplesize blocks, here is the place to do it.
|
|
|
|
*/
|
|
|
|
bs->n = samplesize;
|
|
|
|
bs->t = 0; /* blocks scanned so far */
|
|
|
|
bs->m = 0; /* blocks selected so far */
|
2015-05-15 20:37:10 +02:00
|
|
|
|
|
|
|
sampler_random_init_state(randseed, bs->randstate);
|
2020-01-15 15:02:09 +01:00
|
|
|
|
|
|
|
return Min(bs->n, bs->N);
|
2015-05-15 04:02:54 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
BlockSampler_HasMore(BlockSampler bs)
|
|
|
|
{
|
|
|
|
return (bs->t < bs->N) && (bs->m < bs->n);
|
|
|
|
}
|
|
|
|
|
|
|
|
BlockNumber
|
|
|
|
BlockSampler_Next(BlockSampler bs)
|
|
|
|
{
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
BlockNumber K = bs->N - bs->t; /* remaining blocks */
|
|
|
|
int k = bs->n - bs->m; /* blocks still to sample */
|
2015-05-15 04:02:54 +02:00
|
|
|
double p; /* probability to skip block */
|
|
|
|
double V; /* random */
|
|
|
|
|
|
|
|
Assert(BlockSampler_HasMore(bs)); /* hence K > 0 and k > 0 */
|
|
|
|
|
|
|
|
if ((BlockNumber) k >= K)
|
|
|
|
{
|
|
|
|
/* need all the rest */
|
|
|
|
bs->m++;
|
|
|
|
return bs->t++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*----------
|
|
|
|
* It is not obvious that this code matches Knuth's Algorithm S.
|
|
|
|
* Knuth says to skip the current block with probability 1 - k/K.
|
|
|
|
* If we are to skip, we should advance t (hence decrease K), and
|
|
|
|
* repeat the same probabilistic test for the next block. The naive
|
2015-05-20 15:18:11 +02:00
|
|
|
* implementation thus requires a sampler_random_fract() call for each
|
2015-05-15 04:02:54 +02:00
|
|
|
* block number. But we can reduce this to one sampler_random_fract()
|
|
|
|
* call per selected block, by noting that each time the while-test
|
|
|
|
* succeeds, we can reinterpret V as a uniform random number in the range
|
|
|
|
* 0 to p. Therefore, instead of choosing a new V, we just adjust p to be
|
|
|
|
* the appropriate fraction of its former value, and our next loop
|
|
|
|
* makes the appropriate probabilistic test.
|
|
|
|
*
|
|
|
|
* We have initially K > k > 0. If the loop reduces K to equal k,
|
|
|
|
* the next while-test must fail since p will become exactly zero
|
|
|
|
* (we assume there will not be roundoff error in the division).
|
|
|
|
* (Note: Knuth suggests a "<=" loop condition, but we use "<" just
|
|
|
|
* to be doubly sure about roundoff error.) Therefore K cannot become
|
|
|
|
* less than k, which means that we cannot fail to select enough blocks.
|
|
|
|
*----------
|
|
|
|
*/
|
2015-05-15 20:37:10 +02:00
|
|
|
V = sampler_random_fract(bs->randstate);
|
2015-05-15 04:02:54 +02:00
|
|
|
p = 1.0 - (double) k / (double) K;
|
|
|
|
while (V < p)
|
|
|
|
{
|
|
|
|
/* skip */
|
|
|
|
bs->t++;
|
|
|
|
K--; /* keep K == N - t */
|
|
|
|
|
|
|
|
/* adjust p to be new cutoff point in reduced range */
|
|
|
|
p *= 1.0 - (double) k / (double) K;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* select */
|
|
|
|
bs->m++;
|
|
|
|
return bs->t++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* These two routines embody Algorithm Z from "Random sampling with a
|
|
|
|
* reservoir" by Jeffrey S. Vitter, in ACM Trans. Math. Softw. 11, 1
|
|
|
|
* (Mar. 1985), Pages 37-57. Vitter describes his algorithm in terms
|
|
|
|
* of the count S of records to skip before processing another record.
|
|
|
|
* It is computed primarily based on t, the number of records already read.
|
|
|
|
* The only extra state needed between calls is W, a random state variable.
|
|
|
|
*
|
|
|
|
* reservoir_init_selection_state computes the initial W value.
|
|
|
|
*
|
|
|
|
* Given that we've already read t records (t >= n), reservoir_get_next_S
|
|
|
|
* determines the number of records to skip before the next record is
|
|
|
|
* processed.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
reservoir_init_selection_state(ReservoirState rs, int n)
|
|
|
|
{
|
2015-05-15 20:37:10 +02:00
|
|
|
/*
|
|
|
|
* Reservoir sampling is not used anywhere where it would need to return
|
|
|
|
* repeatable results so we can initialize it randomly.
|
|
|
|
*/
|
|
|
|
sampler_random_init_state(random(), rs->randstate);
|
|
|
|
|
2015-05-15 04:02:54 +02:00
|
|
|
/* Initial value of W (for use when Algorithm Z is first applied) */
|
2015-05-15 20:37:10 +02:00
|
|
|
rs->W = exp(-log(sampler_random_fract(rs->randstate)) / n);
|
2015-05-15 04:02:54 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
double
|
|
|
|
reservoir_get_next_S(ReservoirState rs, double t, int n)
|
|
|
|
{
|
|
|
|
double S;
|
|
|
|
|
|
|
|
/* The magic constant here is T from Vitter's paper */
|
|
|
|
if (t <= (22.0 * n))
|
|
|
|
{
|
|
|
|
/* Process records using Algorithm X until t is large enough */
|
|
|
|
double V,
|
|
|
|
quot;
|
|
|
|
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
V = sampler_random_fract(rs->randstate); /* Generate V */
|
2015-05-15 04:02:54 +02:00
|
|
|
S = 0;
|
|
|
|
t += 1;
|
|
|
|
/* Note: "num" in Vitter's code is always equal to t - n */
|
|
|
|
quot = (t - (double) n) / t;
|
|
|
|
/* Find min S satisfying (4.1) */
|
|
|
|
while (quot > V)
|
|
|
|
{
|
|
|
|
S += 1;
|
|
|
|
t += 1;
|
|
|
|
quot *= (t - (double) n) / t;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* Now apply Algorithm Z */
|
2015-05-15 20:37:10 +02:00
|
|
|
double W = rs->W;
|
2015-05-15 04:02:54 +02:00
|
|
|
double term = t - (double) n + 1;
|
|
|
|
|
|
|
|
for (;;)
|
|
|
|
{
|
|
|
|
double numer,
|
|
|
|
numer_lim,
|
|
|
|
denom;
|
|
|
|
double U,
|
|
|
|
X,
|
|
|
|
lhs,
|
|
|
|
rhs,
|
|
|
|
y,
|
|
|
|
tmp;
|
|
|
|
|
|
|
|
/* Generate U and X */
|
2015-05-15 20:37:10 +02:00
|
|
|
U = sampler_random_fract(rs->randstate);
|
2015-05-15 04:02:54 +02:00
|
|
|
X = t * (W - 1.0);
|
|
|
|
S = floor(X); /* S is tentatively set to floor(X) */
|
|
|
|
/* Test if U <= h(S)/cg(X) in the manner of (6.3) */
|
|
|
|
tmp = (t + 1) / term;
|
|
|
|
lhs = exp(log(((U * tmp * tmp) * (term + S)) / (t + X)) / n);
|
|
|
|
rhs = (((t + X) / (term + S)) * term) / t;
|
|
|
|
if (lhs <= rhs)
|
|
|
|
{
|
|
|
|
W = rhs / lhs;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
/* Test if U <= f(S)/cg(X) */
|
|
|
|
y = (((U * (t + 1)) / term) * (t + S + 1)) / (t + X);
|
|
|
|
if ((double) n < S)
|
|
|
|
{
|
|
|
|
denom = t;
|
|
|
|
numer_lim = term + S;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
denom = t - (double) n + S;
|
|
|
|
numer_lim = t + 1;
|
|
|
|
}
|
|
|
|
for (numer = t + S; numer >= numer_lim; numer -= 1)
|
|
|
|
{
|
|
|
|
y *= numer / denom;
|
|
|
|
denom -= 1;
|
|
|
|
}
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
W = exp(-log(sampler_random_fract(rs->randstate)) / n); /* Generate W in advance */
|
2015-05-15 04:02:54 +02:00
|
|
|
if (exp(log(y) / n) <= (t + X) / t)
|
|
|
|
break;
|
|
|
|
}
|
2015-05-15 20:37:10 +02:00
|
|
|
rs->W = W;
|
2015-05-15 04:02:54 +02:00
|
|
|
}
|
|
|
|
return S;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*----------
|
|
|
|
* Random number generator used by sampling
|
|
|
|
*----------
|
|
|
|
*/
|
2015-05-15 20:37:10 +02:00
|
|
|
void
|
|
|
|
sampler_random_init_state(long seed, SamplerRandomState randstate)
|
|
|
|
{
|
Redesign tablesample method API, and do extensive code review.
The original implementation of TABLESAMPLE modeled the tablesample method
API on index access methods, which wasn't a good choice because, without
specialized DDL commands, there's no way to build an extension that can
implement a TSM. (Raw inserts into system catalogs are not an acceptable
thing to do, because we can't undo them during DROP EXTENSION, nor will
pg_upgrade behave sanely.) Instead adopt an API more like procedural
language handlers or foreign data wrappers, wherein the only SQL-level
support object needed is a single handler function identified by having
a special return type. This lets us get rid of the supporting catalog
altogether, so that no custom DDL support is needed for the feature.
Adjust the API so that it can support non-constant tablesample arguments
(the original coding assumed we could evaluate the argument expressions at
ExecInitSampleScan time, which is undesirable even if it weren't outright
unsafe), and discourage sampling methods from looking at invisible tuples.
Make sure that the BERNOULLI and SYSTEM methods are genuinely repeatable
within and across queries, as required by the SQL standard, and deal more
honestly with methods that can't support that requirement.
Make a full code-review pass over the tablesample additions, and fix
assorted bugs, omissions, infelicities, and cosmetic issues (such as
failure to put the added code stanzas in a consistent ordering).
Improve EXPLAIN's output of tablesample plans, too.
Back-patch to 9.5 so that we don't have to support the original API
in production.
2015-07-25 20:39:00 +02:00
|
|
|
randstate[0] = 0x330e; /* same as pg_erand48, but could be anything */
|
2015-05-15 20:37:10 +02:00
|
|
|
randstate[1] = (unsigned short) seed;
|
|
|
|
randstate[2] = (unsigned short) (seed >> 16);
|
|
|
|
}
|
2015-05-15 04:02:54 +02:00
|
|
|
|
|
|
|
/* Select a random value R uniformly distributed in (0 - 1) */
|
|
|
|
double
|
2015-05-15 20:37:10 +02:00
|
|
|
sampler_random_fract(SamplerRandomState randstate)
|
2015-05-15 04:02:54 +02:00
|
|
|
{
|
2015-07-02 00:07:48 +02:00
|
|
|
double res;
|
|
|
|
|
|
|
|
/* pg_erand48 returns a value in [0.0 - 1.0), so we must reject 0 */
|
|
|
|
do
|
|
|
|
{
|
|
|
|
res = pg_erand48(randstate);
|
|
|
|
} while (res == 0.0);
|
|
|
|
return res;
|
2015-05-15 04:02:54 +02:00
|
|
|
}
|
2015-05-19 00:34:37 +02:00
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Backwards-compatible API for block sampling
|
|
|
|
*
|
|
|
|
* This code is now deprecated, but since it's still in use by many FDWs,
|
|
|
|
* we should keep it for awhile at least. The functionality is the same as
|
|
|
|
* sampler_random_fract/reservoir_init_selection_state/reservoir_get_next_S,
|
|
|
|
* except that a common random state is used across all callers.
|
|
|
|
*/
|
|
|
|
static ReservoirStateData oldrs;
|
|
|
|
|
|
|
|
double
|
|
|
|
anl_random_fract(void)
|
|
|
|
{
|
|
|
|
/* initialize if first time through */
|
|
|
|
if (oldrs.randstate[0] == 0)
|
|
|
|
sampler_random_init_state(random(), oldrs.randstate);
|
|
|
|
|
|
|
|
/* and compute a random fraction */
|
|
|
|
return sampler_random_fract(oldrs.randstate);
|
|
|
|
}
|
|
|
|
|
|
|
|
double
|
|
|
|
anl_init_selection_state(int n)
|
|
|
|
{
|
|
|
|
/* initialize if first time through */
|
|
|
|
if (oldrs.randstate[0] == 0)
|
|
|
|
sampler_random_init_state(random(), oldrs.randstate);
|
|
|
|
|
|
|
|
/* Initial value of W (for use when Algorithm Z is first applied) */
|
|
|
|
return exp(-log(sampler_random_fract(oldrs.randstate)) / n);
|
|
|
|
}
|
|
|
|
|
|
|
|
double
|
|
|
|
anl_get_next_S(double t, int n, double *stateptr)
|
|
|
|
{
|
2015-05-24 03:35:49 +02:00
|
|
|
double result;
|
2015-05-19 00:34:37 +02:00
|
|
|
|
|
|
|
oldrs.W = *stateptr;
|
|
|
|
result = reservoir_get_next_S(&oldrs, t, n);
|
|
|
|
*stateptr = oldrs.W;
|
|
|
|
return result;
|
|
|
|
}
|