2003-09-13 18:44:49 +02:00
|
|
|
--
|
|
|
|
-- Test Statistics Collector
|
|
|
|
--
|
|
|
|
-- Must be run after tenk2 has been created (by create_table),
|
|
|
|
-- populated (by create_misc) and indexed (by create_index).
|
|
|
|
--
|
|
|
|
-- conditio sine qua non
|
2007-09-24 05:12:23 +02:00
|
|
|
SHOW track_counts; -- must be on
|
|
|
|
track_counts
|
|
|
|
--------------
|
2003-09-13 18:44:49 +02:00
|
|
|
on
|
|
|
|
(1 row)
|
|
|
|
|
2011-10-09 05:45:58 +02:00
|
|
|
-- ensure that both seqscan and indexscan plans are allowed
|
|
|
|
SET enable_seqscan TO on;
|
|
|
|
SET enable_indexscan TO on;
|
|
|
|
-- for the moment, we don't want index-only scans here
|
|
|
|
SET enable_indexonlyscan TO off;
|
2003-09-13 18:44:49 +02:00
|
|
|
-- save counters
|
2017-05-13 15:42:12 +02:00
|
|
|
CREATE TABLE prevstats AS
|
2003-09-13 18:44:49 +02:00
|
|
|
SELECT t.seq_scan, t.seq_tup_read, t.idx_scan, t.idx_tup_fetch,
|
|
|
|
(b.heap_blks_read + b.heap_blks_hit) AS heap_blks,
|
2015-02-20 03:36:50 +01:00
|
|
|
(b.idx_blks_read + b.idx_blks_hit) AS idx_blks,
|
|
|
|
pg_stat_get_snapshot_timestamp() as snap_ts
|
2003-09-13 18:44:49 +02:00
|
|
|
FROM pg_catalog.pg_stat_user_tables AS t,
|
|
|
|
pg_catalog.pg_statio_user_tables AS b
|
|
|
|
WHERE t.relname='tenk2' AND b.relname='tenk2';
|
2007-02-08 00:11:30 +01:00
|
|
|
-- function to wait for counters to advance
|
2007-02-07 19:34:56 +01:00
|
|
|
create function wait_for_stats() returns void as $$
|
|
|
|
declare
|
|
|
|
start_time timestamptz := clock_timestamp();
|
Make stats regression test robust in the face of parallel query.
Historically, the wait_for_stats() function in this test has simply checked
for a report of an indexscan on tenk2, corresponding to the last command
issued before we expect stats updates to appear. However, with parallel
query that indexscan could be done by a parallel worker that will emit
its stats counters to the collector before the session's main backend does
(a full second before, in fact, thanks to the "pg_sleep(1.0)" added by
commit 957d08c81f9cc277). That leaves a sizable window in which an
autovacuum-triggered write of the stats files would present a state in
which the indexscan on tenk2 appears to have been done, but none of the
write updates performed by the test have been. This is evidently the
explanation for intermittent failures seen by me and on buildfarm member
mandrill.
To fix, we should check separately for both the tenk2 seqscan and indexscan
counts, since those might be reported by different processes that could be
delayed arbitrarily on an overloaded test machine. And we need to check
for at least one update-related count. If we ever allow parallel workers
to do writes, this will get even more complicated ... but in view of all
the other hard problems that will entail, I don't feel a need to solve this
one today.
Per research by Rahila Syed and myself; part of this patch is Rahila's.
2016-03-04 22:20:44 +01:00
|
|
|
updated1 bool;
|
|
|
|
updated2 bool;
|
|
|
|
updated3 bool;
|
2017-05-15 05:33:03 +02:00
|
|
|
updated4 bool;
|
2007-02-07 19:34:56 +01:00
|
|
|
begin
|
|
|
|
-- we don't want to wait forever; loop will exit after 30 seconds
|
|
|
|
for i in 1 .. 300 loop
|
|
|
|
|
Make stats regression test robust in the face of parallel query.
Historically, the wait_for_stats() function in this test has simply checked
for a report of an indexscan on tenk2, corresponding to the last command
issued before we expect stats updates to appear. However, with parallel
query that indexscan could be done by a parallel worker that will emit
its stats counters to the collector before the session's main backend does
(a full second before, in fact, thanks to the "pg_sleep(1.0)" added by
commit 957d08c81f9cc277). That leaves a sizable window in which an
autovacuum-triggered write of the stats files would present a state in
which the indexscan on tenk2 appears to have been done, but none of the
write updates performed by the test have been. This is evidently the
explanation for intermittent failures seen by me and on buildfarm member
mandrill.
To fix, we should check separately for both the tenk2 seqscan and indexscan
counts, since those might be reported by different processes that could be
delayed arbitrarily on an overloaded test machine. And we need to check
for at least one update-related count. If we ever allow parallel workers
to do writes, this will get even more complicated ... but in view of all
the other hard problems that will entail, I don't feel a need to solve this
one today.
Per research by Rahila Syed and myself; part of this patch is Rahila's.
2016-03-04 22:20:44 +01:00
|
|
|
-- With parallel query, the seqscan and indexscan on tenk2 might be done
|
|
|
|
-- in parallel worker processes, which will send their stats counters
|
|
|
|
-- asynchronously to what our own session does. So we must check for
|
|
|
|
-- those counts to be registered separately from the update counts.
|
|
|
|
|
|
|
|
-- check to see if seqscan has been sensed
|
|
|
|
SELECT (st.seq_scan >= pr.seq_scan + 1) INTO updated1
|
|
|
|
FROM pg_stat_user_tables AS st, pg_class AS cl, prevstats AS pr
|
|
|
|
WHERE st.relname='tenk2' AND cl.relname='tenk2';
|
|
|
|
|
2007-02-08 00:11:30 +01:00
|
|
|
-- check to see if indexscan has been sensed
|
Make stats regression test robust in the face of parallel query.
Historically, the wait_for_stats() function in this test has simply checked
for a report of an indexscan on tenk2, corresponding to the last command
issued before we expect stats updates to appear. However, with parallel
query that indexscan could be done by a parallel worker that will emit
its stats counters to the collector before the session's main backend does
(a full second before, in fact, thanks to the "pg_sleep(1.0)" added by
commit 957d08c81f9cc277). That leaves a sizable window in which an
autovacuum-triggered write of the stats files would present a state in
which the indexscan on tenk2 appears to have been done, but none of the
write updates performed by the test have been. This is evidently the
explanation for intermittent failures seen by me and on buildfarm member
mandrill.
To fix, we should check separately for both the tenk2 seqscan and indexscan
counts, since those might be reported by different processes that could be
delayed arbitrarily on an overloaded test machine. And we need to check
for at least one update-related count. If we ever allow parallel workers
to do writes, this will get even more complicated ... but in view of all
the other hard problems that will entail, I don't feel a need to solve this
one today.
Per research by Rahila Syed and myself; part of this patch is Rahila's.
2016-03-04 22:20:44 +01:00
|
|
|
SELECT (st.idx_scan >= pr.idx_scan + 1) INTO updated2
|
2007-02-08 00:11:30 +01:00
|
|
|
FROM pg_stat_user_tables AS st, pg_class AS cl, prevstats AS pr
|
|
|
|
WHERE st.relname='tenk2' AND cl.relname='tenk2';
|
2007-02-07 19:34:56 +01:00
|
|
|
|
2017-05-15 03:39:10 +02:00
|
|
|
-- check to see if all updates have been sensed
|
Make stats regression test robust in the face of parallel query.
Historically, the wait_for_stats() function in this test has simply checked
for a report of an indexscan on tenk2, corresponding to the last command
issued before we expect stats updates to appear. However, with parallel
query that indexscan could be done by a parallel worker that will emit
its stats counters to the collector before the session's main backend does
(a full second before, in fact, thanks to the "pg_sleep(1.0)" added by
commit 957d08c81f9cc277). That leaves a sizable window in which an
autovacuum-triggered write of the stats files would present a state in
which the indexscan on tenk2 appears to have been done, but none of the
write updates performed by the test have been. This is evidently the
explanation for intermittent failures seen by me and on buildfarm member
mandrill.
To fix, we should check separately for both the tenk2 seqscan and indexscan
counts, since those might be reported by different processes that could be
delayed arbitrarily on an overloaded test machine. And we need to check
for at least one update-related count. If we ever allow parallel workers
to do writes, this will get even more complicated ... but in view of all
the other hard problems that will entail, I don't feel a need to solve this
one today.
Per research by Rahila Syed and myself; part of this patch is Rahila's.
2016-03-04 22:20:44 +01:00
|
|
|
SELECT (n_tup_ins > 0) INTO updated3
|
2017-05-15 03:39:10 +02:00
|
|
|
FROM pg_stat_user_tables WHERE relname='trunc_stats_test4';
|
Make stats regression test robust in the face of parallel query.
Historically, the wait_for_stats() function in this test has simply checked
for a report of an indexscan on tenk2, corresponding to the last command
issued before we expect stats updates to appear. However, with parallel
query that indexscan could be done by a parallel worker that will emit
its stats counters to the collector before the session's main backend does
(a full second before, in fact, thanks to the "pg_sleep(1.0)" added by
commit 957d08c81f9cc277). That leaves a sizable window in which an
autovacuum-triggered write of the stats files would present a state in
which the indexscan on tenk2 appears to have been done, but none of the
write updates performed by the test have been. This is evidently the
explanation for intermittent failures seen by me and on buildfarm member
mandrill.
To fix, we should check separately for both the tenk2 seqscan and indexscan
counts, since those might be reported by different processes that could be
delayed arbitrarily on an overloaded test machine. And we need to check
for at least one update-related count. If we ever allow parallel workers
to do writes, this will get even more complicated ... but in view of all
the other hard problems that will entail, I don't feel a need to solve this
one today.
Per research by Rahila Syed and myself; part of this patch is Rahila's.
2016-03-04 22:20:44 +01:00
|
|
|
|
2017-05-15 05:33:03 +02:00
|
|
|
-- We must also check explicitly that pg_stat_get_snapshot_timestamp has
|
|
|
|
-- advanced, because that comes from the global stats file which might
|
|
|
|
-- be older than the per-DB stats file we got the other values from.
|
|
|
|
SELECT (pr.snap_ts < pg_stat_get_snapshot_timestamp()) INTO updated4
|
|
|
|
FROM prevstats AS pr;
|
|
|
|
|
|
|
|
exit when updated1 and updated2 and updated3 and updated4;
|
2007-02-07 19:34:56 +01:00
|
|
|
|
|
|
|
-- wait a little
|
2017-05-13 15:42:12 +02:00
|
|
|
perform pg_sleep_for('100 milliseconds');
|
2007-02-07 19:34:56 +01:00
|
|
|
|
2007-02-08 00:11:30 +01:00
|
|
|
-- reset stats snapshot so we can test again
|
|
|
|
perform pg_stat_clear_snapshot();
|
|
|
|
|
2007-02-07 19:34:56 +01:00
|
|
|
end loop;
|
|
|
|
|
|
|
|
-- report time waited in postmaster log (where it won't change test output)
|
|
|
|
raise log 'wait_for_stats delayed % seconds',
|
|
|
|
extract(epoch from clock_timestamp() - start_time);
|
|
|
|
end
|
|
|
|
$$ language plpgsql;
|
2015-02-20 16:10:01 +01:00
|
|
|
-- test effects of TRUNCATE on n_live_tup/n_dead_tup counters
|
|
|
|
CREATE TABLE trunc_stats_test(id serial);
|
2018-12-07 18:11:59 +01:00
|
|
|
CREATE TABLE trunc_stats_test1(id serial, stuff text);
|
2015-02-20 16:10:01 +01:00
|
|
|
CREATE TABLE trunc_stats_test2(id serial);
|
2018-12-07 18:11:59 +01:00
|
|
|
CREATE TABLE trunc_stats_test3(id serial, stuff text);
|
2015-02-20 16:10:01 +01:00
|
|
|
CREATE TABLE trunc_stats_test4(id serial);
|
|
|
|
-- check that n_live_tup is reset to 0 after truncate
|
|
|
|
INSERT INTO trunc_stats_test DEFAULT VALUES;
|
|
|
|
INSERT INTO trunc_stats_test DEFAULT VALUES;
|
|
|
|
INSERT INTO trunc_stats_test DEFAULT VALUES;
|
|
|
|
TRUNCATE trunc_stats_test;
|
|
|
|
-- test involving a truncate in a transaction; 4 ins but only 1 live
|
|
|
|
INSERT INTO trunc_stats_test1 DEFAULT VALUES;
|
|
|
|
INSERT INTO trunc_stats_test1 DEFAULT VALUES;
|
|
|
|
INSERT INTO trunc_stats_test1 DEFAULT VALUES;
|
|
|
|
UPDATE trunc_stats_test1 SET id = id + 10 WHERE id IN (1, 2);
|
|
|
|
DELETE FROM trunc_stats_test1 WHERE id = 3;
|
|
|
|
BEGIN;
|
|
|
|
UPDATE trunc_stats_test1 SET id = id + 100;
|
|
|
|
TRUNCATE trunc_stats_test1;
|
|
|
|
INSERT INTO trunc_stats_test1 DEFAULT VALUES;
|
|
|
|
COMMIT;
|
|
|
|
-- use a savepoint: 1 insert, 1 live
|
|
|
|
BEGIN;
|
|
|
|
INSERT INTO trunc_stats_test2 DEFAULT VALUES;
|
|
|
|
INSERT INTO trunc_stats_test2 DEFAULT VALUES;
|
|
|
|
SAVEPOINT p1;
|
|
|
|
INSERT INTO trunc_stats_test2 DEFAULT VALUES;
|
|
|
|
TRUNCATE trunc_stats_test2;
|
|
|
|
INSERT INTO trunc_stats_test2 DEFAULT VALUES;
|
|
|
|
RELEASE SAVEPOINT p1;
|
|
|
|
COMMIT;
|
|
|
|
-- rollback a savepoint: this should count 4 inserts and have 2
|
|
|
|
-- live tuples after commit (and 2 dead ones due to aborted subxact)
|
|
|
|
BEGIN;
|
|
|
|
INSERT INTO trunc_stats_test3 DEFAULT VALUES;
|
|
|
|
INSERT INTO trunc_stats_test3 DEFAULT VALUES;
|
|
|
|
SAVEPOINT p1;
|
|
|
|
INSERT INTO trunc_stats_test3 DEFAULT VALUES;
|
|
|
|
INSERT INTO trunc_stats_test3 DEFAULT VALUES;
|
|
|
|
TRUNCATE trunc_stats_test3;
|
|
|
|
INSERT INTO trunc_stats_test3 DEFAULT VALUES;
|
|
|
|
ROLLBACK TO SAVEPOINT p1;
|
|
|
|
COMMIT;
|
|
|
|
-- rollback a truncate: this should count 2 inserts and produce 2 dead tuples
|
|
|
|
BEGIN;
|
|
|
|
INSERT INTO trunc_stats_test4 DEFAULT VALUES;
|
|
|
|
INSERT INTO trunc_stats_test4 DEFAULT VALUES;
|
|
|
|
TRUNCATE trunc_stats_test4;
|
|
|
|
INSERT INTO trunc_stats_test4 DEFAULT VALUES;
|
|
|
|
ROLLBACK;
|
2007-02-08 00:11:30 +01:00
|
|
|
-- do a seqscan
|
|
|
|
SELECT count(*) FROM tenk2;
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
10000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
-- do an indexscan
|
Allow bitmap scans to operate as index-only scans when possible.
If we don't have to return any columns from heap tuples, and there's
no need to recheck qual conditions, and the heap page is all-visible,
then we can skip fetching the heap page altogether.
Skip prefetching pages too, when possible, on the assumption that the
recheck flag will remain the same from one page to the next. While that
assumption is hardly bulletproof, it seems like a good bet most of the
time, and better than prefetching pages we don't need.
This commit installs the executor infrastructure, but doesn't change
any planner cost estimates, thus possibly causing bitmap scans to
not be chosen in cases where this change renders them the best choice.
I (tgl) am not entirely convinced that we need to account for this
behavior in the planner, because I think typically the bitmap scan would
get chosen anyway if it's the best bet. In any case the submitted patch
took way too many shortcuts, resulting in too many clearly-bad choices,
to be committable.
Alexander Kuzmenkov, reviewed by Alexey Chernyshov, and whacked around
rather heavily by me.
Discussion: https://postgr.es/m/239a8955-c0fc-f506-026d-c837e86c827b@postgrespro.ru
2017-11-01 22:38:12 +01:00
|
|
|
-- make sure it is not a bitmap scan, which might skip fetching heap tuples
|
|
|
|
SET enable_bitmapscan TO off;
|
2007-02-08 00:11:30 +01:00
|
|
|
SELECT count(*) FROM tenk2 WHERE unique1 = 1;
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
1
|
|
|
|
(1 row)
|
|
|
|
|
Allow bitmap scans to operate as index-only scans when possible.
If we don't have to return any columns from heap tuples, and there's
no need to recheck qual conditions, and the heap page is all-visible,
then we can skip fetching the heap page altogether.
Skip prefetching pages too, when possible, on the assumption that the
recheck flag will remain the same from one page to the next. While that
assumption is hardly bulletproof, it seems like a good bet most of the
time, and better than prefetching pages we don't need.
This commit installs the executor infrastructure, but doesn't change
any planner cost estimates, thus possibly causing bitmap scans to
not be chosen in cases where this change renders them the best choice.
I (tgl) am not entirely convinced that we need to account for this
behavior in the planner, because I think typically the bitmap scan would
get chosen anyway if it's the best bet. In any case the submitted patch
took way too many shortcuts, resulting in too many clearly-bad choices,
to be committable.
Alexander Kuzmenkov, reviewed by Alexey Chernyshov, and whacked around
rather heavily by me.
Discussion: https://postgr.es/m/239a8955-c0fc-f506-026d-c837e86c827b@postgrespro.ru
2017-11-01 22:38:12 +01:00
|
|
|
RESET enable_bitmapscan;
|
2017-05-13 15:42:12 +02:00
|
|
|
-- We can't just call wait_for_stats() at this point, because we only
|
|
|
|
-- transmit stats when the session goes idle, and we probably didn't
|
|
|
|
-- transmit the last couple of counts yet thanks to the rate-limiting logic
|
|
|
|
-- in pgstat_report_stat(). But instead of waiting for the rate limiter's
|
|
|
|
-- timeout to elapse, let's just start a new session. The old one will
|
|
|
|
-- then send its stats before dying.
|
|
|
|
\c -
|
2007-02-08 00:11:30 +01:00
|
|
|
-- wait for stats collector to update
|
2007-02-07 19:34:56 +01:00
|
|
|
SELECT wait_for_stats();
|
|
|
|
wait_for_stats
|
|
|
|
----------------
|
2005-07-23 16:18:57 +02:00
|
|
|
|
2003-09-13 18:44:49 +02:00
|
|
|
(1 row)
|
|
|
|
|
|
|
|
-- check effects
|
2015-02-20 16:10:01 +01:00
|
|
|
SELECT relname, n_tup_ins, n_tup_upd, n_tup_del, n_live_tup, n_dead_tup
|
|
|
|
FROM pg_stat_user_tables
|
|
|
|
WHERE relname like 'trunc_stats_test%' order by relname;
|
|
|
|
relname | n_tup_ins | n_tup_upd | n_tup_del | n_live_tup | n_dead_tup
|
|
|
|
-------------------+-----------+-----------+-----------+------------+------------
|
|
|
|
trunc_stats_test | 3 | 0 | 0 | 0 | 0
|
|
|
|
trunc_stats_test1 | 4 | 2 | 1 | 1 | 0
|
|
|
|
trunc_stats_test2 | 1 | 0 | 0 | 1 | 0
|
|
|
|
trunc_stats_test3 | 4 | 0 | 0 | 2 | 2
|
|
|
|
trunc_stats_test4 | 2 | 0 | 0 | 0 | 2
|
|
|
|
(5 rows)
|
|
|
|
|
2003-09-13 18:44:49 +02:00
|
|
|
SELECT st.seq_scan >= pr.seq_scan + 1,
|
|
|
|
st.seq_tup_read >= pr.seq_tup_read + cl.reltuples,
|
|
|
|
st.idx_scan >= pr.idx_scan + 1,
|
|
|
|
st.idx_tup_fetch >= pr.idx_tup_fetch + 1
|
|
|
|
FROM pg_stat_user_tables AS st, pg_class AS cl, prevstats AS pr
|
|
|
|
WHERE st.relname='tenk2' AND cl.relname='tenk2';
|
2003-11-01 04:18:20 +01:00
|
|
|
?column? | ?column? | ?column? | ?column?
|
2003-09-13 18:44:49 +02:00
|
|
|
----------+----------+----------+----------
|
|
|
|
t | t | t | t
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
SELECT st.heap_blks_read + st.heap_blks_hit >= pr.heap_blks + cl.relpages,
|
|
|
|
st.idx_blks_read + st.idx_blks_hit >= pr.idx_blks + 1
|
|
|
|
FROM pg_statio_user_tables AS st, pg_class AS cl, prevstats AS pr
|
|
|
|
WHERE st.relname='tenk2' AND cl.relname='tenk2';
|
2003-11-01 04:18:20 +01:00
|
|
|
?column? | ?column?
|
2003-09-13 18:44:49 +02:00
|
|
|
----------+----------
|
|
|
|
t | t
|
|
|
|
(1 row)
|
|
|
|
|
2015-02-20 03:36:50 +01:00
|
|
|
SELECT pr.snap_ts < pg_stat_get_snapshot_timestamp() as snapshot_newer
|
|
|
|
FROM prevstats AS pr;
|
|
|
|
snapshot_newer
|
|
|
|
----------------
|
|
|
|
t
|
|
|
|
(1 row)
|
|
|
|
|
2015-02-20 16:10:01 +01:00
|
|
|
DROP TABLE trunc_stats_test, trunc_stats_test1, trunc_stats_test2, trunc_stats_test3, trunc_stats_test4;
|
2017-05-13 15:42:12 +02:00
|
|
|
DROP TABLE prevstats;
|
2003-09-13 18:44:49 +02:00
|
|
|
-- End of Stats Test
|