Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
--
|
|
|
|
-- exercises for the hash join code
|
|
|
|
--
|
|
|
|
begin;
|
|
|
|
set local min_parallel_table_scan_size = 0;
|
|
|
|
set local parallel_setup_cost = 0;
|
2019-07-09 08:11:01 +02:00
|
|
|
set local enable_hashjoin = on;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
-- Extract bucket and batch counts from an explain analyze plan. In
|
|
|
|
-- general we can't make assertions about how many batches (or
|
|
|
|
-- buckets) will be required because it can vary, but we can in some
|
|
|
|
-- special cases and we can check for growth.
|
|
|
|
create or replace function find_hash(node json)
|
|
|
|
returns json language plpgsql
|
|
|
|
as
|
|
|
|
$$
|
|
|
|
declare
|
|
|
|
x json;
|
|
|
|
child json;
|
|
|
|
begin
|
|
|
|
if node->>'Node Type' = 'Hash' then
|
|
|
|
return node;
|
|
|
|
else
|
|
|
|
for child in select json_array_elements(node->'Plans')
|
|
|
|
loop
|
|
|
|
x := find_hash(child);
|
|
|
|
if x is not null then
|
|
|
|
return x;
|
|
|
|
end if;
|
|
|
|
end loop;
|
|
|
|
return null;
|
|
|
|
end if;
|
|
|
|
end;
|
|
|
|
$$;
|
|
|
|
create or replace function hash_join_batches(query text)
|
|
|
|
returns table (original int, final int) language plpgsql
|
|
|
|
as
|
|
|
|
$$
|
|
|
|
declare
|
|
|
|
whole_plan json;
|
|
|
|
hash_node json;
|
|
|
|
begin
|
|
|
|
for whole_plan in
|
|
|
|
execute 'explain (analyze, format ''json'') ' || query
|
|
|
|
loop
|
|
|
|
hash_node := find_hash(json_extract_path(whole_plan, '0', 'Plan'));
|
|
|
|
original := hash_node->>'Original Hash Batches';
|
|
|
|
final := hash_node->>'Hash Batches';
|
|
|
|
return next;
|
|
|
|
end loop;
|
|
|
|
end;
|
|
|
|
$$;
|
|
|
|
-- Make a simple relation with well distributed keys and correctly
|
|
|
|
-- estimated size.
|
|
|
|
create table simple as
|
|
|
|
select generate_series(1, 20000) AS id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
|
|
|
|
alter table simple set (parallel_workers = 2);
|
|
|
|
analyze simple;
|
|
|
|
-- Make a relation whose size we will under-estimate. We want stats
|
|
|
|
-- to say 1000 rows, but actually there are 20,000 rows.
|
|
|
|
create table bigger_than_it_looks as
|
|
|
|
select generate_series(1, 20000) as id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
|
|
|
|
alter table bigger_than_it_looks set (autovacuum_enabled = 'false');
|
|
|
|
alter table bigger_than_it_looks set (parallel_workers = 2);
|
|
|
|
analyze bigger_than_it_looks;
|
|
|
|
update pg_class set reltuples = 1000 where relname = 'bigger_than_it_looks';
|
|
|
|
-- Make a relation whose size we underestimate and that also has a
|
|
|
|
-- kind of skew that breaks our batching scheme. We want stats to say
|
|
|
|
-- 2 rows, but actually there are 20,000 rows with the same key.
|
|
|
|
create table extremely_skewed (id int, t text);
|
|
|
|
alter table extremely_skewed set (autovacuum_enabled = 'false');
|
|
|
|
alter table extremely_skewed set (parallel_workers = 2);
|
|
|
|
analyze extremely_skewed;
|
|
|
|
insert into extremely_skewed
|
|
|
|
select 42 as id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
|
|
|
|
from generate_series(1, 20000);
|
|
|
|
update pg_class
|
|
|
|
set reltuples = 2, relpages = pg_relation_size('extremely_skewed') / 8192
|
|
|
|
where relname = 'extremely_skewed';
|
|
|
|
-- Make a relation with a couple of enormous tuples.
|
|
|
|
create table wide as select generate_series(1, 2) as id, rpad('', 320000, 'x') as t;
|
|
|
|
alter table wide set (parallel_workers = 2);
|
|
|
|
-- The "optimal" case: the hash table fits in memory; we plan for 1
|
|
|
|
-- batch, we stick to that number, and peak memory usage stays within
|
|
|
|
-- our work_mem budget
|
|
|
|
-- non-parallel
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 0;
|
|
|
|
set local work_mem = '4MB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
----------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on simple s
|
|
|
|
(6 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
f | f
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- parallel with parallel-oblivious hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
set local work_mem = '4MB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local enable_parallel_hash = off;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
-------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on simple s
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
f | f
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- parallel with parallel-aware hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
set local work_mem = '4MB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local enable_parallel_hash = on;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
-------------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Parallel Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on simple s
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
f | f
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- The "good" case: batches required, but we plan the right number; we
|
|
|
|
-- plan for some number of batches, and we stick to that number, and
|
|
|
|
-- peak memory usage says within our work_mem budget
|
|
|
|
-- non-parallel
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 0;
|
|
|
|
set local work_mem = '128kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
----------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on simple s
|
|
|
|
(6 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
t | f
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- parallel with parallel-oblivious hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
set local work_mem = '128kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local enable_parallel_hash = off;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
-------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on simple s
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
t | f
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- parallel with parallel-aware hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
set local work_mem = '192kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local enable_parallel_hash = on;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
-------------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Parallel Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on simple s
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
t | f
|
|
|
|
(1 row)
|
|
|
|
|
Parallel Hash Full Join.
Full and right outer joins were not supported in the initial
implementation of Parallel Hash Join because of deadlock hazards (see
discussion). Therefore FULL JOIN inhibited parallelism, as the other
join strategies can't do that in parallel either.
Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on
the inner side of one batch's hash table. For now, sidestep the
deadlock problem by terminating parallelism there. The last process to
arrive at that phase emits the unmatched tuples, while others detach and
are free to go and work on other batches, if there are any, but
otherwise they finish the join early.
That unfairness is considered acceptable for now, because it's better
than no parallelism at all. The build and probe phases are run in
parallel, and the new scan-for-unmatched phase, while serial, is usually
applied to the smaller of the two relations and is either limited by
some multiple of work_mem, or it's too big and is partitioned into
batches and then the situation is improved by batch-level parallelism.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Thomas Munro <thomas.munro@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com
2023-03-31 00:01:51 +02:00
|
|
|
-- parallel full multi-batch hash join
|
|
|
|
select count(*) from simple r full outer join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
rollback to settings;
|
|
|
|
-- The "bad" case: during execution we need to increase number of
|
|
|
|
-- batches; in this case we plan for 1 batch, and increase at least a
|
|
|
|
-- couple of times, and peak memory usage stays within our work_mem
|
|
|
|
-- budget
|
|
|
|
-- non-parallel
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 0;
|
|
|
|
set local work_mem = '128kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id);
|
|
|
|
QUERY PLAN
|
|
|
|
------------------------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on bigger_than_it_looks s
|
|
|
|
(6 rows)
|
|
|
|
|
|
|
|
select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
f | t
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- parallel with parallel-oblivious hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
set local work_mem = '128kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local enable_parallel_hash = off;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join bigger_than_it_looks s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
------------------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on bigger_than_it_looks s
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join bigger_than_it_looks s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join bigger_than_it_looks s using (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
f | t
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- parallel with parallel-aware hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 1;
|
|
|
|
set local work_mem = '192kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local enable_parallel_hash = on;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join bigger_than_it_looks s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
---------------------------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 1
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Parallel Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on bigger_than_it_looks s
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join bigger_than_it_looks s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select original > 1 as initially_multibatch, final > original as increased_batches
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join bigger_than_it_looks s using (id);
|
|
|
|
$$);
|
|
|
|
initially_multibatch | increased_batches
|
|
|
|
----------------------+-------------------
|
|
|
|
f | t
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- The "ugly" case: increasing the number of batches during execution
|
|
|
|
-- doesn't help, so stop trying to fit in work_mem and hope for the
|
|
|
|
-- best; in this case we plan for 1 batch, increases just once and
|
|
|
|
-- then stop increasing because that didn't help at all, so we blow
|
|
|
|
-- right through the work_mem budget and hope for the best...
|
|
|
|
-- non-parallel
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 0;
|
|
|
|
set local work_mem = '128kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
--------------------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on extremely_skewed s
|
|
|
|
(6 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select * from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
$$);
|
|
|
|
original | final
|
|
|
|
----------+-------
|
|
|
|
1 | 2
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- parallel with parallel-oblivious hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
set local work_mem = '128kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local enable_parallel_hash = off;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
--------------------------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on extremely_skewed s
|
|
|
|
(8 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select * from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
$$);
|
|
|
|
original | final
|
|
|
|
----------+-------
|
|
|
|
1 | 2
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- parallel with parallel-aware hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 1;
|
|
|
|
set local work_mem = '128kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local enable_parallel_hash = on;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
-----------------------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 1
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Parallel Hash Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on extremely_skewed s
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select * from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join extremely_skewed s using (id);
|
|
|
|
$$);
|
|
|
|
original | final
|
|
|
|
----------+-------
|
|
|
|
1 | 4
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- A couple of other hash join tests unrelated to work_mem management.
|
|
|
|
-- Check that EXPLAIN ANALYZE has data even if the leader doesn't participate
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
set local work_mem = '4MB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set local hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local parallel_leader_participation = off;
|
|
|
|
select * from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from simple r join simple s using (id);
|
|
|
|
$$);
|
|
|
|
original | final
|
|
|
|
----------+-------
|
|
|
|
1 | 1
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- Exercise rescans. We'll turn off parallel_leader_participation so
|
|
|
|
-- that we can check that instrumentation comes back correctly.
|
|
|
|
create table join_foo as select generate_series(1, 3) as id, 'xxxxx'::text as t;
|
|
|
|
alter table join_foo set (parallel_workers = 0);
|
|
|
|
create table join_bar as select generate_series(1, 10000) as id, 'xxxxx'::text as t;
|
|
|
|
alter table join_bar set (parallel_workers = 2);
|
|
|
|
-- multi-batch with rescan, parallel-oblivious
|
|
|
|
savepoint settings;
|
|
|
|
set enable_parallel_hash = off;
|
|
|
|
set parallel_leader_participation = off;
|
|
|
|
set min_parallel_table_scan_size = 0;
|
|
|
|
set parallel_setup_cost = 0;
|
|
|
|
set parallel_tuple_cost = 0;
|
|
|
|
set max_parallel_workers_per_gather = 2;
|
|
|
|
set enable_material = off;
|
|
|
|
set enable_mergejoin = off;
|
|
|
|
set work_mem = '64kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
QUERY PLAN
|
|
|
|
------------------------------------------------------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Nested Loop Left Join
|
|
|
|
Join Filter: ((join_foo.id < (b1.id + 1)) AND (join_foo.id > (b1.id - 1)))
|
|
|
|
-> Seq Scan on join_foo
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (b1.id = b2.id)
|
|
|
|
-> Parallel Seq Scan on join_bar b1
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on join_bar b2
|
|
|
|
(11 rows)
|
|
|
|
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
3
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select final > 1 as multibatch
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
$$);
|
|
|
|
multibatch
|
|
|
|
------------
|
|
|
|
t
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- single-batch with rescan, parallel-oblivious
|
|
|
|
savepoint settings;
|
|
|
|
set enable_parallel_hash = off;
|
|
|
|
set parallel_leader_participation = off;
|
|
|
|
set min_parallel_table_scan_size = 0;
|
|
|
|
set parallel_setup_cost = 0;
|
|
|
|
set parallel_tuple_cost = 0;
|
|
|
|
set max_parallel_workers_per_gather = 2;
|
|
|
|
set enable_material = off;
|
|
|
|
set enable_mergejoin = off;
|
|
|
|
set work_mem = '4MB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
QUERY PLAN
|
|
|
|
------------------------------------------------------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Nested Loop Left Join
|
|
|
|
Join Filter: ((join_foo.id < (b1.id + 1)) AND (join_foo.id > (b1.id - 1)))
|
|
|
|
-> Seq Scan on join_foo
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (b1.id = b2.id)
|
|
|
|
-> Parallel Seq Scan on join_bar b1
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on join_bar b2
|
|
|
|
(11 rows)
|
|
|
|
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
3
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select final > 1 as multibatch
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
$$);
|
|
|
|
multibatch
|
|
|
|
------------
|
|
|
|
f
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- multi-batch with rescan, parallel-aware
|
|
|
|
savepoint settings;
|
|
|
|
set enable_parallel_hash = on;
|
|
|
|
set parallel_leader_participation = off;
|
|
|
|
set min_parallel_table_scan_size = 0;
|
|
|
|
set parallel_setup_cost = 0;
|
|
|
|
set parallel_tuple_cost = 0;
|
|
|
|
set max_parallel_workers_per_gather = 2;
|
|
|
|
set enable_material = off;
|
|
|
|
set enable_mergejoin = off;
|
|
|
|
set work_mem = '64kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
QUERY PLAN
|
|
|
|
------------------------------------------------------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Nested Loop Left Join
|
|
|
|
Join Filter: ((join_foo.id < (b1.id + 1)) AND (join_foo.id > (b1.id - 1)))
|
|
|
|
-> Seq Scan on join_foo
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Parallel Hash Join
|
|
|
|
Hash Cond: (b1.id = b2.id)
|
|
|
|
-> Parallel Seq Scan on join_bar b1
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on join_bar b2
|
|
|
|
(11 rows)
|
|
|
|
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
3
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select final > 1 as multibatch
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
$$);
|
|
|
|
multibatch
|
|
|
|
------------
|
|
|
|
t
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- single-batch with rescan, parallel-aware
|
|
|
|
savepoint settings;
|
|
|
|
set enable_parallel_hash = on;
|
|
|
|
set parallel_leader_participation = off;
|
|
|
|
set min_parallel_table_scan_size = 0;
|
|
|
|
set parallel_setup_cost = 0;
|
|
|
|
set parallel_tuple_cost = 0;
|
|
|
|
set max_parallel_workers_per_gather = 2;
|
|
|
|
set enable_material = off;
|
|
|
|
set enable_mergejoin = off;
|
|
|
|
set work_mem = '4MB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
QUERY PLAN
|
|
|
|
------------------------------------------------------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Nested Loop Left Join
|
|
|
|
Join Filter: ((join_foo.id < (b1.id + 1)) AND (join_foo.id > (b1.id - 1)))
|
|
|
|
-> Seq Scan on join_foo
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Parallel Hash Join
|
|
|
|
Hash Cond: (b1.id = b2.id)
|
|
|
|
-> Parallel Seq Scan on join_bar b1
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on join_bar b2
|
|
|
|
(11 rows)
|
|
|
|
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
3
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select final > 1 as multibatch
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select count(*) from join_foo
|
|
|
|
left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss
|
|
|
|
on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1;
|
|
|
|
$$);
|
|
|
|
multibatch
|
|
|
|
------------
|
|
|
|
f
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- A full outer join where every record is matched.
|
|
|
|
-- non-parallel
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 0;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r full outer join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
----------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Hash Full Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on simple s
|
|
|
|
(6 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r full outer join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
Parallel Hash Full Join.
Full and right outer joins were not supported in the initial
implementation of Parallel Hash Join because of deadlock hazards (see
discussion). Therefore FULL JOIN inhibited parallelism, as the other
join strategies can't do that in parallel either.
Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on
the inner side of one batch's hash table. For now, sidestep the
deadlock problem by terminating parallelism there. The last process to
arrive at that phase emits the unmatched tuples, while others detach and
are free to go and work on other batches, if there are any, but
otherwise they finish the join early.
That unfairness is considered acceptable for now, because it's better
than no parallelism at all. The build and probe phases are run in
parallel, and the new scan-for-unmatched phase, while serial, is usually
applied to the smaller of the two relations and is either limited by
some multiple of work_mem, or it's too big and is partitioned into
batches and then the situation is improved by batch-level parallelism.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Thomas Munro <thomas.munro@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com
2023-03-31 00:01:51 +02:00
|
|
|
-- parallelism not possible with parallel-oblivious full hash join
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
savepoint settings;
|
Parallel Hash Full Join.
Full and right outer joins were not supported in the initial
implementation of Parallel Hash Join because of deadlock hazards (see
discussion). Therefore FULL JOIN inhibited parallelism, as the other
join strategies can't do that in parallel either.
Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on
the inner side of one batch's hash table. For now, sidestep the
deadlock problem by terminating parallelism there. The last process to
arrive at that phase emits the unmatched tuples, while others detach and
are free to go and work on other batches, if there are any, but
otherwise they finish the join early.
That unfairness is considered acceptable for now, because it's better
than no parallelism at all. The build and probe phases are run in
parallel, and the new scan-for-unmatched phase, while serial, is usually
applied to the smaller of the two relations and is either limited by
some multiple of work_mem, or it's too big and is partitioned into
batches and then the situation is improved by batch-level parallelism.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Thomas Munro <thomas.munro@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com
2023-03-31 00:01:51 +02:00
|
|
|
set enable_parallel_hash = off;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r full outer join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
----------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Hash Full Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Seq Scan on simple r
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on simple s
|
|
|
|
(6 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r full outer join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
Parallel Hash Full Join.
Full and right outer joins were not supported in the initial
implementation of Parallel Hash Join because of deadlock hazards (see
discussion). Therefore FULL JOIN inhibited parallelism, as the other
join strategies can't do that in parallel either.
Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on
the inner side of one batch's hash table. For now, sidestep the
deadlock problem by terminating parallelism there. The last process to
arrive at that phase emits the unmatched tuples, while others detach and
are free to go and work on other batches, if there are any, but
otherwise they finish the join early.
That unfairness is considered acceptable for now, because it's better
than no parallelism at all. The build and probe phases are run in
parallel, and the new scan-for-unmatched phase, while serial, is usually
applied to the smaller of the two relations and is either limited by
some multiple of work_mem, or it's too big and is partitioned into
batches and then the situation is improved by batch-level parallelism.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Thomas Munro <thomas.munro@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com
2023-03-31 00:01:51 +02:00
|
|
|
-- parallelism is possible with parallel-aware full hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r full outer join simple s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
-------------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Parallel Hash Full Join
|
|
|
|
Hash Cond: (r.id = s.id)
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on simple s
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r full outer join simple s using (id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
20000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
|
|
|
-- A full outer join where every record is not matched.
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
-- non-parallel
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 0;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r full outer join simple s on (r.id = 0 - s.id);
|
|
|
|
QUERY PLAN
|
|
|
|
----------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Hash Full Join
|
|
|
|
Hash Cond: ((0 - s.id) = r.id)
|
|
|
|
-> Seq Scan on simple s
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on simple r
|
|
|
|
(6 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r full outer join simple s on (r.id = 0 - s.id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
40000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
Parallel Hash Full Join.
Full and right outer joins were not supported in the initial
implementation of Parallel Hash Join because of deadlock hazards (see
discussion). Therefore FULL JOIN inhibited parallelism, as the other
join strategies can't do that in parallel either.
Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on
the inner side of one batch's hash table. For now, sidestep the
deadlock problem by terminating parallelism there. The last process to
arrive at that phase emits the unmatched tuples, while others detach and
are free to go and work on other batches, if there are any, but
otherwise they finish the join early.
That unfairness is considered acceptable for now, because it's better
than no parallelism at all. The build and probe phases are run in
parallel, and the new scan-for-unmatched phase, while serial, is usually
applied to the smaller of the two relations and is either limited by
some multiple of work_mem, or it's too big and is partitioned into
batches and then the situation is improved by batch-level parallelism.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Thomas Munro <thomas.munro@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com
2023-03-31 00:01:51 +02:00
|
|
|
-- parallelism not possible with parallel-oblivious full hash join
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
savepoint settings;
|
Parallel Hash Full Join.
Full and right outer joins were not supported in the initial
implementation of Parallel Hash Join because of deadlock hazards (see
discussion). Therefore FULL JOIN inhibited parallelism, as the other
join strategies can't do that in parallel either.
Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on
the inner side of one batch's hash table. For now, sidestep the
deadlock problem by terminating parallelism there. The last process to
arrive at that phase emits the unmatched tuples, while others detach and
are free to go and work on other batches, if there are any, but
otherwise they finish the join early.
That unfairness is considered acceptable for now, because it's better
than no parallelism at all. The build and probe phases are run in
parallel, and the new scan-for-unmatched phase, while serial, is usually
applied to the smaller of the two relations and is either limited by
some multiple of work_mem, or it's too big and is partitioned into
batches and then the situation is improved by batch-level parallelism.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Thomas Munro <thomas.munro@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com
2023-03-31 00:01:51 +02:00
|
|
|
set enable_parallel_hash = off;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r full outer join simple s on (r.id = 0 - s.id);
|
|
|
|
QUERY PLAN
|
|
|
|
----------------------------------------
|
|
|
|
Aggregate
|
|
|
|
-> Hash Full Join
|
|
|
|
Hash Cond: ((0 - s.id) = r.id)
|
|
|
|
-> Seq Scan on simple s
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on simple r
|
|
|
|
(6 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r full outer join simple s on (r.id = 0 - s.id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
40000
|
|
|
|
(1 row)
|
|
|
|
|
Parallel Hash Full Join.
Full and right outer joins were not supported in the initial
implementation of Parallel Hash Join because of deadlock hazards (see
discussion). Therefore FULL JOIN inhibited parallelism, as the other
join strategies can't do that in parallel either.
Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on
the inner side of one batch's hash table. For now, sidestep the
deadlock problem by terminating parallelism there. The last process to
arrive at that phase emits the unmatched tuples, while others detach and
are free to go and work on other batches, if there are any, but
otherwise they finish the join early.
That unfairness is considered acceptable for now, because it's better
than no parallelism at all. The build and probe phases are run in
parallel, and the new scan-for-unmatched phase, while serial, is usually
applied to the smaller of the two relations and is either limited by
some multiple of work_mem, or it's too big and is partitioned into
batches and then the situation is improved by batch-level parallelism.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Thomas Munro <thomas.munro@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com
2023-03-31 00:01:51 +02:00
|
|
|
rollback to settings;
|
|
|
|
-- parallelism is possible with parallel-aware full hash join
|
|
|
|
savepoint settings;
|
|
|
|
set local max_parallel_workers_per_gather = 2;
|
|
|
|
explain (costs off)
|
|
|
|
select count(*) from simple r full outer join simple s on (r.id = 0 - s.id);
|
|
|
|
QUERY PLAN
|
|
|
|
-------------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Parallel Hash Full Join
|
|
|
|
Hash Cond: ((0 - s.id) = r.id)
|
|
|
|
-> Parallel Seq Scan on simple s
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on simple r
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select count(*) from simple r full outer join simple s on (r.id = 0 - s.id);
|
|
|
|
count
|
|
|
|
-------
|
|
|
|
40000
|
|
|
|
(1 row)
|
|
|
|
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
rollback to settings;
|
|
|
|
-- exercise special code paths for huge tuples (note use of non-strict
|
|
|
|
-- expression and left join required to get the detoasted tuple into
|
|
|
|
-- the hash table)
|
|
|
|
-- parallel with parallel-aware hash join (hits ExecParallelHashLoadTuple and
|
|
|
|
-- sts_puttuple oversized tuple cases because it's multi-batch)
|
|
|
|
savepoint settings;
|
|
|
|
set max_parallel_workers_per_gather = 2;
|
|
|
|
set enable_parallel_hash = on;
|
|
|
|
set work_mem = '128kB';
|
2022-02-17 03:41:52 +01:00
|
|
|
set hash_mem_multiplier = 1.0;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
explain (costs off)
|
|
|
|
select length(max(s.t))
|
|
|
|
from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id);
|
|
|
|
QUERY PLAN
|
|
|
|
----------------------------------------------------------------
|
|
|
|
Finalize Aggregate
|
|
|
|
-> Gather
|
|
|
|
Workers Planned: 2
|
|
|
|
-> Partial Aggregate
|
|
|
|
-> Parallel Hash Left Join
|
|
|
|
Hash Cond: (wide.id = wide_1.id)
|
|
|
|
-> Parallel Seq Scan on wide
|
|
|
|
-> Parallel Hash
|
|
|
|
-> Parallel Seq Scan on wide wide_1
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select length(max(s.t))
|
|
|
|
from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id);
|
|
|
|
length
|
|
|
|
--------
|
|
|
|
320000
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
select final > 1 as multibatch
|
|
|
|
from hash_join_batches(
|
|
|
|
$$
|
|
|
|
select length(max(s.t))
|
|
|
|
from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id);
|
|
|
|
$$);
|
|
|
|
multibatch
|
|
|
|
------------
|
|
|
|
t
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
rollback to settings;
|
2023-04-14 00:52:58 +02:00
|
|
|
-- Hash join reuses the HOT status bit to indicate match status. This can only
|
|
|
|
-- be guaranteed to produce correct results if all the hash join tuple match
|
|
|
|
-- bits are reset before reuse. This is done upon loading them into the
|
|
|
|
-- hashtable.
|
|
|
|
SAVEPOINT settings;
|
|
|
|
SET enable_parallel_hash = on;
|
|
|
|
SET min_parallel_table_scan_size = 0;
|
|
|
|
SET parallel_setup_cost = 0;
|
|
|
|
SET parallel_tuple_cost = 0;
|
|
|
|
CREATE TABLE hjtest_matchbits_t1(id int);
|
|
|
|
CREATE TABLE hjtest_matchbits_t2(id int);
|
|
|
|
INSERT INTO hjtest_matchbits_t1 VALUES (1);
|
|
|
|
INSERT INTO hjtest_matchbits_t2 VALUES (2);
|
|
|
|
-- Update should create a HOT tuple. If this status bit isn't cleared, we won't
|
|
|
|
-- correctly emit the NULL-extended unmatching tuple in full hash join.
|
|
|
|
UPDATE hjtest_matchbits_t2 set id = 2;
|
2023-06-12 05:19:46 +02:00
|
|
|
SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id
|
|
|
|
ORDER BY t1.id;
|
2023-04-14 00:52:58 +02:00
|
|
|
id | id
|
|
|
|
----+----
|
|
|
|
1 |
|
|
|
|
| 2
|
|
|
|
(2 rows)
|
|
|
|
|
|
|
|
-- Test serial full hash join.
|
|
|
|
-- Resetting parallel_setup_cost should force a serial plan.
|
|
|
|
-- Just to be safe, however, set enable_parallel_hash to off, as parallel full
|
|
|
|
-- hash joins are only supported with shared hashtables.
|
|
|
|
RESET parallel_setup_cost;
|
|
|
|
SET enable_parallel_hash = off;
|
|
|
|
SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id;
|
|
|
|
id | id
|
|
|
|
----+----
|
|
|
|
1 |
|
|
|
|
| 2
|
|
|
|
(2 rows)
|
|
|
|
|
|
|
|
ROLLBACK TO settings;
|
Split up a couple of long-running regression test scripts.
The point of this change is to increase the potential for parallelism
while running the core regression tests. Most people these days are
using parallel testing modes on multi-core machines, so we might as
well try a bit harder to keep multiple cores busy. Hence, a test that
runs much longer than others in its parallel group is a candidate to
be sub-divided.
In this patch, create_index.sql and join.sql are split up.
I haven't changed the content of the tests in any way, just
moved them.
I moved create_index.sql's SP-GiST-related tests into a new script
create_index_spgist, and moved its btree multilevel page deletion test
over to the existing script btree_index. (btree_index is a more natural
home for that test, and it's shorter than others in its parallel group,
so this doesn't hurt total runtime of that group.) There might be
room for more aggressive splitting of create_index, but this is enough
to improve matters considerably.
Likewise, I moved join.sql's "exercises for the hash join code" into
a new file join_hash. Those exercises contributed three-quarters of
the script's runtime. Which might well be excessive ... but for the
moment, I'm satisfied with shoving them into a different parallel
group, where they can share runtime with the roughly-equally-lengthy
gist test.
(Note for anybody following along at home: there are interesting
interactions between the runtimes of create_index and anything running
in parallel with it, because the tests of CREATE INDEX CONCURRENTLY
in that file will repeatedly block waiting for concurrent transactions
to commit. As committed in this patch, create_index and
create_index_spgist have roughly equal runtimes, but that's mostly an
artifact of forced synchronization of the CONCURRENTLY tests; when run
serially, create_index is much faster. A followup patch will reduce
the runtime of create_index_spgist and thereby also create_index.)
Discussion: https://postgr.es/m/735.1554935715@sss.pgh.pa.us
2019-04-11 22:15:54 +02:00
|
|
|
rollback;
|
Fix representation of hash keys in Hash/HashJoin nodes.
In 5f32b29c1819 I changed the creation of HashState.hashkeys to
actually use HashState as the parent (instead of HashJoinState, which
was incorrect, as they were executed below HashState), to fix the
problem of hashkeys expressions otherwise relying on slot types
appropriate for HashJoinState, rather than HashState as would be
correct. That reliance was only introduced in 12, which is why it
previously worked to use HashJoinState as the parent (although I'd be
unsurprised if there were problematic cases).
Unfortunately that's not a sufficient solution, because before this
commit, the to-be-hashed expressions referenced inner/outer as
appropriate for the HashJoin, not Hash. That didn't have obvious bad
consequences, because the slots containing the tuples were put into
ecxt_innertuple when hashing a tuple for HashState (even though Hash
doesn't have an inner plan).
There are less common cases where this can cause visible problems
however (rather than just confusion when inspecting such executor
trees). E.g. "ERROR: bogus varno: 65000", when explaining queries
containing a HashJoin where the subsidiary Hash node's hash keys
reference a subplan. While normally hashkeys aren't displayed by
EXPLAIN, if one of those expressions references a subplan, that
subplan may be printed as part of the Hash node - which then failed
because an inner plan was referenced, and Hash doesn't have that.
It seems quite possible that there's other broken cases, too.
Fix the problem by properly splitting the expression for the HashJoin
and Hash nodes at plan time, and have them reference the proper
subsidiary node. While other workarounds are possible, fixing this
correctly seems easy enough. It was a pretty ugly hack to have
ExecInitHashJoin put the expression into the already initialized
HashState, in the first place.
I decided to not just split inner/outer hashkeys inside
make_hashjoin(), but also to separate out hashoperators and
hashcollations at plan time. Otherwise we would have ended up having
two very similar loops, one at plan time and the other during executor
startup. The work seems to more appropriately belong to plan time,
anyway.
Reported-By: Nikita Glukhov, Alexander Korotkov
Author: Andres Freund
Reviewed-By: Tom Lane, in an earlier version
Discussion: https://postgr.es/m/CAPpHfdvGVegF_TKKRiBrSmatJL2dR9uwFCuR+teQ_8tEXU8mxg@mail.gmail.com
Backpatch: 12-
2019-08-02 09:02:46 +02:00
|
|
|
-- Verify that hash key expressions reference the correct
|
|
|
|
-- nodes. Hashjoin's hashkeys need to reference its outer plan, Hash's
|
|
|
|
-- need to reference Hash's outer plan (which is below HashJoin's
|
|
|
|
-- inner plan). It's not trivial to verify that the references are
|
|
|
|
-- correct (we don't display the hashkeys themselves), but if the
|
|
|
|
-- hashkeys contain subplan references, those will be displayed. Force
|
|
|
|
-- subplans to appear just about everywhere.
|
|
|
|
--
|
|
|
|
-- Bug report:
|
|
|
|
-- https://www.postgresql.org/message-id/CAPpHfdvGVegF_TKKRiBrSmatJL2dR9uwFCuR%2BteQ_8tEXU8mxg%40mail.gmail.com
|
|
|
|
--
|
|
|
|
BEGIN;
|
|
|
|
SET LOCAL enable_sort = OFF; -- avoid mergejoins
|
|
|
|
SET LOCAL from_collapse_limit = 1; -- allows easy changing of join order
|
|
|
|
CREATE TABLE hjtest_1 (a text, b int, id int, c bool);
|
|
|
|
CREATE TABLE hjtest_2 (a bool, id int, b text, c int);
|
|
|
|
INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 2, 1, false); -- matches
|
|
|
|
INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 1, 2, false); -- fails id join condition
|
|
|
|
INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 20, 1, false); -- fails < 50
|
|
|
|
INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 1, 1, false); -- fails (SELECT hjtest_1.b * 5) = (SELECT hjtest_2.c*5)
|
|
|
|
INSERT INTO hjtest_2(a, id, b, c) VALUES (true, 1, 'another', 2); -- matches
|
|
|
|
INSERT INTO hjtest_2(a, id, b, c) VALUES (true, 3, 'another', 7); -- fails id join condition
|
|
|
|
INSERT INTO hjtest_2(a, id, b, c) VALUES (true, 1, 'another', 90); -- fails < 55
|
|
|
|
INSERT INTO hjtest_2(a, id, b, c) VALUES (true, 1, 'another', 3); -- fails (SELECT hjtest_1.b * 5) = (SELECT hjtest_2.c*5)
|
|
|
|
INSERT INTO hjtest_2(a, id, b, c) VALUES (true, 1, 'text', 1); -- fails hjtest_1.a <> hjtest_2.b;
|
|
|
|
EXPLAIN (COSTS OFF, VERBOSE)
|
|
|
|
SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2
|
|
|
|
FROM hjtest_1, hjtest_2
|
|
|
|
WHERE
|
|
|
|
hjtest_1.id = (SELECT 1 WHERE hjtest_2.id = 1)
|
|
|
|
AND (SELECT hjtest_1.b * 5) = (SELECT hjtest_2.c*5)
|
|
|
|
AND (SELECT hjtest_1.b * 5) < 50
|
|
|
|
AND (SELECT hjtest_2.c * 5) < 55
|
|
|
|
AND hjtest_1.a <> hjtest_2.b;
|
|
|
|
QUERY PLAN
|
|
|
|
------------------------------------------------------------------------------------------------
|
|
|
|
Hash Join
|
|
|
|
Output: hjtest_1.a, hjtest_2.a, (hjtest_1.tableoid)::regclass, (hjtest_2.tableoid)::regclass
|
|
|
|
Hash Cond: ((hjtest_1.id = (SubPlan 1)) AND ((SubPlan 2) = (SubPlan 3)))
|
|
|
|
Join Filter: (hjtest_1.a <> hjtest_2.b)
|
|
|
|
-> Seq Scan on public.hjtest_1
|
|
|
|
Output: hjtest_1.a, hjtest_1.tableoid, hjtest_1.id, hjtest_1.b
|
|
|
|
Filter: ((SubPlan 4) < 50)
|
|
|
|
SubPlan 4
|
|
|
|
-> Result
|
|
|
|
Output: (hjtest_1.b * 5)
|
|
|
|
-> Hash
|
|
|
|
Output: hjtest_2.a, hjtest_2.tableoid, hjtest_2.id, hjtest_2.c, hjtest_2.b
|
|
|
|
-> Seq Scan on public.hjtest_2
|
|
|
|
Output: hjtest_2.a, hjtest_2.tableoid, hjtest_2.id, hjtest_2.c, hjtest_2.b
|
|
|
|
Filter: ((SubPlan 5) < 55)
|
|
|
|
SubPlan 5
|
|
|
|
-> Result
|
|
|
|
Output: (hjtest_2.c * 5)
|
|
|
|
SubPlan 1
|
|
|
|
-> Result
|
|
|
|
Output: 1
|
|
|
|
One-Time Filter: (hjtest_2.id = 1)
|
|
|
|
SubPlan 3
|
|
|
|
-> Result
|
|
|
|
Output: (hjtest_2.c * 5)
|
|
|
|
SubPlan 2
|
|
|
|
-> Result
|
|
|
|
Output: (hjtest_1.b * 5)
|
|
|
|
(28 rows)
|
|
|
|
|
|
|
|
SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2
|
|
|
|
FROM hjtest_1, hjtest_2
|
|
|
|
WHERE
|
|
|
|
hjtest_1.id = (SELECT 1 WHERE hjtest_2.id = 1)
|
|
|
|
AND (SELECT hjtest_1.b * 5) = (SELECT hjtest_2.c*5)
|
|
|
|
AND (SELECT hjtest_1.b * 5) < 50
|
|
|
|
AND (SELECT hjtest_2.c * 5) < 55
|
|
|
|
AND hjtest_1.a <> hjtest_2.b;
|
|
|
|
a1 | a2 | t1 | t2
|
|
|
|
------+----+----------+----------
|
|
|
|
text | t | hjtest_1 | hjtest_2
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
EXPLAIN (COSTS OFF, VERBOSE)
|
|
|
|
SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2
|
|
|
|
FROM hjtest_2, hjtest_1
|
|
|
|
WHERE
|
|
|
|
hjtest_1.id = (SELECT 1 WHERE hjtest_2.id = 1)
|
|
|
|
AND (SELECT hjtest_1.b * 5) = (SELECT hjtest_2.c*5)
|
|
|
|
AND (SELECT hjtest_1.b * 5) < 50
|
|
|
|
AND (SELECT hjtest_2.c * 5) < 55
|
|
|
|
AND hjtest_1.a <> hjtest_2.b;
|
|
|
|
QUERY PLAN
|
|
|
|
------------------------------------------------------------------------------------------------
|
|
|
|
Hash Join
|
|
|
|
Output: hjtest_1.a, hjtest_2.a, (hjtest_1.tableoid)::regclass, (hjtest_2.tableoid)::regclass
|
|
|
|
Hash Cond: (((SubPlan 1) = hjtest_1.id) AND ((SubPlan 3) = (SubPlan 2)))
|
|
|
|
Join Filter: (hjtest_1.a <> hjtest_2.b)
|
|
|
|
-> Seq Scan on public.hjtest_2
|
|
|
|
Output: hjtest_2.a, hjtest_2.tableoid, hjtest_2.id, hjtest_2.c, hjtest_2.b
|
|
|
|
Filter: ((SubPlan 5) < 55)
|
|
|
|
SubPlan 5
|
|
|
|
-> Result
|
|
|
|
Output: (hjtest_2.c * 5)
|
|
|
|
-> Hash
|
|
|
|
Output: hjtest_1.a, hjtest_1.tableoid, hjtest_1.id, hjtest_1.b
|
|
|
|
-> Seq Scan on public.hjtest_1
|
|
|
|
Output: hjtest_1.a, hjtest_1.tableoid, hjtest_1.id, hjtest_1.b
|
|
|
|
Filter: ((SubPlan 4) < 50)
|
|
|
|
SubPlan 4
|
|
|
|
-> Result
|
|
|
|
Output: (hjtest_1.b * 5)
|
|
|
|
SubPlan 2
|
|
|
|
-> Result
|
|
|
|
Output: (hjtest_1.b * 5)
|
|
|
|
SubPlan 1
|
|
|
|
-> Result
|
|
|
|
Output: 1
|
|
|
|
One-Time Filter: (hjtest_2.id = 1)
|
|
|
|
SubPlan 3
|
|
|
|
-> Result
|
|
|
|
Output: (hjtest_2.c * 5)
|
|
|
|
(28 rows)
|
|
|
|
|
|
|
|
SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2
|
|
|
|
FROM hjtest_2, hjtest_1
|
|
|
|
WHERE
|
|
|
|
hjtest_1.id = (SELECT 1 WHERE hjtest_2.id = 1)
|
|
|
|
AND (SELECT hjtest_1.b * 5) = (SELECT hjtest_2.c*5)
|
|
|
|
AND (SELECT hjtest_1.b * 5) < 50
|
|
|
|
AND (SELECT hjtest_2.c * 5) < 55
|
|
|
|
AND hjtest_1.a <> hjtest_2.b;
|
|
|
|
a1 | a2 | t1 | t2
|
|
|
|
------+----+----------+----------
|
|
|
|
text | t | hjtest_1 | hjtest_2
|
|
|
|
(1 row)
|
|
|
|
|
|
|
|
ROLLBACK;
|
2023-06-20 23:47:36 +02:00
|
|
|
-- Verify that we behave sanely when the inner hash keys contain parameters
|
|
|
|
-- (that is, outer or lateral references). This situation has to defeat
|
|
|
|
-- re-use of the inner hash table across rescans.
|
|
|
|
begin;
|
|
|
|
set local enable_hashjoin = on;
|
|
|
|
explain (costs off)
|
|
|
|
select i8.q2, ss.* from
|
|
|
|
int8_tbl i8,
|
|
|
|
lateral (select t1.fivethous, i4.f1 from tenk1 t1 join int4_tbl i4
|
|
|
|
on t1.fivethous = i4.f1+i8.q2 order by 1,2) ss;
|
|
|
|
QUERY PLAN
|
|
|
|
-----------------------------------------------------------
|
|
|
|
Nested Loop
|
|
|
|
-> Seq Scan on int8_tbl i8
|
|
|
|
-> Sort
|
|
|
|
Sort Key: t1.fivethous, i4.f1
|
|
|
|
-> Hash Join
|
|
|
|
Hash Cond: (t1.fivethous = (i4.f1 + i8.q2))
|
|
|
|
-> Seq Scan on tenk1 t1
|
|
|
|
-> Hash
|
|
|
|
-> Seq Scan on int4_tbl i4
|
|
|
|
(9 rows)
|
|
|
|
|
|
|
|
select i8.q2, ss.* from
|
|
|
|
int8_tbl i8,
|
|
|
|
lateral (select t1.fivethous, i4.f1 from tenk1 t1 join int4_tbl i4
|
|
|
|
on t1.fivethous = i4.f1+i8.q2 order by 1,2) ss;
|
|
|
|
q2 | fivethous | f1
|
|
|
|
-----+-----------+----
|
|
|
|
456 | 456 | 0
|
|
|
|
456 | 456 | 0
|
|
|
|
123 | 123 | 0
|
|
|
|
123 | 123 | 0
|
|
|
|
(4 rows)
|
|
|
|
|
|
|
|
rollback;
|