2011-02-14 02:06:41 +01:00
|
|
|
CREATE EXTENSION intarray;
|
2001-01-12 01:16:26 +01:00
|
|
|
|
2016-11-29 21:05:22 +01:00
|
|
|
-- Check whether any of our opclasses fail amvalidate
|
|
|
|
|
SELECT amname, opcname
|
|
|
|
|
FROM pg_opclass opc LEFT JOIN pg_am am ON am.oid = opcmethod
|
|
|
|
|
WHERE opc.oid >= 16384 AND NOT amvalidate(opc.oid);
|
|
|
|
|
|
2002-10-18 20:41:22 +02:00
|
|
|
SELECT intset(1234);
|
|
|
|
|
SELECT icount('{1234234,234234}');
|
|
|
|
|
SELECT sort('{1234234,-30,234234}');
|
|
|
|
|
SELECT sort('{1234234,-30,234234}','asc');
|
|
|
|
|
SELECT sort('{1234234,-30,234234}','desc');
|
|
|
|
|
SELECT sort_asc('{1234234,-30,234234}');
|
|
|
|
|
SELECT sort_desc('{1234234,-30,234234}');
|
|
|
|
|
SELECT uniq('{1234234,-30,-30,234234,-30}');
|
|
|
|
|
SELECT uniq(sort_asc('{1234234,-30,-30,234234,-30}'));
|
|
|
|
|
SELECT idx('{1234234,-30,-30,234234,-30}',-30);
|
|
|
|
|
SELECT subarray('{1234234,-30,-30,234234,-30}',2,3);
|
|
|
|
|
SELECT subarray('{1234234,-30,-30,234234,-30}',-1,1);
|
|
|
|
|
SELECT subarray('{1234234,-30,-30,234234,-30}',0,-1);
|
August 6, 2002
1. Reworked patch from Andrey Oktyabrski (ano@spider.ru) with
functions: icount, sort, sort_asc, uniq, idx, subarray
operations: #, +, -, |, &
FUNCTIONS:
int icount(int[]) - the number of elements in intarray
int[] sort(int[], 'asc' | 'desc') - sort intarray
int[] sort(int[]) - sort in ascending order
int[] sort_asc(int[]),sort_desc(int[]) - shortcuts for sort
int[] uniq(int[]) - returns unique elements
int idx(int[], int item) - returns index of first intarray matching element
to item, or '0' if matching failed.
int[] subarray(int[],int START [, int LEN]) - returns part of intarray
starting from element number START (from 1)
and length LEN.
OPERATIONS:
int[] && int[] - overlap - returns TRUE if arrays has at least one common elements.
int[] @ int[] - contains - returns TRUE if left array contains right array
int[] ~ int[] - contained - returns TRUE if left array is contained in right array
# int[] - return the number of elements in array
int[] + int - push element to array ( add to end of array)
int[] + int[] - merge of arrays (right array added to the end of left one)
int[] - int - remove entries matched by right argument from array
int[] - int[] - remove left array from right
int[] | int - returns intarray - union of arguments
int[] | int[] - returns intarray as a union of two arrays
int[] & int[] - returns intersection of arrays
Oleg Bartunov
2002-08-10 22:38:29 +02:00
|
|
|
|
2002-10-18 20:41:22 +02:00
|
|
|
SELECT #'{1234234,234234}'::int[];
|
|
|
|
|
SELECT '{123,623,445}'::int[] + 1245;
|
|
|
|
|
SELECT '{123,623,445}'::int[] + 445;
|
|
|
|
|
SELECT '{123,623,445}'::int[] + '{1245,87,445}';
|
|
|
|
|
SELECT '{123,623,445}'::int[] - 623;
|
|
|
|
|
SELECT '{123,623,445}'::int[] - '{1623,623}';
|
|
|
|
|
SELECT '{123,623,445}'::int[] | 623;
|
|
|
|
|
SELECT '{123,623,445}'::int[] | 1623;
|
|
|
|
|
SELECT '{123,623,445}'::int[] | '{1623,623}';
|
|
|
|
|
SELECT '{123,623,445}'::int[] & '{1623,623}';
|
2012-02-17 02:00:11 +01:00
|
|
|
SELECT '{-1,3,1}'::int[] & '{1,2}';
|
2018-07-09 20:28:04 +02:00
|
|
|
SELECT '{1}'::int[] & '{2}'::int[];
|
|
|
|
|
SELECT array_dims('{1}'::int[] & '{2}'::int[]);
|
|
|
|
|
SELECT ('{1}'::int[] & '{2}'::int[]) = '{}'::int[];
|
|
|
|
|
SELECT ('{}'::int[] & '{}'::int[]) = '{}'::int[];
|
August 6, 2002
1. Reworked patch from Andrey Oktyabrski (ano@spider.ru) with
functions: icount, sort, sort_asc, uniq, idx, subarray
operations: #, +, -, |, &
FUNCTIONS:
int icount(int[]) - the number of elements in intarray
int[] sort(int[], 'asc' | 'desc') - sort intarray
int[] sort(int[]) - sort in ascending order
int[] sort_asc(int[]),sort_desc(int[]) - shortcuts for sort
int[] uniq(int[]) - returns unique elements
int idx(int[], int item) - returns index of first intarray matching element
to item, or '0' if matching failed.
int[] subarray(int[],int START [, int LEN]) - returns part of intarray
starting from element number START (from 1)
and length LEN.
OPERATIONS:
int[] && int[] - overlap - returns TRUE if arrays has at least one common elements.
int[] @ int[] - contains - returns TRUE if left array contains right array
int[] ~ int[] - contained - returns TRUE if left array is contained in right array
# int[] - return the number of elements in array
int[] + int - push element to array ( add to end of array)
int[] + int[] - merge of arrays (right array added to the end of left one)
int[] - int - remove entries matched by right argument from array
int[] - int[] - remove left array from right
int[] | int - returns intarray - union of arguments
int[] | int[] - returns intarray as a union of two arrays
int[] & int[] - returns intersection of arrays
Oleg Bartunov
2002-08-10 22:38:29 +02:00
|
|
|
|
|
|
|
|
|
2001-09-23 06:16:16 +02:00
|
|
|
--test query_int
|
2002-10-18 20:41:22 +02:00
|
|
|
SELECT '1'::query_int;
|
|
|
|
|
SELECT ' 1'::query_int;
|
|
|
|
|
SELECT '1 '::query_int;
|
|
|
|
|
SELECT ' 1 '::query_int;
|
|
|
|
|
SELECT ' ! 1 '::query_int;
|
|
|
|
|
SELECT '!1'::query_int;
|
|
|
|
|
SELECT '1|2'::query_int;
|
|
|
|
|
SELECT '1|!2'::query_int;
|
|
|
|
|
SELECT '!1|2'::query_int;
|
|
|
|
|
SELECT '!1|!2'::query_int;
|
|
|
|
|
SELECT '!(!1|!2)'::query_int;
|
|
|
|
|
SELECT '!(!1|2)'::query_int;
|
|
|
|
|
SELECT '!(1|!2)'::query_int;
|
|
|
|
|
SELECT '!(1|2)'::query_int;
|
|
|
|
|
SELECT '1&2'::query_int;
|
|
|
|
|
SELECT '!1&2'::query_int;
|
|
|
|
|
SELECT '1&!2'::query_int;
|
|
|
|
|
SELECT '!1&!2'::query_int;
|
|
|
|
|
SELECT '(1&2)'::query_int;
|
|
|
|
|
SELECT '1&(2)'::query_int;
|
|
|
|
|
SELECT '!(1)&2'::query_int;
|
|
|
|
|
SELECT '!(1&2)'::query_int;
|
|
|
|
|
SELECT '1|2&3'::query_int;
|
|
|
|
|
SELECT '1|(2&3)'::query_int;
|
|
|
|
|
SELECT '(1|2)&3'::query_int;
|
|
|
|
|
SELECT '1|2&!3'::query_int;
|
|
|
|
|
SELECT '1|!2&3'::query_int;
|
|
|
|
|
SELECT '!1|2&3'::query_int;
|
|
|
|
|
SELECT '!1|(2&3)'::query_int;
|
|
|
|
|
SELECT '!(1|2)&3'::query_int;
|
|
|
|
|
SELECT '(!1|2)&3'::query_int;
|
|
|
|
|
SELECT '1|(2|(4|(5|6)))'::query_int;
|
|
|
|
|
SELECT '1|2|4|5|6'::query_int;
|
|
|
|
|
SELECT '1&(2&(4&(5&6)))'::query_int;
|
|
|
|
|
SELECT '1&2&4&5&6'::query_int;
|
|
|
|
|
SELECT '1&(2&(4&(5|6)))'::query_int;
|
|
|
|
|
SELECT '1&(2&(4&(5|!6)))'::query_int;
|
2001-09-23 06:16:16 +02:00
|
|
|
|
2022-12-28 15:53:00 +01:00
|
|
|
-- test non-error-throwing input
|
|
|
|
|
|
|
|
|
|
SELECT str as "query_int",
|
|
|
|
|
pg_input_is_valid(str,'query_int') as ok,
|
2023-02-28 00:04:13 +01:00
|
|
|
errinfo.sql_error_code,
|
|
|
|
|
errinfo.message,
|
|
|
|
|
errinfo.detail,
|
|
|
|
|
errinfo.hint
|
2022-12-28 15:53:00 +01:00
|
|
|
FROM (VALUES ('1&(2&(4&(5|6)))'),
|
|
|
|
|
('1#(2&(4&(5&6)))'),
|
|
|
|
|
('foo'))
|
2023-02-28 00:04:13 +01:00
|
|
|
AS a(str),
|
|
|
|
|
LATERAL pg_input_error_info(a.str, 'query_int') as errinfo;
|
2022-12-28 15:53:00 +01:00
|
|
|
|
|
|
|
|
|
2001-09-23 06:16:16 +02:00
|
|
|
|
2001-01-12 01:16:26 +01:00
|
|
|
CREATE TABLE test__int( a int[] );
|
|
|
|
|
\copy test__int from 'data/test__int.data'
|
2015-07-21 19:54:18 +02:00
|
|
|
ANALYZE test__int;
|
2001-01-12 01:16:26 +01:00
|
|
|
|
|
|
|
|
SELECT count(*) from test__int WHERE a && '{23,50}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '23|50';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{23,50}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '23&50';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}';
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
SELECT count(*) from test__int WHERE a <@ '{73,23,20}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a = '{73,23,20}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '50&68';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)';
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '20 | !21';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '!20 & !21';
|
|
|
|
|
|
|
|
|
|
SET enable_seqscan = off; -- not all of these would use index by default
|
2001-01-12 01:16:26 +01:00
|
|
|
|
2001-08-21 18:36:06 +02:00
|
|
|
CREATE INDEX text_idx on test__int using gist ( a gist__int_ops );
|
2001-03-17 22:59:42 +01:00
|
|
|
|
|
|
|
|
SELECT count(*) from test__int WHERE a && '{23,50}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '23|50';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{23,50}';
|
Implement operator class parameters
PostgreSQL provides set of template index access methods, where opclasses have
much freedom in the semantics of indexing. These index AMs are GiST, GIN,
SP-GiST and BRIN. There opclasses define representation of keys, operations on
them and supported search strategies. So, it's natural that opclasses may be
faced some tradeoffs, which require user-side decision. This commit implements
opclass parameters allowing users to set some values, which tell opclass how to
index the particular dataset.
This commit doesn't introduce new storage in system catalog. Instead it uses
pg_attribute.attoptions, which is used for table column storage options but
unused for index attributes.
In order to evade changing signature of each opclass support function, we
implement unified way to pass options to opclass support functions. Options
are set to fn_expr as the constant bytea expression. It's possible due to the
fact that opclass support functions are executed outside of expressions, so
fn_expr is unused for them.
This commit comes with some examples of opclass options usage. We parametrize
signature length in GiST. That applies to multiple opclasses: tsvector_ops,
gist__intbig_ops, gist_ltree_ops, gist__ltree_ops, gist_trgm_ops and
gist_hstore_ops. Also we parametrize maximum number of integer ranges for
gist__int_ops. However, the main future usage of this feature is expected
to be json, where users would be able to specify which way to index particular
json parts.
Catversion is bumped.
Discussion: https://postgr.es/m/d22c3a18-31c7-1879-fc11-4c1ce2f5e5af%40postgrespro.ru
Author: Nikita Glukhov, revised by me
Reviwed-by: Nikolay Shaplov, Robert Haas, Tom Lane, Tomas Vondra, Alvaro Herrera
2020-03-30 18:17:11 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '23&50';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a <@ '{73,23,20}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a = '{73,23,20}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '50&68';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '20 | !21';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '!20 & !21';
|
|
|
|
|
|
intarray: Prevent out-of-bound memory reads with gist__int_ops
As gist__int_ops stands in intarray, it is possible to store GiST
entries for leaf pages that can cause corruptions when decompressed.
Leaf nodes are stored as decompressed all the time by the compression
method, and the decompression method should map with that, retrieving
the contents of the page without doing any decompression. However, the
code authorized the insertion of leaf page data with a higher number of
array items than what can be supported, generating a NOTICE message to
inform about this matter (199 for a 8k page, for reference). When
calling the decompression method, a decompression would be attempted on
this leaf node item but the contents should be retrieved as they are.
The NOTICE message generated when dealing with the compression of a leaf
page and too many elements in the input array for gist__int_ops has been
introduced by 08ee64e, removing the marker stored in the array to track
if this is actually a leaf node. However, it also missed the fact that
the decompression path should do nothing for a leaf page. Hence, as the
code stand, a too-large array would be stored as uncompressed but the
decompression path would attempt a decompression rather that retrieving
the contents as they are.
This leads to various problems. First, even if 08ee64e tried to address
that, it is possible to do out-of-bound chunk writes with a large input
array, with the backend informing about that with WARNINGs. On
decompression, retrieving the stored leaf data would lead to incorrect
memory reads, leading to crashes or even worse.
Perhaps somebody would be interested in expanding the number of array
items that can be handled in a leaf page for this operator in the
future, which would require revisiting the choice done in 08ee64e, but
based on the lack of reports about this problem since 2005 it does not
look so. For now, this commit prevents the insertion of data for leaf
pages when using more array items that the code can handle on
decompression, switching the NOTICE message to an ERROR. If one wishes
to use more array items, gist__intbig_ops is an optional choice.
While on it, use ERRCODE_PROGRAM_LIMIT_EXCEEDED as error code when a
limit is reached, because that's what the module is facing in such
cases.
Author: Ankit Kumar Pandey, Alexander Lakhin
Reviewed-by: Richard Guo, Michael Paquier
Discussion: https://postgr.es/m/796b65c3-57b7-bddf-b0d5-a8afafb8b627@gmail.com
Discussion: https://postgr.es/m/17888-f72930e6b5ce8c14@postgresql.org
Backpatch-through: 11
2023-06-15 06:45:34 +02:00
|
|
|
INSERT INTO test__int SELECT array(SELECT x FROM generate_series(1, 1001) x); -- should fail
|
|
|
|
|
|
Implement operator class parameters
PostgreSQL provides set of template index access methods, where opclasses have
much freedom in the semantics of indexing. These index AMs are GiST, GIN,
SP-GiST and BRIN. There opclasses define representation of keys, operations on
them and supported search strategies. So, it's natural that opclasses may be
faced some tradeoffs, which require user-side decision. This commit implements
opclass parameters allowing users to set some values, which tell opclass how to
index the particular dataset.
This commit doesn't introduce new storage in system catalog. Instead it uses
pg_attribute.attoptions, which is used for table column storage options but
unused for index attributes.
In order to evade changing signature of each opclass support function, we
implement unified way to pass options to opclass support functions. Options
are set to fn_expr as the constant bytea expression. It's possible due to the
fact that opclass support functions are executed outside of expressions, so
fn_expr is unused for them.
This commit comes with some examples of opclass options usage. We parametrize
signature length in GiST. That applies to multiple opclasses: tsvector_ops,
gist__intbig_ops, gist_ltree_ops, gist__ltree_ops, gist_trgm_ops and
gist_hstore_ops. Also we parametrize maximum number of integer ranges for
gist__int_ops. However, the main future usage of this feature is expected
to be json, where users would be able to specify which way to index particular
json parts.
Catversion is bumped.
Discussion: https://postgr.es/m/d22c3a18-31c7-1879-fc11-4c1ce2f5e5af%40postgrespro.ru
Author: Nikita Glukhov, revised by me
Reviwed-by: Nikolay Shaplov, Robert Haas, Tom Lane, Tomas Vondra, Alvaro Herrera
2020-03-30 18:17:11 +02:00
|
|
|
DROP INDEX text_idx;
|
|
|
|
|
CREATE INDEX text_idx on test__int using gist (a gist__int_ops(numranges = 0));
|
|
|
|
|
CREATE INDEX text_idx on test__int using gist (a gist__int_ops(numranges = 253));
|
|
|
|
|
CREATE INDEX text_idx on test__int using gist (a gist__int_ops(numranges = 252));
|
|
|
|
|
|
|
|
|
|
SELECT count(*) from test__int WHERE a && '{23,50}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '23|50';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @> '{23,50}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '23&50';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a <@ '{73,23,20}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a = '{73,23,20}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '50&68';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '20 | !21';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '!20 & !21';
|
|
|
|
|
|
|
|
|
|
DROP INDEX text_idx;
|
|
|
|
|
CREATE INDEX text_idx on test__int using gist (a gist__intbig_ops(siglen = 0));
|
|
|
|
|
CREATE INDEX text_idx on test__int using gist (a gist__intbig_ops(siglen = 2025));
|
|
|
|
|
CREATE INDEX text_idx on test__int using gist (a gist__intbig_ops(siglen = 2024));
|
|
|
|
|
|
|
|
|
|
SELECT count(*) from test__int WHERE a && '{23,50}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '23|50';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @> '{23,50}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '23&50';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}';
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
SELECT count(*) from test__int WHERE a <@ '{73,23,20}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a = '{73,23,20}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '50&68';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)';
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '20 | !21';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '!20 & !21';
|
2001-03-17 22:59:42 +01:00
|
|
|
|
2002-10-18 20:41:22 +02:00
|
|
|
DROP INDEX text_idx;
|
2001-08-21 18:36:06 +02:00
|
|
|
CREATE INDEX text_idx on test__int using gist ( a gist__intbig_ops );
|
2001-03-17 22:59:42 +01:00
|
|
|
|
|
|
|
|
SELECT count(*) from test__int WHERE a && '{23,50}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '23|50';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{23,50}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '23&50';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}';
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
SELECT count(*) from test__int WHERE a <@ '{73,23,20}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a = '{73,23,20}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '50&68';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}';
|
2001-09-23 06:16:16 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)';
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '20 | !21';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '!20 & !21';
|
2006-05-03 18:31:07 +02:00
|
|
|
|
|
|
|
|
DROP INDEX text_idx;
|
2007-09-14 05:25:31 +02:00
|
|
|
CREATE INDEX text_idx on test__int using gin ( a gin__int_ops );
|
2006-05-03 18:31:07 +02:00
|
|
|
|
|
|
|
|
SELECT count(*) from test__int WHERE a && '{23,50}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '23|50';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{23,50}';
|
2006-05-03 18:31:07 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '23&50';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}';
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
SELECT count(*) from test__int WHERE a <@ '{73,23,20}';
|
|
|
|
|
SELECT count(*) from test__int WHERE a = '{73,23,20}';
|
2006-05-03 18:31:07 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '50&68';
|
2006-09-10 19:36:52 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}';
|
2006-05-03 18:31:07 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)';
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
SELECT count(*) from test__int WHERE a @@ '20 | !21';
|
|
|
|
|
SELECT count(*) from test__int WHERE a @@ '!20 & !21';
|
|
|
|
|
|
Fix another bug in parent page splitting during GiST index build.
Yet another bug in the ilk of commits a7ee7c851 and 741b88435. In
741b88435, we took care to clear the memorized location of the
downlink when we split the parent page, because splitting the parent
page can move the downlink. But we missed that even *updating* a tuple
on the parent can move it, because updating a tuple on a gist page is
implemented as a delete+insert, so the updated tuple gets moved to the
end of the page.
This commit fixes the bug in two different ways (belt and suspenders):
1. Clear the downlink when we update a tuple on the parent page, even
if it's not split. This the same approach as in commits a7ee7c851
and 741b88435.
I also noticed that gistFindCorrectParent did not clear the
'downlinkoffnum' when it stepped to the right sibling. Fix that
too, as it seems like a clear bug even though I haven't been able
to find a test case to hit that.
2. Change gistFindCorrectParent so that it treats 'downlinkoffnum'
merely as a hint. It now always first checks if the downlink is
still at that location, and if not, it scans the page like before.
That's more robust if there are still more cases where we fail to
clear 'downlinkoffnum' that we haven't yet uncovered. With this,
it's no longer necessary to meticulously clear 'downlinkoffnum',
so this makes the previous fixes unnecessary, but I didn't revert
them because it still seems nice to clear it when we know that the
downlink has moved.
Also add the test case using the same test data that Alexander
posted. I tried to reduce it to a smaller test, and I also tried to
reproduce this with different test data, but I was not able to, so
let's just include what we have.
Backpatch to v12, like the previous fixes.
Reported-by: Alexander Lakhin
Discussion: https://www.postgresql.org/message-id/18129-caca016eaf0c3702@postgresql.org
2023-09-26 13:14:49 +02:00
|
|
|
DROP INDEX text_idx;
|
|
|
|
|
|
|
|
|
|
-- Repeat the same queries with an extended data set. The data set is the
|
|
|
|
|
-- same that we used before, except that each element in the array is
|
|
|
|
|
-- repeated three times, offset by 1000 and 2000. For example, {1, 5}
|
|
|
|
|
-- becomes {1, 1001, 2001, 5, 1005, 2005}.
|
|
|
|
|
--
|
|
|
|
|
-- That has proven to be unreasonably effective at exercising codepaths in
|
|
|
|
|
-- core GiST code related to splitting parent pages, which is not covered by
|
|
|
|
|
-- other tests. This is a bit out-of-place as the point is to test core GiST
|
|
|
|
|
-- code rather than this extension, but there is no suitable GiST opclass in
|
|
|
|
|
-- core that would reach the same codepaths.
|
|
|
|
|
CREATE TABLE more__int AS SELECT
|
|
|
|
|
-- Leave alone NULLs, empty arrays and the one row that we use to test
|
2024-01-07 21:19:50 +01:00
|
|
|
-- equality; also skip INT_MAX
|
Fix another bug in parent page splitting during GiST index build.
Yet another bug in the ilk of commits a7ee7c851 and 741b88435. In
741b88435, we took care to clear the memorized location of the
downlink when we split the parent page, because splitting the parent
page can move the downlink. But we missed that even *updating* a tuple
on the parent can move it, because updating a tuple on a gist page is
implemented as a delete+insert, so the updated tuple gets moved to the
end of the page.
This commit fixes the bug in two different ways (belt and suspenders):
1. Clear the downlink when we update a tuple on the parent page, even
if it's not split. This the same approach as in commits a7ee7c851
and 741b88435.
I also noticed that gistFindCorrectParent did not clear the
'downlinkoffnum' when it stepped to the right sibling. Fix that
too, as it seems like a clear bug even though I haven't been able
to find a test case to hit that.
2. Change gistFindCorrectParent so that it treats 'downlinkoffnum'
merely as a hint. It now always first checks if the downlink is
still at that location, and if not, it scans the page like before.
That's more robust if there are still more cases where we fail to
clear 'downlinkoffnum' that we haven't yet uncovered. With this,
it's no longer necessary to meticulously clear 'downlinkoffnum',
so this makes the previous fixes unnecessary, but I didn't revert
them because it still seems nice to clear it when we know that the
downlink has moved.
Also add the test case using the same test data that Alexander
posted. I tried to reduce it to a smaller test, and I also tried to
reproduce this with different test data, but I was not able to, so
let's just include what we have.
Backpatch to v12, like the previous fixes.
Reported-by: Alexander Lakhin
Discussion: https://www.postgresql.org/message-id/18129-caca016eaf0c3702@postgresql.org
2023-09-26 13:14:49 +02:00
|
|
|
CASE WHEN a IS NULL OR a = '{}' OR a = '{73,23,20}' THEN a ELSE
|
2024-01-07 21:19:50 +01:00
|
|
|
(select array_agg(u) || array_agg(u + 1000) || array_agg(u + 2000)
|
|
|
|
|
from unnest(a) u where u < 2000000000)
|
Fix another bug in parent page splitting during GiST index build.
Yet another bug in the ilk of commits a7ee7c851 and 741b88435. In
741b88435, we took care to clear the memorized location of the
downlink when we split the parent page, because splitting the parent
page can move the downlink. But we missed that even *updating* a tuple
on the parent can move it, because updating a tuple on a gist page is
implemented as a delete+insert, so the updated tuple gets moved to the
end of the page.
This commit fixes the bug in two different ways (belt and suspenders):
1. Clear the downlink when we update a tuple on the parent page, even
if it's not split. This the same approach as in commits a7ee7c851
and 741b88435.
I also noticed that gistFindCorrectParent did not clear the
'downlinkoffnum' when it stepped to the right sibling. Fix that
too, as it seems like a clear bug even though I haven't been able
to find a test case to hit that.
2. Change gistFindCorrectParent so that it treats 'downlinkoffnum'
merely as a hint. It now always first checks if the downlink is
still at that location, and if not, it scans the page like before.
That's more robust if there are still more cases where we fail to
clear 'downlinkoffnum' that we haven't yet uncovered. With this,
it's no longer necessary to meticulously clear 'downlinkoffnum',
so this makes the previous fixes unnecessary, but I didn't revert
them because it still seems nice to clear it when we know that the
downlink has moved.
Also add the test case using the same test data that Alexander
posted. I tried to reduce it to a smaller test, and I also tried to
reproduce this with different test data, but I was not able to, so
let's just include what we have.
Backpatch to v12, like the previous fixes.
Reported-by: Alexander Lakhin
Discussion: https://www.postgresql.org/message-id/18129-caca016eaf0c3702@postgresql.org
2023-09-26 13:14:49 +02:00
|
|
|
END AS a, a as b
|
|
|
|
|
FROM test__int;
|
|
|
|
|
CREATE INDEX ON more__int using gist (a gist__int_ops(numranges = 252));
|
|
|
|
|
|
|
|
|
|
SELECT count(*) from more__int WHERE a && '{23,50}';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @@ '23|50';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @> '{23,50}';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @@ '23&50';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @> '{20,23}';
|
|
|
|
|
SELECT count(*) from more__int WHERE a <@ '{73,23,20}';
|
|
|
|
|
SELECT count(*) from more__int WHERE a = '{73,23,20}';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @@ '50&68';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @> '{20,23}' or a @> '{50,68}';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @@ '(20&23)|(50&68)';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @@ '20 | !21';
|
|
|
|
|
SELECT count(*) from more__int WHERE a @@ '!20 & !21';
|
|
|
|
|
|
|
|
|
|
|
Fix intarray's GiST opclasses to not fail for empty arrays with <@.
contrib/intarray considers "arraycol <@ constant-array" to be indexable,
but its GiST opclass code fails to reliably find index entries for empty
array values (which of course should trivially match such queries).
This is because the test condition to see whether we should descend
through a non-leaf node is wrong.
Unfortunately, empty array entries could be anywhere in the index,
as these index opclasses are currently designed. So there's no way
to fix this except by lobotomizing <@ indexscans to scan the whole
index ... which is what this patch does. That's pretty unfortunate:
the performance is now actually worse than a seqscan, in most cases.
We'd be better off to remove <@ from the GiST opclasses entirely,
and perhaps a future non-back-patchable patch will do so.
In the meantime, applications whose performance is adversely impacted
have a couple of options. They could switch to a GIN index, which
doesn't have this bug, or they could replace "arraycol <@ constant-array"
with "arraycol <@ constant-array AND arraycol && constant-array".
That will provide about the same performance as before, and it will find
all non-empty subsets of the given constant-array, which is all that
could reliably be expected of the query before.
While at it, add some more regression test cases to improve code
coverage of contrib/intarray.
In passing, adjust resize_intArrayType so that when it's returning an
empty array, it uses construct_empty_array for that rather than
cowboy hacking on the input array. While the hack produces an array
that looks valid for most purposes, it isn't bitwise equal to empty
arrays produced by other code paths, which could have subtle odd
effects. I don't think this code path is performance-critical
enough to justify such shortcuts. (Back-patch this part only as far
as v11; before commit 01783ac36 we were not careful about this in
other intarray code paths either.)
Back-patch the <@ fixes to all supported versions, since this was
broken from day one.
Patch by me; thanks to Alexander Korotkov for review.
Discussion: https://postgr.es/m/458.1565114141@sss.pgh.pa.us
2019-08-07 00:04:51 +02:00
|
|
|
RESET enable_seqscan;
|
