From 5d79fc60c57d6fee25bc694d466702fc5edb8855 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 7 Apr 2020 16:30:55 -0400 Subject: [PATCH] Adjust bytea get_bit/set_bit to cope with bytea strings > 256MB. Since the existing bit number argument can't exceed INT32_MAX, it's not possible for these functions to manipulate bits beyond the first 256MB of a bytea value. However, it'd be good if they could do at least that much, and not fall over entirely for longer bytea values. Adjust the comparisons to be done in int64 arithmetic so that works. Also tweak the error reports to show sane values in case of overflow. Also add some test cases to improve the miserable code coverage of these functions. Apply patch to back branches only; HEAD has a better solution as of commit 26a944cf2. Extracted from a much larger patch by Movead Li Discussion: https://postgr.es/m/20200312115135445367128@highgo.ca --- src/backend/utils/adt/varlena.c | 10 ++-- src/test/regress/expected/strings.out | 76 +++++++++++++++++++++++++++ src/test/regress/sql/strings.sql | 23 ++++++++ 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 87ecf58ade..6d42bca1ea 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -3081,11 +3081,12 @@ byteaGetBit(PG_FUNCTION_ARGS) len = VARSIZE_ANY_EXHDR(v); - if (n < 0 || n >= len * 8) + /* Do comparison arithmetic in int64 in case len exceeds INT_MAX/8 */ + if (n < 0 || n >= (int64) len * 8) ereport(ERROR, (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), errmsg("index %d out of valid range, 0..%d", - n, len * 8 - 1))); + n, (int) Min((int64) len * 8 - 1, INT_MAX)))); byteNo = n / 8; bitNo = n % 8; @@ -3152,11 +3153,12 @@ byteaSetBit(PG_FUNCTION_ARGS) len = VARSIZE(res) - VARHDRSZ; - if (n < 0 || n >= len * 8) + /* Do comparison arithmetic in int64 in case len exceeds INT_MAX/8 */ + if (n < 0 || n >= (int64) len * 8) ereport(ERROR, (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), errmsg("index %d out of valid range, 0..%d", - n, len * 8 - 1))); + n, (int) Min((int64) len * 8 - 1, INT_MAX)))); byteNo = n / 8; bitNo = n % 8; diff --git a/src/test/regress/expected/strings.out b/src/test/regress/expected/strings.out index 189bdffdca..ae7b1e15e8 100644 --- a/src/test/regress/expected/strings.out +++ b/src/test/regress/expected/strings.out @@ -1509,6 +1509,82 @@ SELECT sha512('The quick brown fox jumps over the lazy dog.'); \x91ea1245f20d46ae9a037a989f54f1f790f0a47607eeb8a14d12890cea77a1bbc6c7ed9cf205e67b7f2b8fd4c7dfd3a7a8617e45f3c463d481c7e586c39ac1ed (1 row) +-- +-- encode/decode +-- +SELECT encode('\x1234567890abcdef00', 'hex'); + encode +-------------------- + 1234567890abcdef00 +(1 row) + +SELECT decode('1234567890abcdef00', 'hex'); + decode +---------------------- + \x1234567890abcdef00 +(1 row) + +SELECT encode(('\x' || repeat('1234567890abcdef0001', 7))::bytea, 'base64'); + encode +------------------------------------------------------------------------------ + EjRWeJCrze8AARI0VniQq83vAAESNFZ4kKvN7wABEjRWeJCrze8AARI0VniQq83vAAESNFZ4kKvN+ + 7wABEjRWeJCrze8AAQ== +(1 row) + +SELECT decode(encode(('\x' || repeat('1234567890abcdef0001', 7))::bytea, + 'base64'), 'base64'); + decode +------------------------------------------------------------------------------------------------------------------------------------------------ + \x1234567890abcdef00011234567890abcdef00011234567890abcdef00011234567890abcdef00011234567890abcdef00011234567890abcdef00011234567890abcdef0001 +(1 row) + +SELECT encode('\x1234567890abcdef00', 'escape'); + encode +----------------------------- + \x124Vx\220\253\315\357\000 +(1 row) + +SELECT decode(encode('\x1234567890abcdef00', 'escape'), 'escape'); + decode +---------------------- + \x1234567890abcdef00 +(1 row) + +-- +-- get_bit/set_bit etc +-- +SELECT get_bit('\x1234567890abcdef00'::bytea, 43); + get_bit +--------- + 1 +(1 row) + +SELECT get_bit('\x1234567890abcdef00'::bytea, 99); -- error +ERROR: index 99 out of valid range, 0..71 +SELECT set_bit('\x1234567890abcdef00'::bytea, 43, 0); + set_bit +---------------------- + \x1234567890a3cdef00 +(1 row) + +SELECT set_bit('\x1234567890abcdef00'::bytea, 99, 0); -- error +ERROR: index 99 out of valid range, 0..71 +SELECT get_byte('\x1234567890abcdef00'::bytea, 3); + get_byte +---------- + 120 +(1 row) + +SELECT get_byte('\x1234567890abcdef00'::bytea, 99); -- error +ERROR: index 99 out of valid range, 0..8 +SELECT set_byte('\x1234567890abcdef00'::bytea, 7, 11); + set_byte +---------------------- + \x1234567890abcd0b00 +(1 row) + +SELECT set_byte('\x1234567890abcdef00'::bytea, 99, 11); -- error +ERROR: index 99 out of valid range, 0..8 -- -- test behavior of escape_string_warning and standard_conforming_strings options -- diff --git a/src/test/regress/sql/strings.sql b/src/test/regress/sql/strings.sql index f2203ef1b1..9433ec1550 100644 --- a/src/test/regress/sql/strings.sql +++ b/src/test/regress/sql/strings.sql @@ -526,6 +526,29 @@ SELECT sha384('The quick brown fox jumps over the lazy dog.'); SELECT sha512(''); SELECT sha512('The quick brown fox jumps over the lazy dog.'); +-- +-- encode/decode +-- +SELECT encode('\x1234567890abcdef00', 'hex'); +SELECT decode('1234567890abcdef00', 'hex'); +SELECT encode(('\x' || repeat('1234567890abcdef0001', 7))::bytea, 'base64'); +SELECT decode(encode(('\x' || repeat('1234567890abcdef0001', 7))::bytea, + 'base64'), 'base64'); +SELECT encode('\x1234567890abcdef00', 'escape'); +SELECT decode(encode('\x1234567890abcdef00', 'escape'), 'escape'); + +-- +-- get_bit/set_bit etc +-- +SELECT get_bit('\x1234567890abcdef00'::bytea, 43); +SELECT get_bit('\x1234567890abcdef00'::bytea, 99); -- error +SELECT set_bit('\x1234567890abcdef00'::bytea, 43, 0); +SELECT set_bit('\x1234567890abcdef00'::bytea, 99, 0); -- error +SELECT get_byte('\x1234567890abcdef00'::bytea, 3); +SELECT get_byte('\x1234567890abcdef00'::bytea, 99); -- error +SELECT set_byte('\x1234567890abcdef00'::bytea, 7, 11); +SELECT set_byte('\x1234567890abcdef00'::bytea, 99, 11); -- error + -- -- test behavior of escape_string_warning and standard_conforming_strings options --