From d952373a987bad331c0e499463159dd142ced1ef Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 10 Jan 2023 05:48:59 +0100 Subject: [PATCH] New header varatt.h split off from postgres.h This new header contains all the variable-length data types support (TOAST support) from postgres.h, which isn't needed by large parts of the backend code. Reviewed-by: Tom Lane Discussion: https://www.postgresql.org/message-id/flat/ddcce239-0f29-6e62-4b47-1f8ca742addf%40enterprisedb.com --- contrib/citext/citext.c | 1 + contrib/cube/cubeparse.y | 1 + contrib/fuzzystrmatch/fuzzystrmatch.c | 1 + contrib/ltree/ltree_io.c | 1 + contrib/ltree/ltxtquery_io.c | 1 + contrib/pg_trgm/trgm_gin.c | 1 + contrib/pg_trgm/trgm_gist.c | 1 + contrib/pg_trgm/trgm_regexp.c | 1 + contrib/pgcrypto/pgcrypto.c | 1 + contrib/uuid-ossp/uuid-ossp.c | 1 + doc/src/sgml/xfunc.sgml | 1 + src/backend/access/common/toast_compression.c | 1 + src/backend/access/hash/hashfunc.c | 1 + src/backend/access/spgist/spgtextproc.c | 1 + src/backend/access/table/toast_helper.c | 1 + src/backend/libpq/be-fsstubs.c | 1 + src/backend/libpq/pqformat.c | 1 + src/backend/tsearch/ts_parse.c | 1 + src/backend/tsearch/ts_typanalyze.c | 1 + src/backend/utils/adt/ascii.c | 1 + src/backend/utils/adt/char.c | 1 + src/backend/utils/adt/cryptohashfuncs.c | 1 + src/backend/utils/adt/encode.c | 1 + src/backend/utils/adt/formatting.c | 1 + src/backend/utils/adt/geo_ops.c | 1 + src/backend/utils/adt/like.c | 1 + src/backend/utils/adt/network_gist.c | 1 + src/backend/utils/adt/network_spgist.c | 1 + src/backend/utils/adt/oracle_compat.c | 1 + src/backend/utils/adt/quote.c | 1 + src/backend/utils/adt/rangetypes.c | 1 + src/backend/utils/adt/rangetypes_typanalyze.c | 1 + src/backend/utils/adt/tsginidx.c | 1 + src/backend/utils/adt/tsquery.c | 1 + src/backend/utils/adt/tsquery_cleanup.c | 1 + src/backend/utils/adt/tsquery_op.c | 1 + src/backend/utils/adt/tsquery_util.c | 1 + src/backend/utils/adt/tsvector.c | 1 + src/backend/utils/cache/attoptcache.c | 1 + src/backend/utils/cache/spccache.c | 1 + src/backend/utils/mb/mbutils.c | 1 + src/include/access/htup_details.h | 1 + src/include/meson.build | 1 + src/include/postgres.h | 358 +----------------- src/include/utils/expandeddatum.h | 2 + src/include/varatt.h | 358 ++++++++++++++++++ src/pl/plperl/Util.xs | 1 + .../modules/spgist_name_ops/spgist_name_ops.c | 1 + .../test_custom_rmgrs/test_custom_rmgrs.c | 1 + src/test/modules/test_shm_mq/test.c | 1 + 50 files changed, 412 insertions(+), 353 deletions(-) create mode 100644 src/include/varatt.h diff --git a/contrib/citext/citext.c b/contrib/citext/citext.c index df139462a6..976c578e1c 100644 --- a/contrib/citext/citext.c +++ b/contrib/citext/citext.c @@ -8,6 +8,7 @@ #include "utils/builtins.h" #include "utils/formatting.h" #include "utils/varlena.h" +#include "varatt.h" PG_MODULE_MAGIC; diff --git a/contrib/cube/cubeparse.y b/contrib/cube/cubeparse.y index 44450d1027..b39fbe63e6 100644 --- a/contrib/cube/cubeparse.y +++ b/contrib/cube/cubeparse.y @@ -9,6 +9,7 @@ #include "cubedata.h" #include "nodes/miscnodes.h" #include "utils/float.h" +#include "varatt.h" /* All grammar constructs return strings */ #define YYSTYPE char * diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.c b/contrib/fuzzystrmatch/fuzzystrmatch.c index 2a7e61c94c..e1222714e4 100644 --- a/contrib/fuzzystrmatch/fuzzystrmatch.c +++ b/contrib/fuzzystrmatch/fuzzystrmatch.c @@ -43,6 +43,7 @@ #include "mb/pg_wchar.h" #include "utils/builtins.h" #include "utils/varlena.h" +#include "varatt.h" PG_MODULE_MAGIC; diff --git a/contrib/ltree/ltree_io.c b/contrib/ltree/ltree_io.c index 56533172e3..5dce70bd1a 100644 --- a/contrib/ltree/ltree_io.c +++ b/contrib/ltree/ltree_io.c @@ -11,6 +11,7 @@ #include "libpq/pqformat.h" #include "ltree.h" #include "utils/memutils.h" +#include "varatt.h" typedef struct diff --git a/contrib/ltree/ltxtquery_io.c b/contrib/ltree/ltxtquery_io.c index c95f94df81..d9910e6c99 100644 --- a/contrib/ltree/ltxtquery_io.c +++ b/contrib/ltree/ltxtquery_io.c @@ -12,6 +12,7 @@ #include "ltree.h" #include "miscadmin.h" #include "nodes/miscnodes.h" +#include "varatt.h" /* parser's states */ diff --git a/contrib/pg_trgm/trgm_gin.c b/contrib/pg_trgm/trgm_gin.c index 32fafef203..29a52eac7a 100644 --- a/contrib/pg_trgm/trgm_gin.c +++ b/contrib/pg_trgm/trgm_gin.c @@ -7,6 +7,7 @@ #include "access/stratnum.h" #include "fmgr.h" #include "trgm.h" +#include "varatt.h" PG_FUNCTION_INFO_V1(gin_extract_trgm); PG_FUNCTION_INFO_V1(gin_extract_value_trgm); diff --git a/contrib/pg_trgm/trgm_gist.c b/contrib/pg_trgm/trgm_gist.c index 3d74a1463a..ef5d8cca78 100644 --- a/contrib/pg_trgm/trgm_gist.c +++ b/contrib/pg_trgm/trgm_gist.c @@ -8,6 +8,7 @@ #include "fmgr.h" #include "port/pg_bitutils.h" #include "trgm.h" +#include "varatt.h" /* gist_trgm_ops opclass options */ typedef struct diff --git a/contrib/pg_trgm/trgm_regexp.c b/contrib/pg_trgm/trgm_regexp.c index 35ea9dbb54..9a00564ae4 100644 --- a/contrib/pg_trgm/trgm_regexp.c +++ b/contrib/pg_trgm/trgm_regexp.c @@ -196,6 +196,7 @@ #include "tsearch/ts_locale.h" #include "utils/hsearch.h" #include "utils/memutils.h" +#include "varatt.h" /* * Uncomment (or use -DTRGM_REGEXP_DEBUG) to print debug info, diff --git a/contrib/pgcrypto/pgcrypto.c b/contrib/pgcrypto/pgcrypto.c index 8f882f4c68..96447c5757 100644 --- a/contrib/pgcrypto/pgcrypto.c +++ b/contrib/pgcrypto/pgcrypto.c @@ -39,6 +39,7 @@ #include "px.h" #include "utils/builtins.h" #include "utils/uuid.h" +#include "varatt.h" PG_MODULE_MAGIC; diff --git a/contrib/uuid-ossp/uuid-ossp.c b/contrib/uuid-ossp/uuid-ossp.c index 3f0e379a9b..6399baf257 100644 --- a/contrib/uuid-ossp/uuid-ossp.c +++ b/contrib/uuid-ossp/uuid-ossp.c @@ -19,6 +19,7 @@ #include "port/pg_bswap.h" #include "utils/builtins.h" #include "utils/uuid.h" +#include "varatt.h" /* * It's possible that there's more than one uuid.h header file present. diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index c19131b399..52e5aa17bf 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -2390,6 +2390,7 @@ PG_FUNCTION_INFO_V1(funcname); #include #include "fmgr.h" #include "utils/geo_decls.h" +#include "varatt.h" PG_MODULE_MAGIC; diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index 797cfd8c6e..4cf956a759 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -22,6 +22,7 @@ #include "common/pg_lzcompress.h" #include "fmgr.h" #include "utils/builtins.h" +#include "varatt.h" /* GUC */ int default_toast_compression = TOAST_PGLZ_COMPRESSION; diff --git a/src/backend/access/hash/hashfunc.c b/src/backend/access/hash/hashfunc.c index 8e60a6f161..e3e40d6c21 100644 --- a/src/backend/access/hash/hashfunc.c +++ b/src/backend/access/hash/hashfunc.c @@ -32,6 +32,7 @@ #include "utils/builtins.h" #include "utils/float.h" #include "utils/pg_locale.h" +#include "varatt.h" /* * Datatype-specific hash functions. diff --git a/src/backend/access/spgist/spgtextproc.c b/src/backend/access/spgist/spgtextproc.c index 5d2aefb845..03a7afdbab 100644 --- a/src/backend/access/spgist/spgtextproc.c +++ b/src/backend/access/spgist/spgtextproc.c @@ -46,6 +46,7 @@ #include "utils/datum.h" #include "utils/pg_locale.h" #include "utils/varlena.h" +#include "varatt.h" /* diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index 7052e2ae99..b5cfeb21aa 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -19,6 +19,7 @@ #include "access/toast_helper.h" #include "access/toast_internals.h" #include "catalog/pg_type_d.h" +#include "varatt.h" /* diff --git a/src/backend/libpq/be-fsstubs.c b/src/backend/libpq/be-fsstubs.c index 3471aa5ccf..d189044a4f 100644 --- a/src/backend/libpq/be-fsstubs.c +++ b/src/backend/libpq/be-fsstubs.c @@ -53,6 +53,7 @@ #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/snapmgr.h" +#include "varatt.h" /* define this to enable debug logging */ /* #define FSDB 1 */ diff --git a/src/backend/libpq/pqformat.c b/src/backend/libpq/pqformat.c index 418f83cb45..b7e2c7b6c9 100644 --- a/src/backend/libpq/pqformat.c +++ b/src/backend/libpq/pqformat.c @@ -77,6 +77,7 @@ #include "libpq/pqformat.h" #include "mb/pg_wchar.h" #include "port/pg_bswap.h" +#include "varatt.h" /* -------------------------------- diff --git a/src/backend/tsearch/ts_parse.c b/src/backend/tsearch/ts_parse.c index 0742967189..9b6d934958 100644 --- a/src/backend/tsearch/ts_parse.c +++ b/src/backend/tsearch/ts_parse.c @@ -16,6 +16,7 @@ #include "tsearch/ts_cache.h" #include "tsearch/ts_utils.h" +#include "varatt.h" #define IGNORE_LONGLEXEME 1 diff --git a/src/backend/tsearch/ts_typanalyze.c b/src/backend/tsearch/ts_typanalyze.c index 04221b2235..ae4b5d10f2 100644 --- a/src/backend/tsearch/ts_typanalyze.c +++ b/src/backend/tsearch/ts_typanalyze.c @@ -19,6 +19,7 @@ #include "common/hashfn.h" #include "tsearch/ts_type.h" #include "utils/builtins.h" +#include "varatt.h" /* A hash key for lexemes */ diff --git a/src/backend/utils/adt/ascii.c b/src/backend/utils/adt/ascii.c index 47b313b447..b6944d8093 100644 --- a/src/backend/utils/adt/ascii.c +++ b/src/backend/utils/adt/ascii.c @@ -14,6 +14,7 @@ #include "mb/pg_wchar.h" #include "utils/ascii.h" #include "utils/builtins.h" +#include "varatt.h" static void pg_to_ascii(unsigned char *src, unsigned char *src_end, unsigned char *dest, int enc); diff --git a/src/backend/utils/adt/char.c b/src/backend/utils/adt/char.c index 476c59d8ca..3366259539 100644 --- a/src/backend/utils/adt/char.c +++ b/src/backend/utils/adt/char.c @@ -19,6 +19,7 @@ #include "libpq/pqformat.h" #include "utils/builtins.h" +#include "varatt.h" #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7')) #define TOOCTAL(c) ((c) + '0') diff --git a/src/backend/utils/adt/cryptohashfuncs.c b/src/backend/utils/adt/cryptohashfuncs.c index 546890e971..f960327958 100644 --- a/src/backend/utils/adt/cryptohashfuncs.c +++ b/src/backend/utils/adt/cryptohashfuncs.c @@ -17,6 +17,7 @@ #include "common/md5.h" #include "common/sha2.h" #include "utils/builtins.h" +#include "varatt.h" /* diff --git a/src/backend/utils/adt/encode.c b/src/backend/utils/adt/encode.c index 7bfdc25245..b92191de81 100644 --- a/src/backend/utils/adt/encode.c +++ b/src/backend/utils/adt/encode.c @@ -18,6 +18,7 @@ #include "mb/pg_wchar.h" #include "utils/builtins.h" #include "utils/memutils.h" +#include "varatt.h" /* diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index 56b865547b..a4b524ea3a 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -88,6 +88,7 @@ #include "utils/memutils.h" #include "utils/numeric.h" #include "utils/pg_locale.h" +#include "varatt.h" /* ---------- diff --git a/src/backend/utils/adt/geo_ops.c b/src/backend/utils/adt/geo_ops.c index bd78b8f878..53ee4b6f9c 100644 --- a/src/backend/utils/adt/geo_ops.c +++ b/src/backend/utils/adt/geo_ops.c @@ -35,6 +35,7 @@ #include "utils/float.h" #include "utils/fmgrprotos.h" #include "utils/geo_decls.h" +#include "varatt.h" /* * * Type constructors have this form: diff --git a/src/backend/utils/adt/like.c b/src/backend/utils/adt/like.c index 70c42c3876..fc6cb7f5b7 100644 --- a/src/backend/utils/adt/like.c +++ b/src/backend/utils/adt/like.c @@ -24,6 +24,7 @@ #include "miscadmin.h" #include "utils/builtins.h" #include "utils/pg_locale.h" +#include "varatt.h" #define LIKE_TRUE 1 diff --git a/src/backend/utils/adt/network_gist.c b/src/backend/utils/adt/network_gist.c index 243efa308d..32cde28f00 100644 --- a/src/backend/utils/adt/network_gist.c +++ b/src/backend/utils/adt/network_gist.c @@ -51,6 +51,7 @@ #include "access/stratnum.h" #include "utils/builtins.h" #include "utils/inet.h" +#include "varatt.h" /* * Operator strategy numbers used in the GiST inet_ops opclass diff --git a/src/backend/utils/adt/network_spgist.c b/src/backend/utils/adt/network_spgist.c index 0fe8b83407..5d3697306c 100644 --- a/src/backend/utils/adt/network_spgist.c +++ b/src/backend/utils/adt/network_spgist.c @@ -37,6 +37,7 @@ #include "catalog/pg_type.h" #include "utils/builtins.h" #include "utils/inet.h" +#include "varatt.h" static int inet_spg_node_number(const inet *val, int commonbits); diff --git a/src/backend/utils/adt/oracle_compat.c b/src/backend/utils/adt/oracle_compat.c index f4e6983e91..5f1bc59d87 100644 --- a/src/backend/utils/adt/oracle_compat.c +++ b/src/backend/utils/adt/oracle_compat.c @@ -21,6 +21,7 @@ #include "utils/builtins.h" #include "utils/formatting.h" #include "utils/memutils.h" +#include "varatt.h" static text *dotrim(const char *string, int stringlen, diff --git a/src/backend/utils/adt/quote.c b/src/backend/utils/adt/quote.c index a9ec96bb0d..f2f633befa 100644 --- a/src/backend/utils/adt/quote.c +++ b/src/backend/utils/adt/quote.c @@ -14,6 +14,7 @@ #include "postgres.h" #include "utils/builtins.h" +#include "varatt.h" /* diff --git a/src/backend/utils/adt/rangetypes.c b/src/backend/utils/adt/rangetypes.c index ee6fe67bc4..d65e5625c7 100644 --- a/src/backend/utils/adt/rangetypes.c +++ b/src/backend/utils/adt/rangetypes.c @@ -42,6 +42,7 @@ #include "utils/lsyscache.h" #include "utils/rangetypes.h" #include "utils/timestamp.h" +#include "varatt.h" /* fn_extra cache entry for one of the range I/O functions */ diff --git a/src/backend/utils/adt/rangetypes_typanalyze.c b/src/backend/utils/adt/rangetypes_typanalyze.c index d7a947b48a..86810a1a6e 100644 --- a/src/backend/utils/adt/rangetypes_typanalyze.c +++ b/src/backend/utils/adt/rangetypes_typanalyze.c @@ -31,6 +31,7 @@ #include "utils/lsyscache.h" #include "utils/rangetypes.h" #include "utils/multirangetypes.h" +#include "varatt.h" static int float8_qsort_cmp(const void *a1, const void *a2, void *arg); static int range_bound_qsort_cmp(const void *a1, const void *a2, void *arg); diff --git a/src/backend/utils/adt/tsginidx.c b/src/backend/utils/adt/tsginidx.c index 114f961263..484a003827 100644 --- a/src/backend/utils/adt/tsginidx.c +++ b/src/backend/utils/adt/tsginidx.c @@ -19,6 +19,7 @@ #include "tsearch/ts_type.h" #include "tsearch/ts_utils.h" #include "utils/builtins.h" +#include "varatt.h" Datum diff --git a/src/backend/utils/adt/tsquery.c b/src/backend/utils/adt/tsquery.c index 79cf30b424..25150c6d16 100644 --- a/src/backend/utils/adt/tsquery.c +++ b/src/backend/utils/adt/tsquery.c @@ -23,6 +23,7 @@ #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/pg_crc.h" +#include "varatt.h" /* FTS operator priorities, see ts_type.h */ const int tsearch_op_priority[OP_COUNT] = diff --git a/src/backend/utils/adt/tsquery_cleanup.c b/src/backend/utils/adt/tsquery_cleanup.c index 32554d67ea..59b3e859c3 100644 --- a/src/backend/utils/adt/tsquery_cleanup.c +++ b/src/backend/utils/adt/tsquery_cleanup.c @@ -17,6 +17,7 @@ #include "miscadmin.h" #include "tsearch/ts_utils.h" +#include "varatt.h" typedef struct NODE { diff --git a/src/backend/utils/adt/tsquery_op.c b/src/backend/utils/adt/tsquery_op.c index 02bd37cd7a..7e3bd51c1f 100644 --- a/src/backend/utils/adt/tsquery_op.c +++ b/src/backend/utils/adt/tsquery_op.c @@ -17,6 +17,7 @@ #include "lib/qunique.h" #include "tsearch/ts_utils.h" #include "utils/builtins.h" +#include "varatt.h" Datum tsquery_numnode(PG_FUNCTION_ARGS) diff --git a/src/backend/utils/adt/tsquery_util.c b/src/backend/utils/adt/tsquery_util.c index 27a59f9dec..f266b9f067 100644 --- a/src/backend/utils/adt/tsquery_util.c +++ b/src/backend/utils/adt/tsquery_util.c @@ -16,6 +16,7 @@ #include "miscadmin.h" #include "tsearch/ts_utils.h" +#include "varatt.h" /* * Build QTNode tree for a tsquery given in QueryItem array format. diff --git a/src/backend/utils/adt/tsvector.c b/src/backend/utils/adt/tsvector.c index 67ef084962..c7e20ce4ec 100644 --- a/src/backend/utils/adt/tsvector.c +++ b/src/backend/utils/adt/tsvector.c @@ -20,6 +20,7 @@ #include "tsearch/ts_utils.h" #include "utils/builtins.h" #include "utils/memutils.h" +#include "varatt.h" typedef struct { diff --git a/src/backend/utils/cache/attoptcache.c b/src/backend/utils/cache/attoptcache.c index dbf81bdf1b..28a99f0fc4 100644 --- a/src/backend/utils/cache/attoptcache.c +++ b/src/backend/utils/cache/attoptcache.c @@ -22,6 +22,7 @@ #include "utils/hsearch.h" #include "utils/inval.h" #include "utils/syscache.h" +#include "varatt.h" /* Hash table for information about each attribute's options */ diff --git a/src/backend/utils/cache/spccache.c b/src/backend/utils/cache/spccache.c index 44c63bd478..aabe6ba64b 100644 --- a/src/backend/utils/cache/spccache.c +++ b/src/backend/utils/cache/spccache.c @@ -29,6 +29,7 @@ #include "utils/inval.h" #include "utils/spccache.h" #include "utils/syscache.h" +#include "varatt.h" /* Hash table for information about each tablespace */ diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 2dbd2fe8cd..033647011b 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -40,6 +40,7 @@ #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/syscache.h" +#include "varatt.h" /* * We maintain a simple linked list caching the fmgr lookup info for the diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index d590114de0..e01f4f35c8 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -19,6 +19,7 @@ #include "access/tupdesc.h" #include "access/tupmacs.h" #include "storage/bufpage.h" +#include "varatt.h" /* * MaxTupleAttributeNumber limits the number of (user) columns in a tuple. diff --git a/src/include/meson.build b/src/include/meson.build index 5ecbc7323c..51ad06cecb 100644 --- a/src/include/meson.build +++ b/src/include/meson.build @@ -113,6 +113,7 @@ install_headers( 'postgres.h', 'postgres_ext.h', 'postgres_fe.h', + 'varatt.h', 'windowapi.h', pg_config_ext, pg_config_os, diff --git a/src/include/postgres.h b/src/include/postgres.h index c45274dc13..8a028ff789 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -23,9 +23,8 @@ * * section description * ------- ------------------------------------------------ - * 1) variable-length datatypes (TOAST support) - * 2) Datum type + support functions - * 3) miscellaneous + * 1) Datum type + support functions + * 2) miscellaneous * * NOTES * @@ -36,8 +35,7 @@ * postgres_fe.h. We do that since those type definitions are needed by * frontend modules that want to deal with binary data transmission to or * from the backend. Type definitions in this file should be for - * representations that never escape the backend, such as Datum or - * TOASTed varlena objects. + * representations that never escape the backend, such as Datum. * *---------------------------------------------------------------- */ @@ -49,353 +47,7 @@ #include "utils/palloc.h" /* ---------------------------------------------------------------- - * Section 1: variable-length datatypes (TOAST support) - * ---------------------------------------------------------------- - */ - -/* - * struct varatt_external is a traditional "TOAST pointer", that is, the - * information needed to fetch a Datum stored out-of-line in a TOAST table. - * The data is compressed if and only if the external size stored in - * va_extinfo is less than va_rawsize - VARHDRSZ. - * - * This struct must not contain any padding, because we sometimes compare - * these pointers using memcmp. - * - * Note that this information is stored unaligned within actual tuples, so - * you need to memcpy from the tuple into a local struct variable before - * you can look at these fields! (The reason we use memcmp is to avoid - * having to do that just to detect equality of two TOAST pointers...) - */ -typedef struct varatt_external -{ - int32 va_rawsize; /* Original data size (includes header) */ - uint32 va_extinfo; /* External saved size (without header) and - * compression method */ - Oid va_valueid; /* Unique ID of value within TOAST table */ - Oid va_toastrelid; /* RelID of TOAST table containing it */ -} varatt_external; - -/* - * These macros define the "saved size" portion of va_extinfo. Its remaining - * two high-order bits identify the compression method. - */ -#define VARLENA_EXTSIZE_BITS 30 -#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1) - -/* - * struct varatt_indirect is a "TOAST pointer" representing an out-of-line - * Datum that's stored in memory, not in an external toast relation. - * The creator of such a Datum is entirely responsible that the referenced - * storage survives for as long as referencing pointer Datums can exist. - * - * Note that just as for struct varatt_external, this struct is stored - * unaligned within any containing tuple. - */ -typedef struct varatt_indirect -{ - struct varlena *pointer; /* Pointer to in-memory varlena */ -} varatt_indirect; - -/* - * struct varatt_expanded is a "TOAST pointer" representing an out-of-line - * Datum that is stored in memory, in some type-specific, not necessarily - * physically contiguous format that is convenient for computation not - * storage. APIs for this, in particular the definition of struct - * ExpandedObjectHeader, are in src/include/utils/expandeddatum.h. - * - * Note that just as for struct varatt_external, this struct is stored - * unaligned within any containing tuple. - */ -typedef struct ExpandedObjectHeader ExpandedObjectHeader; - -typedef struct varatt_expanded -{ - ExpandedObjectHeader *eohptr; -} varatt_expanded; - -/* - * Type tag for the various sorts of "TOAST pointer" datums. The peculiar - * value for VARTAG_ONDISK comes from a requirement for on-disk compatibility - * with a previous notion that the tag field was the pointer datum's length. - */ -typedef enum vartag_external -{ - VARTAG_INDIRECT = 1, - VARTAG_EXPANDED_RO = 2, - VARTAG_EXPANDED_RW = 3, - VARTAG_ONDISK = 18 -} vartag_external; - -/* this test relies on the specific tag values above */ -#define VARTAG_IS_EXPANDED(tag) \ - (((tag) & ~1) == VARTAG_EXPANDED_RO) - -#define VARTAG_SIZE(tag) \ - ((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \ - VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \ - (tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \ - (AssertMacro(false), 0)) - -/* - * These structs describe the header of a varlena object that may have been - * TOASTed. Generally, don't reference these structs directly, but use the - * macros below. - * - * We use separate structs for the aligned and unaligned cases because the - * compiler might otherwise think it could generate code that assumes - * alignment while touching fields of a 1-byte-header varlena. - */ -typedef union -{ - struct /* Normal varlena (4-byte length) */ - { - uint32 va_header; - char va_data[FLEXIBLE_ARRAY_MEMBER]; - } va_4byte; - struct /* Compressed-in-line format */ - { - uint32 va_header; - uint32 va_tcinfo; /* Original data size (excludes header) and - * compression method; see va_extinfo */ - char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */ - } va_compressed; -} varattrib_4b; - -typedef struct -{ - uint8 va_header; - char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Data begins here */ -} varattrib_1b; - -/* TOAST pointers are a subset of varattrib_1b with an identifying tag byte */ -typedef struct -{ - uint8 va_header; /* Always 0x80 or 0x01 */ - uint8 va_tag; /* Type of datum */ - char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Type-specific data */ -} varattrib_1b_e; - -/* - * Bit layouts for varlena headers on big-endian machines: - * - * 00xxxxxx 4-byte length word, aligned, uncompressed data (up to 1G) - * 01xxxxxx 4-byte length word, aligned, *compressed* data (up to 1G) - * 10000000 1-byte length word, unaligned, TOAST pointer - * 1xxxxxxx 1-byte length word, unaligned, uncompressed data (up to 126b) - * - * Bit layouts for varlena headers on little-endian machines: - * - * xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G) - * xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G) - * 00000001 1-byte length word, unaligned, TOAST pointer - * xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to 126b) - * - * The "xxx" bits are the length field (which includes itself in all cases). - * In the big-endian case we mask to extract the length, in the little-endian - * case we shift. Note that in both cases the flag bits are in the physically - * first byte. Also, it is not possible for a 1-byte length word to be zero; - * this lets us disambiguate alignment padding bytes from the start of an - * unaligned datum. (We now *require* pad bytes to be filled with zero!) - * - * In TOAST pointers the va_tag field (see varattrib_1b_e) is used to discern - * the specific type and length of the pointer datum. - */ - -/* - * Endian-dependent macros. These are considered internal --- use the - * external macros below instead of using these directly. - * - * Note: IS_1B is true for external toast records but VARSIZE_1B will return 0 - * for such records. Hence you should usually check for IS_EXTERNAL before - * checking for IS_1B. - */ - -#ifdef WORDS_BIGENDIAN - -#define VARATT_IS_4B(PTR) \ - ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00) -#define VARATT_IS_4B_U(PTR) \ - ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x00) -#define VARATT_IS_4B_C(PTR) \ - ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x40) -#define VARATT_IS_1B(PTR) \ - ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x80) -#define VARATT_IS_1B_E(PTR) \ - ((((varattrib_1b *) (PTR))->va_header) == 0x80) -#define VARATT_NOT_PAD_BYTE(PTR) \ - (*((uint8 *) (PTR)) != 0) - -/* VARSIZE_4B() should only be used on known-aligned data */ -#define VARSIZE_4B(PTR) \ - (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF) -#define VARSIZE_1B(PTR) \ - (((varattrib_1b *) (PTR))->va_header & 0x7F) -#define VARTAG_1B_E(PTR) \ - (((varattrib_1b_e *) (PTR))->va_tag) - -#define SET_VARSIZE_4B(PTR,len) \ - (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF) -#define SET_VARSIZE_4B_C(PTR,len) \ - (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000) -#define SET_VARSIZE_1B(PTR,len) \ - (((varattrib_1b *) (PTR))->va_header = (len) | 0x80) -#define SET_VARTAG_1B_E(PTR,tag) \ - (((varattrib_1b_e *) (PTR))->va_header = 0x80, \ - ((varattrib_1b_e *) (PTR))->va_tag = (tag)) - -#else /* !WORDS_BIGENDIAN */ - -#define VARATT_IS_4B(PTR) \ - ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00) -#define VARATT_IS_4B_U(PTR) \ - ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x00) -#define VARATT_IS_4B_C(PTR) \ - ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x02) -#define VARATT_IS_1B(PTR) \ - ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x01) -#define VARATT_IS_1B_E(PTR) \ - ((((varattrib_1b *) (PTR))->va_header) == 0x01) -#define VARATT_NOT_PAD_BYTE(PTR) \ - (*((uint8 *) (PTR)) != 0) - -/* VARSIZE_4B() should only be used on known-aligned data */ -#define VARSIZE_4B(PTR) \ - ((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF) -#define VARSIZE_1B(PTR) \ - ((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F) -#define VARTAG_1B_E(PTR) \ - (((varattrib_1b_e *) (PTR))->va_tag) - -#define SET_VARSIZE_4B(PTR,len) \ - (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2)) -#define SET_VARSIZE_4B_C(PTR,len) \ - (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02) -#define SET_VARSIZE_1B(PTR,len) \ - (((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01) -#define SET_VARTAG_1B_E(PTR,tag) \ - (((varattrib_1b_e *) (PTR))->va_header = 0x01, \ - ((varattrib_1b_e *) (PTR))->va_tag = (tag)) - -#endif /* WORDS_BIGENDIAN */ - -#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) -#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data) -#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data) -#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data) - -/* - * Externally visible TOAST macros begin here. - */ - -#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data) -#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data) -#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data) - -#define VARATT_SHORT_MAX 0x7F -#define VARATT_CAN_MAKE_SHORT(PTR) \ - (VARATT_IS_4B_U(PTR) && \ - (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) <= VARATT_SHORT_MAX) -#define VARATT_CONVERTED_SHORT_SIZE(PTR) \ - (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) - -/* - * In consumers oblivious to data alignment, call PG_DETOAST_DATUM_PACKED(), - * VARDATA_ANY(), VARSIZE_ANY() and VARSIZE_ANY_EXHDR(). Elsewhere, call - * PG_DETOAST_DATUM(), VARDATA() and VARSIZE(). Directly fetching an int16, - * int32 or wider field in the struct representing the datum layout requires - * aligned data. memcpy() is alignment-oblivious, as are most operations on - * datatypes, such as text, whose layout struct contains only char fields. - * - * Code assembling a new datum should call VARDATA() and SET_VARSIZE(). - * (Datums begin life untoasted.) - * - * Other macros here should usually be used only by tuple assembly/disassembly - * code and code that specifically wants to work with still-toasted Datums. - */ -#define VARDATA(PTR) VARDATA_4B(PTR) -#define VARSIZE(PTR) VARSIZE_4B(PTR) - -#define VARSIZE_SHORT(PTR) VARSIZE_1B(PTR) -#define VARDATA_SHORT(PTR) VARDATA_1B(PTR) - -#define VARTAG_EXTERNAL(PTR) VARTAG_1B_E(PTR) -#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR))) -#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR) - -#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR) -#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) -#define VARATT_IS_EXTERNAL_ONDISK(PTR) \ - (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK) -#define VARATT_IS_EXTERNAL_INDIRECT(PTR) \ - (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT) -#define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \ - (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RO) -#define VARATT_IS_EXTERNAL_EXPANDED_RW(PTR) \ - (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RW) -#define VARATT_IS_EXTERNAL_EXPANDED(PTR) \ - (VARATT_IS_EXTERNAL(PTR) && VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR))) -#define VARATT_IS_EXTERNAL_NON_EXPANDED(PTR) \ - (VARATT_IS_EXTERNAL(PTR) && !VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR))) -#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR) -#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR)) - -#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len) -#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len) -#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len) - -#define SET_VARTAG_EXTERNAL(PTR, tag) SET_VARTAG_1B_E(PTR, tag) - -#define VARSIZE_ANY(PTR) \ - (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR) : \ - (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \ - VARSIZE_4B(PTR))) - -/* Size of a varlena data, excluding header */ -#define VARSIZE_ANY_EXHDR(PTR) \ - (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR)-VARHDRSZ_EXTERNAL : \ - (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-VARHDRSZ_SHORT : \ - VARSIZE_4B(PTR)-VARHDRSZ)) - -/* caution: this will not work on an external or compressed-in-line Datum */ -/* caution: this will return a possibly unaligned pointer */ -#define VARDATA_ANY(PTR) \ - (VARATT_IS_1B(PTR) ? VARDATA_1B(PTR) : VARDATA_4B(PTR)) - -/* Decompressed size and compression method of a compressed-in-line Datum */ -#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \ - (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK) -#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \ - (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) - -/* Same for external Datums; but note argument is a struct varatt_external */ -#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \ - ((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK) -#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \ - ((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS) - -#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \ - do { \ - Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \ - (cm) == TOAST_LZ4_COMPRESSION_ID); \ - ((toast_pointer).va_extinfo = \ - (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \ - } while (0) - -/* - * Testing whether an externally-stored value is compressed now requires - * comparing size stored in va_extinfo (the actual length of the external data) - * to rawsize (the original uncompressed datum's size). The latter includes - * VARHDRSZ overhead, the former doesn't. We never use compression unless it - * actually saves space, so we expect either equality or less-than. - */ -#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ - (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \ - (toast_pointer).va_rawsize - VARHDRSZ) - - -/* ---------------------------------------------------------------- - * Section 2: Datum type + support functions + * Section 1: Datum type + support functions * ---------------------------------------------------------------- */ @@ -907,7 +559,7 @@ extern Datum Float8GetDatum(float8 X); /* ---------------------------------------------------------------- - * Section 3: miscellaneous + * Section 2: miscellaneous * ---------------------------------------------------------------- */ diff --git a/src/include/utils/expandeddatum.h b/src/include/utils/expandeddatum.h index eefc912481..a77bb7e7e9 100644 --- a/src/include/utils/expandeddatum.h +++ b/src/include/utils/expandeddatum.h @@ -44,6 +44,8 @@ #ifndef EXPANDEDDATUM_H #define EXPANDEDDATUM_H +#include "varatt.h" + /* Size of an EXTERNAL datum that contains a pointer to an expanded object */ #define EXPANDED_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_expanded)) diff --git a/src/include/varatt.h b/src/include/varatt.h new file mode 100644 index 0000000000..e34870526b --- /dev/null +++ b/src/include/varatt.h @@ -0,0 +1,358 @@ +/*------------------------------------------------------------------------- + * + * varatt.h + * variable-length datatypes (TOAST support) + * + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1995, Regents of the University of California + * + * src/include/varatt.h + * + *------------------------------------------------------------------------- + */ + +#ifndef VARATT_H +#define VARATT_H + +/* + * struct varatt_external is a traditional "TOAST pointer", that is, the + * information needed to fetch a Datum stored out-of-line in a TOAST table. + * The data is compressed if and only if the external size stored in + * va_extinfo is less than va_rawsize - VARHDRSZ. + * + * This struct must not contain any padding, because we sometimes compare + * these pointers using memcmp. + * + * Note that this information is stored unaligned within actual tuples, so + * you need to memcpy from the tuple into a local struct variable before + * you can look at these fields! (The reason we use memcmp is to avoid + * having to do that just to detect equality of two TOAST pointers...) + */ +typedef struct varatt_external +{ + int32 va_rawsize; /* Original data size (includes header) */ + uint32 va_extinfo; /* External saved size (without header) and + * compression method */ + Oid va_valueid; /* Unique ID of value within TOAST table */ + Oid va_toastrelid; /* RelID of TOAST table containing it */ +} varatt_external; + +/* + * These macros define the "saved size" portion of va_extinfo. Its remaining + * two high-order bits identify the compression method. + */ +#define VARLENA_EXTSIZE_BITS 30 +#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1) + +/* + * struct varatt_indirect is a "TOAST pointer" representing an out-of-line + * Datum that's stored in memory, not in an external toast relation. + * The creator of such a Datum is entirely responsible that the referenced + * storage survives for as long as referencing pointer Datums can exist. + * + * Note that just as for struct varatt_external, this struct is stored + * unaligned within any containing tuple. + */ +typedef struct varatt_indirect +{ + struct varlena *pointer; /* Pointer to in-memory varlena */ +} varatt_indirect; + +/* + * struct varatt_expanded is a "TOAST pointer" representing an out-of-line + * Datum that is stored in memory, in some type-specific, not necessarily + * physically contiguous format that is convenient for computation not + * storage. APIs for this, in particular the definition of struct + * ExpandedObjectHeader, are in src/include/utils/expandeddatum.h. + * + * Note that just as for struct varatt_external, this struct is stored + * unaligned within any containing tuple. + */ +typedef struct ExpandedObjectHeader ExpandedObjectHeader; + +typedef struct varatt_expanded +{ + ExpandedObjectHeader *eohptr; +} varatt_expanded; + +/* + * Type tag for the various sorts of "TOAST pointer" datums. The peculiar + * value for VARTAG_ONDISK comes from a requirement for on-disk compatibility + * with a previous notion that the tag field was the pointer datum's length. + */ +typedef enum vartag_external +{ + VARTAG_INDIRECT = 1, + VARTAG_EXPANDED_RO = 2, + VARTAG_EXPANDED_RW = 3, + VARTAG_ONDISK = 18 +} vartag_external; + +/* this test relies on the specific tag values above */ +#define VARTAG_IS_EXPANDED(tag) \ + (((tag) & ~1) == VARTAG_EXPANDED_RO) + +#define VARTAG_SIZE(tag) \ + ((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \ + VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \ + (tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \ + (AssertMacro(false), 0)) + +/* + * These structs describe the header of a varlena object that may have been + * TOASTed. Generally, don't reference these structs directly, but use the + * macros below. + * + * We use separate structs for the aligned and unaligned cases because the + * compiler might otherwise think it could generate code that assumes + * alignment while touching fields of a 1-byte-header varlena. + */ +typedef union +{ + struct /* Normal varlena (4-byte length) */ + { + uint32 va_header; + char va_data[FLEXIBLE_ARRAY_MEMBER]; + } va_4byte; + struct /* Compressed-in-line format */ + { + uint32 va_header; + uint32 va_tcinfo; /* Original data size (excludes header) and + * compression method; see va_extinfo */ + char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */ + } va_compressed; +} varattrib_4b; + +typedef struct +{ + uint8 va_header; + char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Data begins here */ +} varattrib_1b; + +/* TOAST pointers are a subset of varattrib_1b with an identifying tag byte */ +typedef struct +{ + uint8 va_header; /* Always 0x80 or 0x01 */ + uint8 va_tag; /* Type of datum */ + char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Type-specific data */ +} varattrib_1b_e; + +/* + * Bit layouts for varlena headers on big-endian machines: + * + * 00xxxxxx 4-byte length word, aligned, uncompressed data (up to 1G) + * 01xxxxxx 4-byte length word, aligned, *compressed* data (up to 1G) + * 10000000 1-byte length word, unaligned, TOAST pointer + * 1xxxxxxx 1-byte length word, unaligned, uncompressed data (up to 126b) + * + * Bit layouts for varlena headers on little-endian machines: + * + * xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G) + * xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G) + * 00000001 1-byte length word, unaligned, TOAST pointer + * xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to 126b) + * + * The "xxx" bits are the length field (which includes itself in all cases). + * In the big-endian case we mask to extract the length, in the little-endian + * case we shift. Note that in both cases the flag bits are in the physically + * first byte. Also, it is not possible for a 1-byte length word to be zero; + * this lets us disambiguate alignment padding bytes from the start of an + * unaligned datum. (We now *require* pad bytes to be filled with zero!) + * + * In TOAST pointers the va_tag field (see varattrib_1b_e) is used to discern + * the specific type and length of the pointer datum. + */ + +/* + * Endian-dependent macros. These are considered internal --- use the + * external macros below instead of using these directly. + * + * Note: IS_1B is true for external toast records but VARSIZE_1B will return 0 + * for such records. Hence you should usually check for IS_EXTERNAL before + * checking for IS_1B. + */ + +#ifdef WORDS_BIGENDIAN + +#define VARATT_IS_4B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00) +#define VARATT_IS_4B_U(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x00) +#define VARATT_IS_4B_C(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x40) +#define VARATT_IS_1B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x80) +#define VARATT_IS_1B_E(PTR) \ + ((((varattrib_1b *) (PTR))->va_header) == 0x80) +#define VARATT_NOT_PAD_BYTE(PTR) \ + (*((uint8 *) (PTR)) != 0) + +/* VARSIZE_4B() should only be used on known-aligned data */ +#define VARSIZE_4B(PTR) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF) +#define VARSIZE_1B(PTR) \ + (((varattrib_1b *) (PTR))->va_header & 0x7F) +#define VARTAG_1B_E(PTR) \ + (((varattrib_1b_e *) (PTR))->va_tag) + +#define SET_VARSIZE_4B(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF) +#define SET_VARSIZE_4B_C(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000) +#define SET_VARSIZE_1B(PTR,len) \ + (((varattrib_1b *) (PTR))->va_header = (len) | 0x80) +#define SET_VARTAG_1B_E(PTR,tag) \ + (((varattrib_1b_e *) (PTR))->va_header = 0x80, \ + ((varattrib_1b_e *) (PTR))->va_tag = (tag)) + +#else /* !WORDS_BIGENDIAN */ + +#define VARATT_IS_4B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00) +#define VARATT_IS_4B_U(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x00) +#define VARATT_IS_4B_C(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x02) +#define VARATT_IS_1B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x01) +#define VARATT_IS_1B_E(PTR) \ + ((((varattrib_1b *) (PTR))->va_header) == 0x01) +#define VARATT_NOT_PAD_BYTE(PTR) \ + (*((uint8 *) (PTR)) != 0) + +/* VARSIZE_4B() should only be used on known-aligned data */ +#define VARSIZE_4B(PTR) \ + ((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF) +#define VARSIZE_1B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F) +#define VARTAG_1B_E(PTR) \ + (((varattrib_1b_e *) (PTR))->va_tag) + +#define SET_VARSIZE_4B(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2)) +#define SET_VARSIZE_4B_C(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02) +#define SET_VARSIZE_1B(PTR,len) \ + (((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01) +#define SET_VARTAG_1B_E(PTR,tag) \ + (((varattrib_1b_e *) (PTR))->va_header = 0x01, \ + ((varattrib_1b_e *) (PTR))->va_tag = (tag)) + +#endif /* WORDS_BIGENDIAN */ + +#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) +#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data) +#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data) +#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data) + +/* + * Externally visible TOAST macros begin here. + */ + +#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data) +#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data) +#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data) + +#define VARATT_SHORT_MAX 0x7F +#define VARATT_CAN_MAKE_SHORT(PTR) \ + (VARATT_IS_4B_U(PTR) && \ + (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) <= VARATT_SHORT_MAX) +#define VARATT_CONVERTED_SHORT_SIZE(PTR) \ + (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) + +/* + * In consumers oblivious to data alignment, call PG_DETOAST_DATUM_PACKED(), + * VARDATA_ANY(), VARSIZE_ANY() and VARSIZE_ANY_EXHDR(). Elsewhere, call + * PG_DETOAST_DATUM(), VARDATA() and VARSIZE(). Directly fetching an int16, + * int32 or wider field in the struct representing the datum layout requires + * aligned data. memcpy() is alignment-oblivious, as are most operations on + * datatypes, such as text, whose layout struct contains only char fields. + * + * Code assembling a new datum should call VARDATA() and SET_VARSIZE(). + * (Datums begin life untoasted.) + * + * Other macros here should usually be used only by tuple assembly/disassembly + * code and code that specifically wants to work with still-toasted Datums. + */ +#define VARDATA(PTR) VARDATA_4B(PTR) +#define VARSIZE(PTR) VARSIZE_4B(PTR) + +#define VARSIZE_SHORT(PTR) VARSIZE_1B(PTR) +#define VARDATA_SHORT(PTR) VARDATA_1B(PTR) + +#define VARTAG_EXTERNAL(PTR) VARTAG_1B_E(PTR) +#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR))) +#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR) + +#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR) +#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) +#define VARATT_IS_EXTERNAL_ONDISK(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK) +#define VARATT_IS_EXTERNAL_INDIRECT(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT) +#define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RO) +#define VARATT_IS_EXTERNAL_EXPANDED_RW(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RW) +#define VARATT_IS_EXTERNAL_EXPANDED(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR))) +#define VARATT_IS_EXTERNAL_NON_EXPANDED(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && !VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR))) +#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR) +#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR)) + +#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len) +#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len) +#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len) + +#define SET_VARTAG_EXTERNAL(PTR, tag) SET_VARTAG_1B_E(PTR, tag) + +#define VARSIZE_ANY(PTR) \ + (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR) : \ + (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \ + VARSIZE_4B(PTR))) + +/* Size of a varlena data, excluding header */ +#define VARSIZE_ANY_EXHDR(PTR) \ + (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR)-VARHDRSZ_EXTERNAL : \ + (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-VARHDRSZ_SHORT : \ + VARSIZE_4B(PTR)-VARHDRSZ)) + +/* caution: this will not work on an external or compressed-in-line Datum */ +/* caution: this will return a possibly unaligned pointer */ +#define VARDATA_ANY(PTR) \ + (VARATT_IS_1B(PTR) ? VARDATA_1B(PTR) : VARDATA_4B(PTR)) + +/* Decompressed size and compression method of a compressed-in-line Datum */ +#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \ + (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK) +#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \ + (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) + +/* Same for external Datums; but note argument is a struct varatt_external */ +#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \ + ((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK) +#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \ + ((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS) + +#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \ + do { \ + Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \ + (cm) == TOAST_LZ4_COMPRESSION_ID); \ + ((toast_pointer).va_extinfo = \ + (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \ + } while (0) + +/* + * Testing whether an externally-stored value is compressed now requires + * comparing size stored in va_extinfo (the actual length of the external data) + * to rawsize (the original uncompressed datum's size). The latter includes + * VARHDRSZ overhead, the former doesn't. We never use compression unless it + * actually saves space, so we expect either equality or less-than. + */ +#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ + (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \ + (toast_pointer).va_rawsize - VARHDRSZ) + +#endif diff --git a/src/pl/plperl/Util.xs b/src/pl/plperl/Util.xs index bb4580ebfa..0ecb87fe33 100644 --- a/src/pl/plperl/Util.xs +++ b/src/pl/plperl/Util.xs @@ -16,6 +16,7 @@ #include "fmgr.h" #include "utils/builtins.h" #include "utils/bytea.h" /* for byteain & byteaout */ +#include "varatt.h" /* perl stuff */ #define PG_NEED_PERL_XSUB_H diff --git a/src/test/modules/spgist_name_ops/spgist_name_ops.c b/src/test/modules/spgist_name_ops/spgist_name_ops.c index 1ff40d8012..99f9267df9 100644 --- a/src/test/modules/spgist_name_ops/spgist_name_ops.c +++ b/src/test/modules/spgist_name_ops/spgist_name_ops.c @@ -24,6 +24,7 @@ #include "access/spgist.h" #include "catalog/pg_type.h" #include "utils/datum.h" +#include "varatt.h" PG_MODULE_MAGIC; diff --git a/src/test/modules/test_custom_rmgrs/test_custom_rmgrs.c b/src/test/modules/test_custom_rmgrs/test_custom_rmgrs.c index a06a31404a..1727910ce7 100644 --- a/src/test/modules/test_custom_rmgrs/test_custom_rmgrs.c +++ b/src/test/modules/test_custom_rmgrs/test_custom_rmgrs.c @@ -22,6 +22,7 @@ #include "access/xloginsert.h" #include "fmgr.h" #include "utils/pg_lsn.h" +#include "varatt.h" PG_MODULE_MAGIC; diff --git a/src/test/modules/test_shm_mq/test.c b/src/test/modules/test_shm_mq/test.c index cfe3af6631..906e943e2d 100644 --- a/src/test/modules/test_shm_mq/test.c +++ b/src/test/modules/test_shm_mq/test.c @@ -16,6 +16,7 @@ #include "fmgr.h" #include "miscadmin.h" #include "pgstat.h" +#include "varatt.h" #include "test_shm_mq.h"