From 756ab29124d7850d4392e2227b67b69b61576cd6 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 13 Jan 2021 10:33:33 +0200 Subject: [PATCH] Add functions to 'pageinspect' to inspect GiST indexes. Author: Andrey Borodin and me Discussion: https://www.postgresql.org/message-id/3E4F9093-A1B5-4DF8-A292-0B48692E3954%40yandex-team.ru --- contrib/pageinspect/Makefile | 6 +- contrib/pageinspect/expected/gist.out | 87 ++++++ contrib/pageinspect/gistfuncs.c | 287 ++++++++++++++++++ contrib/pageinspect/pageinspect--1.8--1.9.sql | 41 +++ contrib/pageinspect/pageinspect.control | 2 +- contrib/pageinspect/sql/gist.sql | 18 ++ doc/src/sgml/pageinspect.sgml | 89 ++++++ 7 files changed, 527 insertions(+), 3 deletions(-) create mode 100644 contrib/pageinspect/expected/gist.out create mode 100644 contrib/pageinspect/gistfuncs.c create mode 100644 contrib/pageinspect/pageinspect--1.8--1.9.sql create mode 100644 contrib/pageinspect/sql/gist.sql diff --git a/contrib/pageinspect/Makefile b/contrib/pageinspect/Makefile index d9d8177116..4539f0aef7 100644 --- a/contrib/pageinspect/Makefile +++ b/contrib/pageinspect/Makefile @@ -7,19 +7,21 @@ OBJS = \ btreefuncs.o \ fsmfuncs.o \ ginfuncs.o \ + gistfuncs.o \ hashfuncs.o \ heapfuncs.o \ rawpage.o EXTENSION = pageinspect -DATA = pageinspect--1.7--1.8.sql pageinspect--1.6--1.7.sql \ +DATA = pageinspect--1.8--1.9.sql \ + pageinspect--1.7--1.8.sql pageinspect--1.6--1.7.sql \ pageinspect--1.5.sql pageinspect--1.5--1.6.sql \ pageinspect--1.4--1.5.sql pageinspect--1.3--1.4.sql \ pageinspect--1.2--1.3.sql pageinspect--1.1--1.2.sql \ pageinspect--1.0--1.1.sql PGFILEDESC = "pageinspect - functions to inspect contents of database pages" -REGRESS = page btree brin gin hash checksum +REGRESS = page btree brin gin gist hash checksum ifdef USE_PGXS PG_CONFIG = pg_config diff --git a/contrib/pageinspect/expected/gist.out b/contrib/pageinspect/expected/gist.out new file mode 100644 index 0000000000..89294b5a17 --- /dev/null +++ b/contrib/pageinspect/expected/gist.out @@ -0,0 +1,87 @@ +CREATE TABLE test_gist AS SELECT point(i,i) p, i::text t FROM + generate_series(1,1000) i; +CREATE INDEX test_gist_idx ON test_gist USING gist (p); +-- Page 0 is the root, the rest are leaf pages +SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 0)); + lsn | nsn | rightlink | flags +-----+-----+------------+------- + 0/1 | 0/0 | 4294967295 | {} +(1 row) + +SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 1)); + lsn | nsn | rightlink | flags +-----+-----+------------+-------- + 0/1 | 0/0 | 4294967295 | {leaf} +(1 row) + +SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 2)); + lsn | nsn | rightlink | flags +-----+-----+-----------+-------- + 0/1 | 0/0 | 1 | {leaf} +(1 row) + +SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 0), 'test_gist_idx'); + itemoffset | ctid | itemlen | keys +------------+-----------+---------+------------------- + 1 | (1,65535) | 40 | (p)=((166,166)) + 2 | (2,65535) | 40 | (p)=((332,332)) + 3 | (3,65535) | 40 | (p)=((498,498)) + 4 | (4,65535) | 40 | (p)=((664,664)) + 5 | (5,65535) | 40 | (p)=((830,830)) + 6 | (6,65535) | 40 | (p)=((996,996)) + 7 | (7,65535) | 40 | (p)=((1000,1000)) +(7 rows) + +SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 1), 'test_gist_idx') LIMIT 5; + itemoffset | ctid | itemlen | keys +------------+-------+---------+------------- + 1 | (0,1) | 40 | (p)=((1,1)) + 2 | (0,2) | 40 | (p)=((2,2)) + 3 | (0,3) | 40 | (p)=((3,3)) + 4 | (0,4) | 40 | (p)=((4,4)) + 5 | (0,5) | 40 | (p)=((5,5)) +(5 rows) + +SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 2), 'test_gist_idx') LIMIT 5; + itemoffset | ctid | itemlen | keys +------------+--------+---------+----------------- + 1 | (1,10) | 40 | (p)=((167,167)) + 2 | (1,11) | 40 | (p)=((168,168)) + 3 | (1,12) | 40 | (p)=((169,169)) + 4 | (1,13) | 40 | (p)=((170,170)) + 5 | (1,14) | 40 | (p)=((171,171)) +(5 rows) + +SELECT * FROM gist_page_items_bytea(get_raw_page('test_gist_idx', 0)); + itemoffset | ctid | itemlen | key_data +------------+-----------+---------+------------------------------------------------------------------------------------ + 1 | (1,65535) | 40 | \x00000100ffff28000000000000c064400000000000c06440000000000000f03f000000000000f03f + 2 | (2,65535) | 40 | \x00000200ffff28000000000000c074400000000000c074400000000000e064400000000000e06440 + 3 | (3,65535) | 40 | \x00000300ffff28000000000000207f400000000000207f400000000000d074400000000000d07440 + 4 | (4,65535) | 40 | \x00000400ffff28000000000000c084400000000000c084400000000000307f400000000000307f40 + 5 | (5,65535) | 40 | \x00000500ffff28000000000000f089400000000000f089400000000000c884400000000000c88440 + 6 | (6,65535) | 40 | \x00000600ffff28000000000000208f400000000000208f400000000000f889400000000000f88940 + 7 | (7,65535) | 40 | \x00000700ffff28000000000000408f400000000000408f400000000000288f400000000000288f40 +(7 rows) + +SELECT * FROM gist_page_items_bytea(get_raw_page('test_gist_idx', 1)) LIMIT 5; + itemoffset | ctid | itemlen | key_data +------------+-------+---------+------------------------------------------------------------------------------------ + 1 | (0,1) | 40 | \x0000000001002800000000000000f03f000000000000f03f000000000000f03f000000000000f03f + 2 | (0,2) | 40 | \x00000000020028000000000000000040000000000000004000000000000000400000000000000040 + 3 | (0,3) | 40 | \x00000000030028000000000000000840000000000000084000000000000008400000000000000840 + 4 | (0,4) | 40 | \x00000000040028000000000000001040000000000000104000000000000010400000000000001040 + 5 | (0,5) | 40 | \x00000000050028000000000000001440000000000000144000000000000014400000000000001440 +(5 rows) + +SELECT * FROM gist_page_items_bytea(get_raw_page('test_gist_idx', 2)) LIMIT 5; + itemoffset | ctid | itemlen | key_data +------------+--------+---------+------------------------------------------------------------------------------------ + 1 | (1,10) | 40 | \x000001000a0028000000000000e064400000000000e064400000000000e064400000000000e06440 + 2 | (1,11) | 40 | \x000001000b0028000000000000006540000000000000654000000000000065400000000000006540 + 3 | (1,12) | 40 | \x000001000c0028000000000000206540000000000020654000000000002065400000000000206540 + 4 | (1,13) | 40 | \x000001000d0028000000000000406540000000000040654000000000004065400000000000406540 + 5 | (1,14) | 40 | \x000001000e0028000000000000606540000000000060654000000000006065400000000000606540 +(5 rows) + +DROP TABLE test_gist; diff --git a/contrib/pageinspect/gistfuncs.c b/contrib/pageinspect/gistfuncs.c new file mode 100644 index 0000000000..146b2e91b6 --- /dev/null +++ b/contrib/pageinspect/gistfuncs.c @@ -0,0 +1,287 @@ +/* + * gistfuncs.c + * Functions to investigate the content of GiST indexes + * + * Copyright (c) 2014-2020, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/pageinspect/gistfuncs.c + */ +#include "postgres.h" + +#include "access/gist.h" +#include "access/gist_private.h" +#include "access/htup.h" +#include "access/relation.h" +#include "catalog/namespace.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pageinspect.h" +#include "storage/itemptr.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/rel.h" +#include "utils/pg_lsn.h" +#include "utils/varlena.h" + +PG_FUNCTION_INFO_V1(gist_page_opaque_info); +PG_FUNCTION_INFO_V1(gist_page_items); +PG_FUNCTION_INFO_V1(gist_page_items_bytea); + +#define ItemPointerGetDatum(X) PointerGetDatum(X) + + +Datum +gist_page_opaque_info(PG_FUNCTION_ARGS) +{ + bytea *raw_page = PG_GETARG_BYTEA_P(0); + TupleDesc tupdesc; + Page page; + GISTPageOpaque opaq; + HeapTuple resultTuple; + Datum values[4]; + bool nulls[4]; + Datum flags[16]; + int nflags = 0; + uint16 flagbits; + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to use raw page functions"))); + + page = get_page_from_raw(raw_page); + + opaq = (GISTPageOpaque) PageGetSpecialPointer(page); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + /* Convert the flags bitmask to an array of human-readable names */ + flagbits = opaq->flags; + if (flagbits & F_LEAF) + flags[nflags++] = CStringGetTextDatum("leaf"); + if (flagbits & F_DELETED) + flags[nflags++] = CStringGetTextDatum("deleted"); + if (flagbits & F_TUPLES_DELETED) + flags[nflags++] = CStringGetTextDatum("tuples_deleted"); + if (flagbits & F_FOLLOW_RIGHT) + flags[nflags++] = CStringGetTextDatum("follow_right"); + if (flagbits & F_HAS_GARBAGE) + flags[nflags++] = CStringGetTextDatum("has_garbage"); + flagbits &= ~(F_LEAF | F_DELETED | F_TUPLES_DELETED | F_FOLLOW_RIGHT | F_HAS_GARBAGE); + if (flagbits) + { + /* any flags we don't recognize are printed in hex */ + flags[nflags++] = DirectFunctionCall1(to_hex32, Int32GetDatum(flagbits)); + } + + memset(nulls, 0, sizeof(nulls)); + + values[0] = LSNGetDatum(PageGetLSN(page)); + values[1] = LSNGetDatum(GistPageGetNSN(page)); + values[2] = Int64GetDatum(opaq->rightlink); + values[3] = PointerGetDatum(construct_array(flags, nflags, + TEXTOID, + -1, false, TYPALIGN_INT)); + + /* Build and return the result tuple. */ + resultTuple = heap_form_tuple(tupdesc, values, nulls); + + return HeapTupleGetDatum(resultTuple); +} + +typedef struct gist_page_items_state +{ + Page page; + TupleDesc tupd; + OffsetNumber offset; + Relation rel; +} gist_page_items_state; + +Datum +gist_page_items_bytea(PG_FUNCTION_ARGS) +{ + bytea *raw_page = PG_GETARG_BYTEA_P(0); + FuncCallContext *fctx; + gist_page_items_state *inter_call_data; + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to use raw page functions"))); + + if (SRF_IS_FIRSTCALL()) + { + TupleDesc tupdesc; + MemoryContext mctx; + Page page; + + fctx = SRF_FIRSTCALL_INIT(); + mctx = MemoryContextSwitchTo(fctx->multi_call_memory_ctx); + + page = get_page_from_raw(raw_page); + + inter_call_data = palloc(sizeof(gist_page_items_state)); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + if (GistPageIsDeleted(page)) + elog(NOTICE, "page is deleted"); + + inter_call_data->page = page; + inter_call_data->tupd = tupdesc; + inter_call_data->offset = FirstOffsetNumber; + + fctx->max_calls = PageGetMaxOffsetNumber(page); + fctx->user_fctx = inter_call_data; + + MemoryContextSwitchTo(mctx); + } + + fctx = SRF_PERCALL_SETUP(); + inter_call_data = fctx->user_fctx; + + if (fctx->call_cntr < fctx->max_calls) + { + Page page = inter_call_data->page; + OffsetNumber offset = inter_call_data->offset; + HeapTuple resultTuple; + Datum result; + Datum values[4]; + bool nulls[4]; + ItemId id; + IndexTuple itup; + bytea *tuple_bytea; + int tuple_len; + + id = PageGetItemId(page, offset); + + if (!ItemIdIsValid(id)) + elog(ERROR, "invalid ItemId"); + + itup = (IndexTuple) PageGetItem(page, id); + tuple_len = IndexTupleSize(itup); + + memset(nulls, 0, sizeof(nulls)); + + values[0] = DatumGetInt16(offset); + values[1] = ItemPointerGetDatum(&itup->t_tid); + values[2] = Int32GetDatum((int) IndexTupleSize(itup)); + + tuple_bytea = (bytea *) palloc(tuple_len + VARHDRSZ); + SET_VARSIZE(tuple_bytea, tuple_len + VARHDRSZ); + memcpy(VARDATA(tuple_bytea), itup, tuple_len); + values[3] = PointerGetDatum(tuple_bytea); + + /* Build and return the result tuple. */ + resultTuple = heap_form_tuple(inter_call_data->tupd, values, nulls); + result = HeapTupleGetDatum(resultTuple); + + inter_call_data->offset++; + SRF_RETURN_NEXT(fctx, result); + } + + SRF_RETURN_DONE(fctx); +} + +Datum +gist_page_items(PG_FUNCTION_ARGS) +{ + bytea *raw_page = PG_GETARG_BYTEA_P(0); + Oid indexRelid = PG_GETARG_OID(1); + FuncCallContext *fctx; + gist_page_items_state *inter_call_data; + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to use raw page functions"))); + + if (SRF_IS_FIRSTCALL()) + { + Relation indexRel; + TupleDesc tupdesc; + MemoryContext mctx; + Page page; + + fctx = SRF_FIRSTCALL_INIT(); + mctx = MemoryContextSwitchTo(fctx->multi_call_memory_ctx); + + page = get_page_from_raw(raw_page); + + inter_call_data = palloc(sizeof(gist_page_items_state)); + + /* Open the relation */ + indexRel = index_open(indexRelid, AccessShareLock); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + if (GistPageIsDeleted(page)) + elog(NOTICE, "page is deleted"); + + inter_call_data->page = page; + inter_call_data->tupd = tupdesc; + inter_call_data->offset = FirstOffsetNumber; + inter_call_data->rel = indexRel; + + fctx->max_calls = PageGetMaxOffsetNumber(page); + fctx->user_fctx = inter_call_data; + + MemoryContextSwitchTo(mctx); + } + + fctx = SRF_PERCALL_SETUP(); + inter_call_data = fctx->user_fctx; + + if (fctx->call_cntr < fctx->max_calls) + { + Page page = inter_call_data->page; + OffsetNumber offset = inter_call_data->offset; + HeapTuple resultTuple; + Datum result; + Datum values[4]; + bool nulls[4]; + ItemId id; + IndexTuple itup; + Datum itup_values[INDEX_MAX_KEYS]; + bool itup_isnull[INDEX_MAX_KEYS]; + char *key_desc; + + id = PageGetItemId(page, offset); + + if (!ItemIdIsValid(id)) + elog(ERROR, "invalid ItemId"); + + itup = (IndexTuple) PageGetItem(page, id); + + index_deform_tuple(itup, RelationGetDescr(inter_call_data->rel), + itup_values, itup_isnull); + + key_desc = BuildIndexValueDescription(inter_call_data->rel, itup_values, + itup_isnull); + + memset(nulls, 0, sizeof(nulls)); + + values[0] = DatumGetInt16(offset); + values[1] = ItemPointerGetDatum(&itup->t_tid); + values[2] = Int32GetDatum((int) IndexTupleSize(itup)); + values[3] = CStringGetTextDatum(key_desc); + + /* Build and return the result tuple. */ + resultTuple = heap_form_tuple(inter_call_data->tupd, values, nulls); + result = HeapTupleGetDatum(resultTuple); + + inter_call_data->offset++; + SRF_RETURN_NEXT(fctx, result); + } + + relation_close(inter_call_data->rel, AccessShareLock); + + SRF_RETURN_DONE(fctx); +} diff --git a/contrib/pageinspect/pageinspect--1.8--1.9.sql b/contrib/pageinspect/pageinspect--1.8--1.9.sql new file mode 100644 index 0000000000..9dc342fabc --- /dev/null +++ b/contrib/pageinspect/pageinspect--1.8--1.9.sql @@ -0,0 +1,41 @@ +/* contrib/pageinspect/pageinspect--1.8--1.9.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pageinspect UPDATE TO '1.9'" to load this file. \quit + +-- +-- gist_page_opaque_info() +-- +CREATE FUNCTION gist_page_opaque_info(IN page bytea, + OUT lsn pg_lsn, + OUT nsn pg_lsn, + OUT rightlink bigint, + OUT flags text[]) +AS 'MODULE_PATHNAME', 'gist_page_opaque_info' +LANGUAGE C STRICT PARALLEL SAFE; + + +-- +-- gist_page_items_bytea() +-- +CREATE FUNCTION gist_page_items_bytea(IN page bytea, + OUT itemoffset smallint, + OUT ctid tid, + OUT itemlen smallint, + OUT key_data bytea) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gist_page_items_bytea' +LANGUAGE C STRICT PARALLEL SAFE; + +-- +-- gist_page_items() +-- +CREATE FUNCTION gist_page_items(IN page bytea, + IN index_oid regclass, + OUT itemoffset smallint, + OUT ctid tid, + OUT itemlen smallint, + OUT keys text) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gist_page_items' +LANGUAGE C STRICT PARALLEL SAFE; diff --git a/contrib/pageinspect/pageinspect.control b/contrib/pageinspect/pageinspect.control index f8cdf526c6..bd716769a1 100644 --- a/contrib/pageinspect/pageinspect.control +++ b/contrib/pageinspect/pageinspect.control @@ -1,5 +1,5 @@ # pageinspect extension comment = 'inspect the contents of database pages at a low level' -default_version = '1.8' +default_version = '1.9' module_pathname = '$libdir/pageinspect' relocatable = true diff --git a/contrib/pageinspect/sql/gist.sql b/contrib/pageinspect/sql/gist.sql new file mode 100644 index 0000000000..b38da4cb03 --- /dev/null +++ b/contrib/pageinspect/sql/gist.sql @@ -0,0 +1,18 @@ +CREATE TABLE test_gist AS SELECT point(i,i) p, i::text t FROM + generate_series(1,1000) i; +CREATE INDEX test_gist_idx ON test_gist USING gist (p); + +-- Page 0 is the root, the rest are leaf pages +SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 0)); +SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 1)); +SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 2)); + +SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 0), 'test_gist_idx'); +SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 1), 'test_gist_idx') LIMIT 5; +SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 2), 'test_gist_idx') LIMIT 5; + +SELECT * FROM gist_page_items_bytea(get_raw_page('test_gist_idx', 0)); +SELECT * FROM gist_page_items_bytea(get_raw_page('test_gist_idx', 1)) LIMIT 5; +SELECT * FROM gist_page_items_bytea(get_raw_page('test_gist_idx', 2)) LIMIT 5; + +DROP TABLE test_gist; diff --git a/doc/src/sgml/pageinspect.sgml b/doc/src/sgml/pageinspect.sgml index 687c3606ba..35858e1557 100644 --- a/doc/src/sgml/pageinspect.sgml +++ b/doc/src/sgml/pageinspect.sgml @@ -671,6 +671,95 @@ test=# SELECT first_tid, nbytes, tids[0:5] AS some_tids + + GiST Functions + + + + + gist_page_opaque_info(page bytea) returns record + + gist_page_opaque_info + + + + + + gist_page_opaque_info returns information about + a GiST index opaque area, like the NSN, rightlink and + page type. + For example: + +test=# SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 2)); + lsn | nsn | rightlink | flags +-----+-----+-----------+-------- + 0/1 | 0/0 | 1 | {leaf} +(1 row) + + + + + + + + gist_page_items(page bytea, index oid) returns setof record + + gist_page_items + + + + + + gist_page_items returns information about + the data stored in a page of GiST index. For example: + +test=# SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 0), 'test_gist_idx'); + itemoffset | ctid | itemlen | keys +------------+-----------+---------+------------------- + 1 | (1,65535) | 40 | (p)=((166,166)) + 2 | (2,65535) | 40 | (p)=((332,332)) + 3 | (3,65535) | 40 | (p)=((498,498)) + 4 | (4,65535) | 40 | (p)=((664,664)) + 5 | (5,65535) | 40 | (p)=((830,830)) + 6 | (6,65535) | 40 | (p)=((996,996)) + 7 | (7,65535) | 40 | (p)=((1000,1000)) +(7 rows) + + + + + + + + gist_page_items_bytea(page bytea) returns setof record + + gist_page_items_bytea + + + + + + Same as gist_page_items, but returns the key data as a raw + bytea blob. For example: + +test=# SELECT * FROM gist_page_items_bytea(get_raw_page('test_gist_idx', 0)); + itemoffset | ctid | itemlen | key_data +------------+-----------+---------+------------------------------------------------------------------------------------ + 1 | (1,65535) | 40 | \x00000100ffff28000000000000c064400000000000c06440000000000000f03f000000000000f03f + 2 | (2,65535) | 40 | \x00000200ffff28000000000000c074400000000000c074400000000000e064400000000000e06440 + 3 | (3,65535) | 40 | \x00000300ffff28000000000000207f400000000000207f400000000000d074400000000000d07440 + 4 | (4,65535) | 40 | \x00000400ffff28000000000000c084400000000000c084400000000000307f400000000000307f40 + 5 | (5,65535) | 40 | \x00000500ffff28000000000000f089400000000000f089400000000000c884400000000000c88440 + 6 | (6,65535) | 40 | \x00000600ffff28000000000000208f400000000000208f400000000000f889400000000000f88940 + 7 | (7,65535) | 40 | \x00000700ffff28000000000000408f400000000000408f400000000000288f400000000000288f40 +(7 rows) + + + + + + + Hash Functions