diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index fc37ceb4a3..445a7ed9fb 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -44,9 +44,6 @@ #undef TOAST_DEBUG -/* Size of an EXTERNAL datum that contains a standard TOAST pointer */ -#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_external)) - /* * Testing whether an externally-stored value is compressed now requires * comparing extsize (the actual length of the external data) to rawsize @@ -87,11 +84,11 @@ static struct varlena *toast_fetch_datum_slice(struct varlena * attr, * heap_tuple_fetch_attr - * * Public entry point to get back a toasted value from - * external storage (possibly still in compressed format). + * external source (possibly still in compressed format). * * This will return a datum that contains all the data internally, ie, not - * relying on external storage, but it can still be compressed or have a short - * header. + * relying on external storage or memory, but it can still be compressed or + * have a short header. ---------- */ struct varlena * @@ -99,13 +96,35 @@ heap_tuple_fetch_attr(struct varlena * attr) { struct varlena *result; - if (VARATT_IS_EXTERNAL(attr)) + if (VARATT_IS_EXTERNAL_ONDISK(attr)) { /* * This is an external stored plain value */ result = toast_fetch_datum(attr); } + else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) + { + /* + * copy into the caller's memory context. That's not required in all + * cases but sufficient for now since this is mainly used when we need + * to persist a Datum for unusually long time, like in a HOLD cursor. + */ + struct varatt_indirect redirect; + VARATT_EXTERNAL_GET_POINTER(redirect, attr); + attr = (struct varlena *)redirect.pointer; + + /* nested indirect Datums aren't allowed */ + Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr)); + + /* doesn't make much sense, but better handle it */ + if (VARATT_IS_EXTERNAL_ONDISK(attr)) + return heap_tuple_fetch_attr(attr); + + /* copy datum verbatim */ + result = (struct varlena *) palloc(VARSIZE_ANY(attr)); + memcpy(result, attr, VARSIZE_ANY(attr)); + } else { /* @@ -128,7 +147,7 @@ heap_tuple_fetch_attr(struct varlena * attr) struct varlena * heap_tuple_untoast_attr(struct varlena * attr) { - if (VARATT_IS_EXTERNAL(attr)) + if (VARATT_IS_EXTERNAL_ONDISK(attr)) { /* * This is an externally stored datum --- fetch it back from there @@ -145,6 +164,17 @@ heap_tuple_untoast_attr(struct varlena * attr) pfree(tmp); } } + else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) + { + struct varatt_indirect redirect; + VARATT_EXTERNAL_GET_POINTER(redirect, attr); + attr = (struct varlena *)redirect.pointer; + + /* nested indirect Datums aren't allowed */ + Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr)); + + attr = heap_tuple_untoast_attr(attr); + } else if (VARATT_IS_COMPRESSED(attr)) { /* @@ -191,7 +221,7 @@ heap_tuple_untoast_attr_slice(struct varlena * attr, char *attrdata; int32 attrsize; - if (VARATT_IS_EXTERNAL(attr)) + if (VARATT_IS_EXTERNAL_ONDISK(attr)) { struct varatt_external toast_pointer; @@ -204,6 +234,17 @@ heap_tuple_untoast_attr_slice(struct varlena * attr, /* fetch it back (compressed marker will get set automatically) */ preslice = toast_fetch_datum(attr); } + else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) + { + struct varatt_indirect redirect; + VARATT_EXTERNAL_GET_POINTER(redirect, attr); + + /* nested indirect Datums aren't allowed */ + Assert(!VARATT_IS_EXTERNAL_INDIRECT(redirect.pointer)); + + return heap_tuple_untoast_attr_slice(redirect.pointer, + sliceoffset, slicelength); + } else preslice = attr; @@ -267,7 +308,7 @@ toast_raw_datum_size(Datum value) struct varlena *attr = (struct varlena *) DatumGetPointer(value); Size result; - if (VARATT_IS_EXTERNAL(attr)) + if (VARATT_IS_EXTERNAL_ONDISK(attr)) { /* va_rawsize is the size of the original datum -- including header */ struct varatt_external toast_pointer; @@ -275,6 +316,16 @@ toast_raw_datum_size(Datum value) VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); result = toast_pointer.va_rawsize; } + else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) + { + struct varatt_indirect toast_pointer; + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + + /* nested indirect Datums aren't allowed */ + Assert(!VARATT_IS_EXTERNAL_INDIRECT(toast_pointer.pointer)); + + return toast_raw_datum_size(PointerGetDatum(toast_pointer.pointer)); + } else if (VARATT_IS_COMPRESSED(attr)) { /* here, va_rawsize is just the payload size */ @@ -308,7 +359,7 @@ toast_datum_size(Datum value) struct varlena *attr = (struct varlena *) DatumGetPointer(value); Size result; - if (VARATT_IS_EXTERNAL(attr)) + if (VARATT_IS_EXTERNAL_ONDISK(attr)) { /* * Attribute is stored externally - return the extsize whether @@ -320,6 +371,16 @@ toast_datum_size(Datum value) VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); result = toast_pointer.va_extsize; } + else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) + { + struct varatt_indirect toast_pointer; + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + + /* nested indirect Datums aren't allowed */ + Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr)); + + return toast_datum_size(PointerGetDatum(toast_pointer.pointer)); + } else if (VARATT_IS_SHORT(attr)) { result = VARSIZE_SHORT(attr); @@ -387,8 +448,12 @@ toast_delete(Relation rel, HeapTuple oldtup) { Datum value = toast_values[i]; - if (!toast_isnull[i] && VARATT_IS_EXTERNAL(PointerGetDatum(value))) + if (toast_isnull[i]) + continue; + else if (VARATT_IS_EXTERNAL_ONDISK(PointerGetDatum(value))) toast_delete_datum(rel, value); + else if (VARATT_IS_EXTERNAL_INDIRECT(PointerGetDatum(value))) + elog(ERROR, "attempt to delete tuple containing indirect datums"); } } } @@ -490,13 +555,13 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, new_value = (struct varlena *) DatumGetPointer(toast_values[i]); /* - * If the old value is an external stored one, check if it has - * changed so we have to delete it later. + * If the old value is stored on disk, check if it has changed so + * we have to delete it later. */ if (att[i]->attlen == -1 && !toast_oldisnull[i] && - VARATT_IS_EXTERNAL(old_value)) + VARATT_IS_EXTERNAL_ONDISK(old_value)) { - if (toast_isnull[i] || !VARATT_IS_EXTERNAL(new_value) || + if (toast_isnull[i] || !VARATT_IS_EXTERNAL_ONDISK(new_value) || memcmp((char *) old_value, (char *) new_value, VARSIZE_EXTERNAL(old_value)) != 0) { @@ -1258,6 +1323,8 @@ toast_save_datum(Relation rel, Datum value, int32 data_todo; Pointer dval = DatumGetPointer(value); + Assert(!VARATT_IS_EXTERNAL(value)); + /* * Open the toast relation and its index. We can use the index to check * uniqueness of the OID we assign to the toasted item, even though it has @@ -1341,7 +1408,7 @@ toast_save_datum(Relation rel, Datum value, { struct varatt_external old_toast_pointer; - Assert(VARATT_IS_EXTERNAL(oldexternal)); + Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); /* Must copy to access aligned fields */ VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) @@ -1456,7 +1523,7 @@ toast_save_datum(Relation rel, Datum value, * Create the TOAST pointer value that we'll return */ result = (struct varlena *) palloc(TOAST_POINTER_SIZE); - SET_VARSIZE_EXTERNAL(result, TOAST_POINTER_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK); memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); return PointerGetDatum(result); @@ -1480,7 +1547,7 @@ toast_delete_datum(Relation rel, Datum value) SysScanDesc toastscan; HeapTuple toasttup; - if (!VARATT_IS_EXTERNAL(attr)) + if (!VARATT_IS_EXTERNAL_ONDISK(attr)) return; /* Must copy to access aligned fields */ @@ -1608,6 +1675,9 @@ toast_fetch_datum(struct varlena * attr) char *chunkdata; int32 chunksize; + if (VARATT_IS_EXTERNAL_INDIRECT(attr)) + elog(ERROR, "shouldn't be called for indirect tuples"); + /* Must copy to access aligned fields */ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); @@ -1775,7 +1845,7 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length) int32 chcpystrt; int32 chcpyend; - Assert(VARATT_IS_EXTERNAL(attr)); + Assert(VARATT_IS_EXTERNAL_ONDISK(attr)); /* Must copy to access aligned fields */ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h index 6f4fc4545d..d0c17fde03 100644 --- a/src/include/access/tuptoaster.h +++ b/src/include/access/tuptoaster.h @@ -94,6 +94,11 @@ sizeof(int32) - \ VARHDRSZ) +/* Size of an EXTERNAL datum that contains a standard TOAST pointer */ +#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_external)) + +/* Size of an indirect datum that contains an indirect TOAST pointer */ +#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_indirect)) /* ---------- * toast_insert_or_update - diff --git a/src/include/postgres.h b/src/include/postgres.h index 30e1dee187..cb9d196c51 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -54,23 +54,53 @@ */ /* - * struct varatt_external is a "TOAST pointer", that is, the information - * needed to fetch a stored-out-of-line Datum. The data is compressed - * if and only if va_extsize < va_rawsize - VARHDRSZ. This struct must not - * contain any padding, because we sometimes compare pointers using memcmp. + * struct varatt_external is a "TOAST pointer", that is, the information needed + * to fetch a Datum stored in an out-of-line on-disk Datum. The data is + * compressed if and only if va_extsize < va_rawsize - VARHDRSZ. This struct + * must not contain any padding, because we sometimes compare pointers using + * memcmp. * * Note that this information is stored unaligned within actual tuples, so * you need to memcpy from the tuple into a local struct variable before * you can look at these fields! (The reason we use memcmp is to avoid * having to do that just to detect equality of two TOAST pointers...) */ -struct varatt_external +typedef struct varatt_external { int32 va_rawsize; /* Original data size (includes header) */ int32 va_extsize; /* External saved size (doesn't) */ Oid va_valueid; /* Unique ID of value within TOAST table */ Oid va_toastrelid; /* RelID of TOAST table containing it */ -}; +} varatt_external; + +/* + * Out-of-line Datum thats stored in memory in contrast to varatt_external + * pointers which points to data in an external toast relation. + * + * Note that just as varatt_external's this is stored unaligned within the + * tuple. + */ +typedef struct varatt_indirect +{ + struct varlena *pointer; /* Pointer to in-memory varlena */ +} varatt_indirect; + + +/* + * Type of external toast datum stored. The peculiar value for VARTAG_ONDISK + * comes from the requirement for on-disk compatibility with the older + * definitions of varattrib_1b_e where v_tag was named va_len_1be... + */ +typedef enum vartag_external +{ + VARTAG_INDIRECT = 1, + VARTAG_ONDISK = 18 +} vartag_external; + +#define VARTAG_SIZE(tag) \ + ((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \ + (tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \ + TrapMacro(true, "unknown vartag")) /* * These structs describe the header of a varlena object that may have been @@ -102,11 +132,12 @@ typedef struct char va_data[1]; /* Data begins here */ } varattrib_1b; +/* inline portion of a short varlena pointing to an external resource */ typedef struct { uint8 va_header; /* Always 0x80 or 0x01 */ - uint8 va_len_1be; /* Physical length of datum */ - char va_data[1]; /* Data (for now always a TOAST pointer) */ + uint8 va_tag; /* Type of datum */ + char va_data[1]; /* Data (of the type indicated by va_tag) */ } varattrib_1b_e; /* @@ -130,6 +161,9 @@ typedef struct * first byte. Also, it is not possible for a 1-byte length word to be zero; * this lets us disambiguate alignment padding bytes from the start of an * unaligned datum. (We now *require* pad bytes to be filled with zero!) + * + * In TOAST datums the tag field in varattrib_1b_e is used to discern whether + * its an indirection pointer or more commonly an on-disk tuple. */ /* @@ -161,8 +195,8 @@ typedef struct (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF) #define VARSIZE_1B(PTR) \ (((varattrib_1b *) (PTR))->va_header & 0x7F) -#define VARSIZE_1B_E(PTR) \ - (((varattrib_1b_e *) (PTR))->va_len_1be) +#define VARTAG_1B_E(PTR) \ + (((varattrib_1b_e *) (PTR))->va_tag) #define SET_VARSIZE_4B(PTR,len) \ (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF) @@ -170,9 +204,9 @@ typedef struct (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000) #define SET_VARSIZE_1B(PTR,len) \ (((varattrib_1b *) (PTR))->va_header = (len) | 0x80) -#define SET_VARSIZE_1B_E(PTR,len) \ +#define SET_VARTAG_1B_E(PTR,tag) \ (((varattrib_1b_e *) (PTR))->va_header = 0x80, \ - ((varattrib_1b_e *) (PTR))->va_len_1be = (len)) + ((varattrib_1b_e *) (PTR))->va_tag = (tag)) #else /* !WORDS_BIGENDIAN */ #define VARATT_IS_4B(PTR) \ @@ -193,8 +227,8 @@ typedef struct ((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF) #define VARSIZE_1B(PTR) \ ((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F) -#define VARSIZE_1B_E(PTR) \ - (((varattrib_1b_e *) (PTR))->va_len_1be) +#define VARTAG_1B_E(PTR) \ + (((varattrib_1b_e *) (PTR))->va_tag) #define SET_VARSIZE_4B(PTR,len) \ (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2)) @@ -202,12 +236,12 @@ typedef struct (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02) #define SET_VARSIZE_1B(PTR,len) \ (((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01) -#define SET_VARSIZE_1B_E(PTR,len) \ +#define SET_VARTAG_1B_E(PTR,tag) \ (((varattrib_1b_e *) (PTR))->va_header = 0x01, \ - ((varattrib_1b_e *) (PTR))->va_len_1be = (len)) + ((varattrib_1b_e *) (PTR))->va_tag = (tag)) #endif /* WORDS_BIGENDIAN */ -#define VARHDRSZ_SHORT 1 +#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data) #define VARATT_SHORT_MAX 0x7F #define VARATT_CAN_MAKE_SHORT(PTR) \ (VARATT_IS_4B_U(PTR) && \ @@ -215,7 +249,7 @@ typedef struct #define VARATT_CONVERTED_SHORT_SIZE(PTR) \ (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) -#define VARHDRSZ_EXTERNAL 2 +#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data) #define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) #define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data) @@ -249,26 +283,32 @@ typedef struct #define VARSIZE_SHORT(PTR) VARSIZE_1B(PTR) #define VARDATA_SHORT(PTR) VARDATA_1B(PTR) -#define VARSIZE_EXTERNAL(PTR) VARSIZE_1B_E(PTR) +#define VARTAG_EXTERNAL(PTR) VARTAG_1B_E(PTR) +#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR))) #define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR) #define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR) #define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) +#define VARATT_IS_EXTERNAL_ONDISK(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK) +#define VARATT_IS_EXTERNAL_INDIRECT(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT) #define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR) #define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR)) #define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len) #define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len) #define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len) -#define SET_VARSIZE_EXTERNAL(PTR, len) SET_VARSIZE_1B_E(PTR, len) + +#define SET_VARTAG_EXTERNAL(PTR, tag) SET_VARTAG_1B_E(PTR, tag) #define VARSIZE_ANY(PTR) \ - (VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR) : \ + (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR) : \ (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \ VARSIZE_4B(PTR))) #define VARSIZE_ANY_EXHDR(PTR) \ - (VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-VARHDRSZ_EXTERNAL : \ + (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR)-VARHDRSZ_EXTERNAL : \ (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-VARHDRSZ_SHORT : \ VARSIZE_4B(PTR)-VARHDRSZ)) diff --git a/src/test/regress/expected/indirect_toast.out b/src/test/regress/expected/indirect_toast.out new file mode 100644 index 0000000000..4f4bf41973 --- /dev/null +++ b/src/test/regress/expected/indirect_toast.out @@ -0,0 +1,151 @@ +CREATE TABLE toasttest(descr text, cnt int DEFAULT 0, f1 text, f2 text); +INSERT INTO toasttest(descr, f1, f2) VALUES('two-compressed', repeat('1234567890',1000), repeat('1234567890',1000)); +INSERT INTO toasttest(descr, f1, f2) VALUES('two-toasted', repeat('1234567890',30000), repeat('1234567890',50000)); +INSERT INTO toasttest(descr, f1, f2) VALUES('one-compressed,one-null', NULL, repeat('1234567890',1000)); +INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted,one-null', NULL, repeat('1234567890',50000)); +-- check whether indirect tuples works on the most basic level +SELECT descr, substring(make_tuple_indirect(toasttest)::text, 1, 200) FROM toasttest; + descr | substring +-------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + two-compressed | (two-compressed,0,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012 + two-toasted | (two-toasted,0,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 + one-compressed,one-null | ("one-compressed,one-null",0,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + one-toasted,one-null | ("one-toasted,one-null",0,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +-- modification without changing varlenas +UPDATE toasttest SET cnt = cnt +1 RETURNING substring(toasttest::text, 1, 200); + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,1,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012 + (two-toasted,1,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 + ("one-compressed,one-null",1,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",1,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +-- modification without modifying asigned value +UPDATE toasttest SET cnt = cnt +1, f1 = f1 RETURNING substring(toasttest::text, 1, 200); + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,2,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012 + (two-toasted,2,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 + ("one-compressed,one-null",2,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",2,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +-- modification modifying, but effectively not changing +UPDATE toasttest SET cnt = cnt +1, f1 = f1||'' RETURNING substring(toasttest::text, 1, 200); + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,3,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012 + (two-toasted,3,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 + ("one-compressed,one-null",3,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",3,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +UPDATE toasttest SET cnt = cnt +1, f1 = '-'||f1||'-' RETURNING substring(toasttest::text, 1, 200); + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901 + (two-toasted,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234 + ("one-compressed,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +SELECT substring(toasttest::text, 1, 200) FROM toasttest; + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901 + (two-toasted,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234 + ("one-compressed,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +-- check we didn't screw with main/toast tuple visiblity +VACUUM FREEZE toasttest; +SELECT substring(toasttest::text, 1, 200) FROM toasttest; + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901 + (two-toasted,4,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234 + ("one-compressed,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",4,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +-- now create a trigger that forces all Datums to be indirect ones +CREATE FUNCTION update_using_indirect() + RETURNS trigger + LANGUAGE plpgsql AS $$ +BEGIN + NEW := make_tuple_indirect(NEW); + RETURN NEW; +END$$; +CREATE TRIGGER toasttest_update_indirect + BEFORE INSERT OR UPDATE + ON toasttest + FOR EACH ROW + EXECUTE PROCEDURE update_using_indirect(); +-- modification without changing varlenas +UPDATE toasttest SET cnt = cnt +1 RETURNING substring(toasttest::text, 1, 200); + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,5,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901 + (two-toasted,5,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234 + ("one-compressed,one-null",5,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",5,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +-- modification without modifying asigned value +UPDATE toasttest SET cnt = cnt +1, f1 = f1 RETURNING substring(toasttest::text, 1, 200); + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,6,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901 + (two-toasted,6,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234 + ("one-compressed,one-null",6,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",6,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +-- modification modifying, but effectively not changing +UPDATE toasttest SET cnt = cnt +1, f1 = f1||'' RETURNING substring(toasttest::text, 1, 200); + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,7,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901 + (two-toasted,7,-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234 + ("one-compressed,one-null",7,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",7,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +UPDATE toasttest SET cnt = cnt +1, f1 = '-'||f1||'-' RETURNING substring(toasttest::text, 1, 200); + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + (two-toasted,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 + ("one-compressed,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 +(4 rows) + +INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted,one-null, via indirect', repeat('1234567890',30000), NULL); +SELECT substring(toasttest::text, 1, 200) FROM toasttest; + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + (two-toasted,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 + ("one-compressed,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 + ("one-toasted,one-null, via indirect",0,1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +(5 rows) + +-- check we didn't screw with main/toast tuple visiblity +VACUUM FREEZE toasttest; +SELECT substring(toasttest::text, 1, 200) FROM toasttest; + substring +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (two-compressed,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + (two-toasted,8,--123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 + ("one-compressed,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 + ("one-toasted,one-null",8,,12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123 + ("one-toasted,one-null, via indirect",0,1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 +(5 rows) + +DROP TABLE toasttest; +DROP FUNCTION update_using_indirect(); diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index a72dd9861c..aef1518287 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -52,6 +52,11 @@ CREATE FUNCTION set_ttdummy (int4) AS '@libdir@/regress@DLSUFFIX@' LANGUAGE C STRICT; +CREATE FUNCTION make_tuple_indirect (record) + RETURNS record + AS '@libdir@/regress@DLSUFFIX@' + LANGUAGE C STRICT; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 61b87ed953..9761d127e1 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -47,6 +47,10 @@ CREATE FUNCTION set_ttdummy (int4) RETURNS int4 AS '@libdir@/regress@DLSUFFIX@' LANGUAGE C STRICT; +CREATE FUNCTION make_tuple_indirect (record) + RETURNS record + AS '@libdir@/regress@DLSUFFIX@' + LANGUAGE C STRICT; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 2af28b1502..4bb9cc7857 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -98,7 +98,7 @@ test: event_trigger # ---------- # Another group of parallel tests # ---------- -test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock json +test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock json indirect_toast # ---------- # Another group of parallel tests diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index e5136cfa7c..3bd8a15285 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -7,7 +7,9 @@ #include #include +#include "access/htup_details.h" #include "access/transam.h" +#include "access/tuptoaster.h" #include "access/xact.h" #include "catalog/pg_type.h" #include "commands/sequence.h" @@ -17,6 +19,8 @@ #include "utils/builtins.h" #include "utils/geo_decls.h" #include "utils/rel.h" +#include "utils/typcache.h" +#include "utils/memutils.h" #define P_MAXDIG 12 @@ -35,6 +39,7 @@ extern char *reverse_name(char *string); extern int oldstyle_length(int n, text *t); extern Datum int44in(PG_FUNCTION_ARGS); extern Datum int44out(PG_FUNCTION_ARGS); +extern Datum make_tuple_indirect(PG_FUNCTION_ARGS); #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; @@ -737,3 +742,90 @@ int44out(PG_FUNCTION_ARGS) *--walk = '\0'; PG_RETURN_CSTRING(result); } + +PG_FUNCTION_INFO_V1(make_tuple_indirect); +Datum +make_tuple_indirect(PG_FUNCTION_ARGS) +{ + HeapTupleHeader rec = PG_GETARG_HEAPTUPLEHEADER(0); + HeapTupleData tuple; + int ncolumns; + Datum *values; + bool *nulls; + + Oid tupType; + int32 tupTypmod; + TupleDesc tupdesc; + + HeapTuple newtup; + + int i; + + MemoryContext old_context; + + /* Extract type info from the tuple itself */ + tupType = HeapTupleHeaderGetTypeId(rec); + tupTypmod = HeapTupleHeaderGetTypMod(rec); + tupdesc = lookup_rowtype_tupdesc(tupType, tupTypmod); + ncolumns = tupdesc->natts; + + /* Build a temporary HeapTuple control structure */ + tuple.t_len = HeapTupleHeaderGetDatumLength(rec); + ItemPointerSetInvalid(&(tuple.t_self)); + tuple.t_tableOid = InvalidOid; + tuple.t_data = rec; + + values = (Datum *) palloc(ncolumns * sizeof(Datum)); + nulls = (bool *) palloc(ncolumns * sizeof(bool)); + + heap_deform_tuple(&tuple, tupdesc, values, nulls); + + old_context = MemoryContextSwitchTo(TopTransactionContext); + + for (i = 0; i < ncolumns; i++) + { + struct varlena *attr; + struct varlena *new_attr; + struct varatt_indirect redirect_pointer; + + /* only work on existing, not-null varlenas */ + if (tupdesc->attrs[i]->attisdropped || + nulls[i] || + tupdesc->attrs[i]->attlen != -1) + continue; + + attr = (struct varlena *) DatumGetPointer(values[i]); + + /* don't recursively indirect */ + if (VARATT_IS_EXTERNAL_INDIRECT(attr)) + continue; + + /* copy datum, so it still lives later */ + if (VARATT_IS_EXTERNAL_ONDISK(attr)) + attr = heap_tuple_fetch_attr(attr); + else + { + struct varlena *oldattr = attr; + attr = palloc0(VARSIZE_ANY(oldattr)); + memcpy(attr, oldattr, VARSIZE_ANY(oldattr)); + } + + /* build indirection Datum */ + new_attr = (struct varlena *) palloc0(INDIRECT_POINTER_SIZE); + redirect_pointer.pointer = attr; + SET_VARTAG_EXTERNAL(new_attr, VARTAG_INDIRECT); + memcpy(VARDATA_EXTERNAL(new_attr), &redirect_pointer, + sizeof(redirect_pointer)); + + values[i] = PointerGetDatum(new_attr); + } + + newtup = heap_form_tuple(tupdesc, values, nulls); + pfree(values); + pfree(nulls); + ReleaseTupleDesc(tupdesc); + + MemoryContextSwitchTo(old_context); + + PG_RETURN_HEAPTUPLEHEADER(newtup->t_data); +} diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index d6eaa7aa4d..ceeca734d3 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -117,6 +117,7 @@ test: xmlmap test: functional_deps test: advisory_lock test: json +test: indirect_toast test: plancache test: limit test: plpgsql diff --git a/src/test/regress/sql/indirect_toast.sql b/src/test/regress/sql/indirect_toast.sql new file mode 100644 index 0000000000..d502480ad3 --- /dev/null +++ b/src/test/regress/sql/indirect_toast.sql @@ -0,0 +1,61 @@ +CREATE TABLE toasttest(descr text, cnt int DEFAULT 0, f1 text, f2 text); + +INSERT INTO toasttest(descr, f1, f2) VALUES('two-compressed', repeat('1234567890',1000), repeat('1234567890',1000)); +INSERT INTO toasttest(descr, f1, f2) VALUES('two-toasted', repeat('1234567890',30000), repeat('1234567890',50000)); +INSERT INTO toasttest(descr, f1, f2) VALUES('one-compressed,one-null', NULL, repeat('1234567890',1000)); +INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted,one-null', NULL, repeat('1234567890',50000)); + +-- check whether indirect tuples works on the most basic level +SELECT descr, substring(make_tuple_indirect(toasttest)::text, 1, 200) FROM toasttest; + +-- modification without changing varlenas +UPDATE toasttest SET cnt = cnt +1 RETURNING substring(toasttest::text, 1, 200); + +-- modification without modifying asigned value +UPDATE toasttest SET cnt = cnt +1, f1 = f1 RETURNING substring(toasttest::text, 1, 200); + +-- modification modifying, but effectively not changing +UPDATE toasttest SET cnt = cnt +1, f1 = f1||'' RETURNING substring(toasttest::text, 1, 200); + +UPDATE toasttest SET cnt = cnt +1, f1 = '-'||f1||'-' RETURNING substring(toasttest::text, 1, 200); + +SELECT substring(toasttest::text, 1, 200) FROM toasttest; +-- check we didn't screw with main/toast tuple visiblity +VACUUM FREEZE toasttest; +SELECT substring(toasttest::text, 1, 200) FROM toasttest; + +-- now create a trigger that forces all Datums to be indirect ones +CREATE FUNCTION update_using_indirect() + RETURNS trigger + LANGUAGE plpgsql AS $$ +BEGIN + NEW := make_tuple_indirect(NEW); + RETURN NEW; +END$$; + +CREATE TRIGGER toasttest_update_indirect + BEFORE INSERT OR UPDATE + ON toasttest + FOR EACH ROW + EXECUTE PROCEDURE update_using_indirect(); + +-- modification without changing varlenas +UPDATE toasttest SET cnt = cnt +1 RETURNING substring(toasttest::text, 1, 200); + +-- modification without modifying asigned value +UPDATE toasttest SET cnt = cnt +1, f1 = f1 RETURNING substring(toasttest::text, 1, 200); + +-- modification modifying, but effectively not changing +UPDATE toasttest SET cnt = cnt +1, f1 = f1||'' RETURNING substring(toasttest::text, 1, 200); + +UPDATE toasttest SET cnt = cnt +1, f1 = '-'||f1||'-' RETURNING substring(toasttest::text, 1, 200); + +INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted,one-null, via indirect', repeat('1234567890',30000), NULL); + +SELECT substring(toasttest::text, 1, 200) FROM toasttest; +-- check we didn't screw with main/toast tuple visiblity +VACUUM FREEZE toasttest; +SELECT substring(toasttest::text, 1, 200) FROM toasttest; + +DROP TABLE toasttest; +DROP FUNCTION update_using_indirect();