postgresql/contrib/tsearch/gistidx.c

788 lines
16 KiB
C

#include "postgres.h"
#include <float.h>
#include "access/gist.h"
#include "access/itup.h"
#include "access/rtree.h"
#include "utils/elog.h"
#include "utils/palloc.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "storage/bufpage.h"
#include "access/tuptoaster.h"
#include "txtidx.h"
#include "query.h"
#include "gistidx.h"
#include "crc32.h"
PG_FUNCTION_INFO_V1(gtxtidx_in);
Datum gtxtidx_in(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(gtxtidx_out);
Datum gtxtidx_out(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(gtxtidx_compress);
Datum gtxtidx_compress(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(gtxtidx_decompress);
Datum gtxtidx_decompress(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(gtxtidx_consistent);
Datum gtxtidx_consistent(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(gtxtidx_union);
Datum gtxtidx_union(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(gtxtidx_same);
Datum gtxtidx_same(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(gtxtidx_penalty);
Datum gtxtidx_penalty(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(gtxtidx_picksplit);
Datum gtxtidx_picksplit(PG_FUNCTION_ARGS);
#define GETENTRY(vec,pos) ((GISTTYPE *) DatumGetPointer(((GISTENTRY *) VARDATA(vec))[(pos)].key))
#define SUMBIT(val) ( \
GETBITBYTE(val,0) + \
GETBITBYTE(val,1) + \
GETBITBYTE(val,2) + \
GETBITBYTE(val,3) + \
GETBITBYTE(val,4) + \
GETBITBYTE(val,5) + \
GETBITBYTE(val,6) + \
GETBITBYTE(val,7) \
)
Datum
gtxtidx_in(PG_FUNCTION_ARGS)
{
elog(ERROR, "Not implemented");
PG_RETURN_DATUM(0);
}
Datum
gtxtidx_out(PG_FUNCTION_ARGS)
{
elog(ERROR, "Not implemented");
PG_RETURN_DATUM(0);
}
static int
compareint(const void *a, const void *b)
{
if (*((int4 *) a) == *((int4 *) b))
return 0;
return (*((int4 *) a) > *((int4 *) b)) ? 1 : -1;
}
static int
uniqueint(int4 *a, int4 l)
{
int4 *ptr,
*res;
if (l == 1)
return l;
ptr = res = a;
qsort((void *) a, l, sizeof(int4), compareint);
while (ptr - a < l)
if (*ptr != *res)
*(++res) = *ptr++;
else
ptr++;
return res + 1 - a;
}
static void
makesign(BITVECP sign, GISTTYPE * a)
{
int4 k,
len = ARRNELEM(a);
int4 *ptr = GETARR(a);
MemSet((void *) sign, 0, sizeof(BITVEC));
for (k = 0; k < len; k++)
HASH(sign, ptr[k]);
}
Datum
gtxtidx_compress(PG_FUNCTION_ARGS)
{
GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
GISTENTRY *retval = entry;
if (entry->leafkey)
{ /* txtidx */
GISTTYPE *res;
txtidx *toastedval = (txtidx *) DatumGetPointer(entry->key);
txtidx *val = (txtidx *) DatumGetPointer(PG_DETOAST_DATUM(entry->key));
int4 len;
int4 *arr;
WordEntry *ptr = ARRPTR(val);
char *words = STRPTR(val);
len = CALCGTSIZE(ARRKEY, val->size);
res = (GISTTYPE *) palloc(len);
res->len = len;
res->flag = ARRKEY;
arr = GETARR(res);
len = val->size;
while (len--)
{
*arr = crc32_sz((uint8 *) &words[ptr->pos], ptr->len);
arr++;
ptr++;
}
len = uniqueint(GETARR(res), val->size);
if (len != val->size)
{
/*
* there is a collision of hash-function; len is always less
* than val->size
*/
len = CALCGTSIZE(ARRKEY, len);
res = (GISTTYPE *) repalloc((void *) res, len);
res->len = len;
}
if (val != toastedval)
pfree(val);
/* make signature, if array is too long */
if (res->len > TOAST_INDEX_TARGET)
{
GISTTYPE *ressign;
len = CALCGTSIZE(SIGNKEY, 0);
ressign = (GISTTYPE *) palloc(len);
ressign->len = len;
ressign->flag = SIGNKEY;
makesign(GETSIGN(ressign), res);
pfree(res);
res = ressign;
}
retval = (GISTENTRY *) palloc(sizeof(GISTENTRY));
gistentryinit(*retval, PointerGetDatum(res),
entry->rel, entry->page,
entry->offset, res->len, FALSE);
}
else if (ISSIGNKEY(DatumGetPointer(entry->key)) &&
!ISALLTRUE(DatumGetPointer(entry->key)))
{
int4 i,
len;
GISTTYPE *res;
BITVECP sign = GETSIGN(DatumGetPointer(entry->key));
LOOPBYTE(
if ((sign[i] & 0xff) != 0xff)
PG_RETURN_POINTER(retval);
);
len = CALCGTSIZE(SIGNKEY | ALLISTRUE, 0);
res = (GISTTYPE *) palloc(len);
res->len = len;
res->flag = SIGNKEY | ALLISTRUE;
retval = (GISTENTRY *) palloc(sizeof(GISTENTRY));
gistentryinit(*retval, PointerGetDatum(res),
entry->rel, entry->page,
entry->offset, res->len, FALSE);
}
PG_RETURN_POINTER(retval);
}
Datum
gtxtidx_decompress(PG_FUNCTION_ARGS)
{
GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
GISTTYPE *key = (GISTTYPE *) DatumGetPointer(PG_DETOAST_DATUM(entry->key));
if (key != (GISTTYPE *) DatumGetPointer(entry->key))
{
GISTENTRY *retval = (GISTENTRY *) palloc(sizeof(GISTENTRY));
gistentryinit(*retval, PointerGetDatum(key),
entry->rel, entry->page,
entry->offset, key->len, FALSE);
PG_RETURN_POINTER(retval);
}
PG_RETURN_POINTER(entry);
}
typedef struct
{
int4 *arrb;
int4 *arre;
} CHKVAL;
/*
* is there value 'val' in array or not ?
*/
static bool
checkcondition_arr(void *checkval, ITEM * val)
{
int4 *StopLow = ((CHKVAL *) checkval)->arrb;
int4 *StopHigh = ((CHKVAL *) checkval)->arre;
int4 *StopMiddle;
/* Loop invariant: StopLow <= val < StopHigh */
while (StopLow < StopHigh)
{
StopMiddle = StopLow + (StopHigh - StopLow) / 2;
if (*StopMiddle == val->val)
return (true);
else if (*StopMiddle < val->val)
StopLow = StopMiddle + 1;
else
StopHigh = StopMiddle;
}
return (false);
}
static bool
checkcondition_bit(void *checkval, ITEM * val)
{
return GETBIT(checkval, HASHVAL(val->val));
}
Datum
gtxtidx_consistent(PG_FUNCTION_ARGS)
{
QUERYTYPE *query = (QUERYTYPE *) PG_GETARG_POINTER(1);
GISTTYPE *key = (GISTTYPE *) DatumGetPointer(
((GISTENTRY *) PG_GETARG_POINTER(0))->key
);
if (!query->size)
PG_RETURN_BOOL(false);
if (ISSIGNKEY(key))
{
if (ISALLTRUE(key))
PG_RETURN_BOOL(true);
PG_RETURN_BOOL(execute(
GETQUERY(query),
(void *) GETSIGN(key), false,
checkcondition_bit
));
}
else
{ /* only leaf pages */
CHKVAL chkval;
chkval.arrb = GETARR(key);
chkval.arre = chkval.arrb + ARRNELEM(key);
PG_RETURN_BOOL(execute(
GETQUERY(query),
(void *) &chkval, true,
checkcondition_arr
));
}
}
static int4
unionkey(BITVECP sbase, GISTTYPE * add)
{
int4 i;
if (ISSIGNKEY(add))
{
BITVECP sadd = GETSIGN(add);
if (ISALLTRUE(add))
return 1;
LOOPBYTE(
sbase[i] |= sadd[i];
);
}
else
{
int4 *ptr = GETARR(add);
for (i = 0; i < ARRNELEM(add); i++)
HASH(sbase, ptr[i]);
}
return 0;
}
Datum
gtxtidx_union(PG_FUNCTION_ARGS)
{
bytea *entryvec = (bytea *) PG_GETARG_POINTER(0);
int *size = (int *) PG_GETARG_POINTER(1);
BITVEC base;
int4 len = (VARSIZE(entryvec) - VARHDRSZ) / sizeof(GISTENTRY);
int4 i;
int4 flag = 0;
GISTTYPE *result;
MemSet((void *) base, 0, sizeof(BITVEC));
for (i = 0; i < len; i++)
{
if (unionkey(base, GETENTRY(entryvec, i)))
{
flag = ALLISTRUE;
break;
}
}
flag |= SIGNKEY;
len = CALCGTSIZE(flag, 0);
result = (GISTTYPE *) palloc(len);
*size = result->len = len;
result->flag = flag;
if (!ISALLTRUE(result))
memcpy((void *) GETSIGN(result), (void *) base, sizeof(BITVEC));
PG_RETURN_POINTER(result);
}
Datum
gtxtidx_same(PG_FUNCTION_ARGS)
{
GISTTYPE *a = (GISTTYPE *) PG_GETARG_POINTER(0);
GISTTYPE *b = (GISTTYPE *) PG_GETARG_POINTER(1);
bool *result = (bool *) PG_GETARG_POINTER(2);
if (ISSIGNKEY(a))
{ /* then b also ISSIGNKEY */
if (ISALLTRUE(a) && ISALLTRUE(b))
*result = true;
else if (ISALLTRUE(a))
*result = false;
else if (ISALLTRUE(b))
*result = false;
else
{
int4 i;
BITVECP sa = GETSIGN(a),
sb = GETSIGN(b);
*result = true;
LOOPBYTE(
if (sa[i] != sb[i])
{
*result = false;
break;
}
);
}
}
else
{ /* a and b ISARRKEY */
int4 lena = ARRNELEM(a),
lenb = ARRNELEM(b);
if (lena != lenb)
*result = false;
else
{
int4 *ptra = GETARR(a),
*ptrb = GETARR(b);
int4 i;
*result = true;
for (i = 0; i < lena; i++)
if (ptra[i] != ptrb[i])
{
*result = false;
break;
}
}
}
PG_RETURN_POINTER(result);
}
static int4
sizebitvec(BITVECP sign)
{
int4 size = 0,
i;
LOOPBYTE(
size += SUMBIT(*(char *) sign);
sign = (BITVECP) (((char *) sign) + 1);
);
return size;
}
Datum
gtxtidx_penalty(PG_FUNCTION_ARGS)
{
GISTENTRY *origentry = (GISTENTRY *) PG_GETARG_POINTER(0); /* always ISSIGNKEY */
GISTENTRY *newentry = (GISTENTRY *) PG_GETARG_POINTER(1);
float *penalty = (float *) PG_GETARG_POINTER(2);
GISTTYPE *origval = (GISTTYPE *) DatumGetPointer(origentry->key);
GISTTYPE *newval = (GISTTYPE *) DatumGetPointer(newentry->key);
int4 unionsize = 0;
BITVECP orig = GETSIGN(origval);
if (ISALLTRUE(origval))
{
*penalty = 0.0;
PG_RETURN_POINTER(penalty);
}
if (ISARRKEY(newval))
{
int4 *ptr = GETARR(newval),
n = ARRNELEM(newval);
while (n--)
{
if (GETBIT(orig, HASHVAL(*ptr)) == 0)
unionsize++;
ptr++;
}
*penalty = (float) unionsize;
}
else
{
if (ISALLTRUE(newval))
*penalty = (float) (SIGLENBIT - sizebitvec(orig));
else
{
char valtmp;
BITVECP nval = GETSIGN(newval);
int4 i;
LOOPBYTE(
valtmp = nval[i] | orig[i];
unionsize += SUMBIT(valtmp) - SUMBIT(orig[i]);
);
*penalty = (float) unionsize;
}
}
PG_RETURN_POINTER(penalty);
}
typedef struct
{
bool allistrue;
BITVEC sign;
} CACHESIGN;
static void
fillcache(CACHESIGN * item, GISTTYPE * key)
{
item->allistrue = false;
if (ISARRKEY(key))
makesign(item->sign, key);
else if (ISALLTRUE(key))
item->allistrue = true;
else
memcpy((void *) item->sign, (void *) GETSIGN(key), sizeof(BITVEC));
}
#define WISH_F(a,b,c) (double)( -(double)(((a)-(b))*((a)-(b))*((a)-(b)))*(c) )
typedef struct
{
OffsetNumber pos;
int4 cost;
} SPLITCOST;
static int
comparecost(const void *a, const void *b)
{
if (((SPLITCOST *) a)->cost == ((SPLITCOST *) b)->cost)
return 0;
else
return (((SPLITCOST *) a)->cost > ((SPLITCOST *) b)->cost) ? 1 : -1;
}
Datum
gtxtidx_picksplit(PG_FUNCTION_ARGS)
{
bytea *entryvec = (bytea *) PG_GETARG_POINTER(0);
GIST_SPLITVEC *v = (GIST_SPLITVEC *) PG_GETARG_POINTER(1);
OffsetNumber k,
j;
GISTTYPE *datum_l,
*datum_r;
BITVEC union_l,
union_r;
bool firsttime = true;
int4 size_alpha,
size_beta,
sizeu,
sizei;
int4 size_waste,
waste = 0.0;
int4 size_l,
size_r;
int4 nbytes;
OffsetNumber seed_1 = 0,
seed_2 = 0;
OffsetNumber *left,
*right;
OffsetNumber maxoff;
BITVECP ptra,
ptrb,
ptrc;
int i;
CACHESIGN *cache;
char valtmp;
SPLITCOST *costvector;
maxoff = ((VARSIZE(entryvec) - VARHDRSZ) / sizeof(GISTENTRY)) - 2;
nbytes = (maxoff + 2) * sizeof(OffsetNumber);
v->spl_left = (OffsetNumber *) palloc(nbytes);
v->spl_right = (OffsetNumber *) palloc(nbytes);
cache = (CACHESIGN *) palloc(sizeof(CACHESIGN) * (maxoff + 2));
fillcache(&cache[FirstOffsetNumber], GETENTRY(entryvec, FirstOffsetNumber));
for (k = FirstOffsetNumber; k < maxoff; k = OffsetNumberNext(k))
{
for (j = OffsetNumberNext(k); j <= maxoff; j = OffsetNumberNext(j))
{
if (k == FirstOffsetNumber)
fillcache(&cache[j], GETENTRY(entryvec, j));
if (cache[k].allistrue || cache[j].allistrue)
{
sizeu = SIGLENBIT;
if (cache[k].allistrue && cache[j].allistrue)
sizei = SIGLENBIT;
else
sizei = (cache[k].allistrue) ?
sizebitvec(cache[j].sign) : sizebitvec(cache[k].sign);
}
else
{
sizeu = sizei = 0;
ptra = cache[j].sign;
ptrb = cache[k].sign;
/* critical section for bench !!! */
#define COUNT(pos) do { \
if ( GETBITBYTE(*(char*)ptra,pos) ) { \
sizeu++; \
if ( GETBITBYTE(*(char*)ptrb, pos) ) \
sizei++; \
} else if ( GETBITBYTE(*(char*)ptrb, pos) ) \
sizeu++; \
} while(0)
LOOPBYTE(
COUNT(0);
COUNT(1);
COUNT(2);
COUNT(3);
COUNT(4);
COUNT(5);
COUNT(6);
COUNT(7);
ptra = (BITVECP) (((char *) ptra) + 1);
ptrb = (BITVECP) (((char *) ptrb) + 1);
);
}
size_waste = sizeu - sizei;
if (size_waste > waste || firsttime)
{
waste = size_waste;
seed_1 = k;
seed_2 = j;
firsttime = false;
}
}
}
left = v->spl_left;
v->spl_nleft = 0;
right = v->spl_right;
v->spl_nright = 0;
if (seed_1 == 0 || seed_2 == 0)
{
seed_1 = 1;
seed_2 = 2;
}
/* form initial .. */
if (cache[seed_1].allistrue)
{
datum_l = (GISTTYPE *) palloc(CALCGTSIZE(SIGNKEY | ALLISTRUE, 0));
datum_l->len = CALCGTSIZE(SIGNKEY | ALLISTRUE, 0);
datum_l->flag = SIGNKEY | ALLISTRUE;
size_l = SIGLENBIT;
}
else
{
datum_l = (GISTTYPE *) palloc(CALCGTSIZE(SIGNKEY, 0));
datum_l->len = CALCGTSIZE(SIGNKEY, 0);
datum_l->flag = SIGNKEY;
memcpy((void *) GETSIGN(datum_l), (void *) cache[seed_1].sign, sizeof(BITVEC));
size_l = sizebitvec(GETSIGN(datum_l));
}
if (cache[seed_2].allistrue)
{
datum_r = (GISTTYPE *) palloc(CALCGTSIZE(SIGNKEY | ALLISTRUE, 0));
datum_r->len = CALCGTSIZE(SIGNKEY | ALLISTRUE, 0);
datum_r->flag = SIGNKEY | ALLISTRUE;
size_r = SIGLENBIT;
}
else
{
datum_r = (GISTTYPE *) palloc(CALCGTSIZE(SIGNKEY, 0));
datum_r->len = CALCGTSIZE(SIGNKEY, 0);
datum_r->flag = SIGNKEY;
memcpy((void *) GETSIGN(datum_r), (void *) cache[seed_2].sign, sizeof(BITVEC));
size_r = sizebitvec(GETSIGN(datum_r));
}
maxoff = OffsetNumberNext(maxoff);
fillcache(&cache[maxoff], GETENTRY(entryvec, maxoff));
/* sort before ... */
costvector = (SPLITCOST *) palloc(sizeof(SPLITCOST) * maxoff);
for (j = FirstOffsetNumber; j <= maxoff; j = OffsetNumberNext(j))
{
costvector[j - 1].pos = j;
if (cache[j].allistrue)
{
size_alpha = SIGLENBIT - size_l;
size_beta = SIGLENBIT - size_r;
}
else
{
ptra = cache[seed_1].sign;
ptrb = cache[seed_2].sign;
ptrc = cache[j].sign;
size_beta = size_alpha = 0;
if (cache[seed_1].allistrue)
{
if (!cache[seed_2].allistrue)
{
LOOPBIT(
if (GETBIT(ptrc, i) && !GETBIT(ptrb, i))
size_beta++;
);
}
}
else if (cache[seed_2].allistrue)
{
if (!cache[seed_1].allistrue)
{
LOOPBIT(
if (GETBIT(ptrc, i) && !GETBIT(ptra, i))
size_alpha++;
);
}
}
else
{
LOOPBIT(
if (GETBIT(ptrc, i) && !GETBIT(ptra, i))
size_alpha++;
if (GETBIT(ptrc, i) && !GETBIT(ptrb, i))
size_beta++;
);
}
}
costvector[j - 1].cost = abs(size_alpha - size_beta);
}
qsort((void *) costvector, maxoff, sizeof(SPLITCOST), comparecost);
for (k = 0; k < maxoff; k++)
{
j = costvector[k].pos;
if (j == seed_1)
{
*left++ = j;
v->spl_nleft++;
continue;
}
else if (j == seed_2)
{
*right++ = j;
v->spl_nright++;
continue;
}
if (ISALLTRUE(datum_l) || cache[j].allistrue)
size_alpha = SIGLENBIT;
else
{
ptra = cache[j].sign;
ptrb = GETSIGN(datum_l);
size_alpha = 0;
LOOPBYTE(
valtmp = union_l[i] = ptra[i] | ptrb[i];
size_alpha += SUMBIT(valtmp);
);
}
if (ISALLTRUE(datum_r) || cache[j].allistrue)
size_beta = SIGLENBIT;
else
{
ptra = cache[j].sign;
ptrb = GETSIGN(datum_r);
size_beta = 0;
LOOPBYTE(
valtmp = union_r[i] = ptra[i] | ptrb[i];
size_beta += SUMBIT(valtmp);
);
}
if (size_alpha - size_l < size_beta - size_r + WISH_F(v->spl_nleft, v->spl_nright, 0.1))
{
if (!ISALLTRUE(datum_l))
{
if (size_alpha == SIGLENBIT)
{
if (size_alpha != size_l)
MemSet((void *) GETSIGN(datum_l), 0xff, sizeof(BITVEC));
}
else
memcpy((void *) GETSIGN(datum_l), (void *) union_l, sizeof(BITVEC));
}
size_l = size_alpha;
*left++ = j;
v->spl_nleft++;
}
else
{
if (!ISALLTRUE(datum_r))
{
if (size_beta == SIGLENBIT)
{
if (size_beta != size_r)
MemSet((void *) GETSIGN(datum_r), 0xff, sizeof(BITVEC));
}
else
memcpy((void *) GETSIGN(datum_r), (void *) union_r, sizeof(BITVEC));
}
size_r = size_beta;
*right++ = j;
v->spl_nright++;
}
}
*right = *left = FirstOffsetNumber;
pfree(costvector);
pfree(cache);
v->spl_ldatum = PointerGetDatum(datum_l);
v->spl_rdatum = PointerGetDatum(datum_r);
PG_RETURN_POINTER(v);
}