2007-08-21 03:11:32 +02:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* tsrank.c
|
|
|
|
* rank tsvector by tsquery
|
|
|
|
*
|
|
|
|
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
|
|
|
|
*
|
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
2007-09-11 18:01:40 +02:00
|
|
|
* $PostgreSQL: pgsql/src/backend/utils/adt/tsrank.c,v 1.6 2007/09/11 16:01:40 teodor Exp $
|
2007-08-21 03:11:32 +02:00
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include <math.h>
|
|
|
|
|
|
|
|
#include "tsearch/ts_type.h"
|
|
|
|
#include "tsearch/ts_utils.h"
|
|
|
|
#include "utils/array.h"
|
2007-09-07 18:03:40 +02:00
|
|
|
#include "miscadmin.h"
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
|
|
|
|
static float weights[] = {0.1, 0.2, 0.4, 1.0};
|
|
|
|
|
|
|
|
#define wpos(wep) ( w[ WEP_GETWEIGHT(wep) ] )
|
|
|
|
|
|
|
|
#define RANK_NO_NORM 0x00
|
|
|
|
#define RANK_NORM_LOGLENGTH 0x01
|
|
|
|
#define RANK_NORM_LENGTH 0x02
|
|
|
|
#define RANK_NORM_EXTDIST 0x04
|
|
|
|
#define RANK_NORM_UNIQ 0x08
|
|
|
|
#define RANK_NORM_LOGUNIQ 0x10
|
|
|
|
#define DEF_NORM_METHOD RANK_NO_NORM
|
|
|
|
|
|
|
|
static float calc_rank_or(float *w, TSVector t, TSQuery q);
|
|
|
|
static float calc_rank_and(float *w, TSVector t, TSQuery q);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Returns a weight of a word collocation
|
|
|
|
*/
|
|
|
|
static float4
|
|
|
|
word_distance(int4 w)
|
|
|
|
{
|
|
|
|
if (w > 100)
|
|
|
|
return 1e-30;
|
|
|
|
|
|
|
|
return 1.0 / (1.005 + 0.05 * exp(((float4) w) / 1.5 - 2));
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
cnt_length(TSVector t)
|
|
|
|
{
|
|
|
|
WordEntry *ptr = ARRPTR(t),
|
|
|
|
*end = (WordEntry *) STRPTR(t);
|
2007-09-11 10:46:29 +02:00
|
|
|
int len = 0;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
while (ptr < end)
|
|
|
|
{
|
2007-09-11 10:46:29 +02:00
|
|
|
int clen = POSDATALEN(t, ptr);
|
|
|
|
|
|
|
|
if (clen == 0)
|
2007-08-21 03:11:32 +02:00
|
|
|
len += 1;
|
|
|
|
else
|
|
|
|
len += clen;
|
2007-09-11 10:46:29 +02:00
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return len;
|
|
|
|
}
|
|
|
|
|
2007-09-11 10:46:29 +02:00
|
|
|
static int
|
2007-09-07 17:09:56 +02:00
|
|
|
WordECompareQueryItem(char *eval, char *qval, WordEntry *ptr, QueryOperand *item)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
if (ptr->len == item->length)
|
|
|
|
return strncmp(
|
|
|
|
eval + ptr->pos,
|
|
|
|
qval + item->distance,
|
|
|
|
item->length);
|
|
|
|
|
|
|
|
return (ptr->len > item->length) ? 1 : -1;
|
|
|
|
}
|
|
|
|
|
2007-09-11 10:46:29 +02:00
|
|
|
/*
|
|
|
|
* Returns a pointer to a WordEntry corresponding 'item' from tsvector 't'. 'q'
|
|
|
|
* is the TSQuery containing 'item'. Returns NULL if not found.
|
|
|
|
*/
|
2007-08-21 03:11:32 +02:00
|
|
|
static WordEntry *
|
2007-09-07 17:09:56 +02:00
|
|
|
find_wordentry(TSVector t, TSQuery q, QueryOperand *item)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
WordEntry *StopLow = ARRPTR(t);
|
|
|
|
WordEntry *StopHigh = (WordEntry *) STRPTR(t);
|
|
|
|
WordEntry *StopMiddle;
|
|
|
|
int difference;
|
|
|
|
|
|
|
|
/* Loop invariant: StopLow <= item < StopHigh */
|
|
|
|
|
|
|
|
while (StopLow < StopHigh)
|
|
|
|
{
|
|
|
|
StopMiddle = StopLow + (StopHigh - StopLow) / 2;
|
|
|
|
difference = WordECompareQueryItem(STRPTR(t), GETOPERAND(q), StopMiddle, item);
|
|
|
|
if (difference == 0)
|
|
|
|
return StopMiddle;
|
|
|
|
else if (difference < 0)
|
|
|
|
StopLow = StopMiddle + 1;
|
|
|
|
else
|
|
|
|
StopHigh = StopMiddle;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/*
|
|
|
|
* sort QueryOperands by (length, word)
|
|
|
|
*/
|
2007-08-21 03:11:32 +02:00
|
|
|
static int
|
2007-09-07 17:09:56 +02:00
|
|
|
compareQueryOperand(const void *a, const void *b, void *arg)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
char *operand = (char *) arg;
|
2007-09-07 17:09:56 +02:00
|
|
|
QueryOperand *qa = (*(QueryOperand **) a);
|
|
|
|
QueryOperand *qb = (*(QueryOperand **) b);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
if (qa->length == qb->length)
|
|
|
|
return strncmp(operand + qa->distance,
|
|
|
|
operand + qb->distance,
|
|
|
|
qb->length);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
return (qa->length > qb->length) ? 1 : -1;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/*
|
|
|
|
* Returns a sorted, de-duplicated array of QueryOperands in a query.
|
|
|
|
* The returned QueryOperands are pointers to the original QueryOperands
|
|
|
|
* in the query.
|
|
|
|
*
|
|
|
|
* Length of the returned array is stored in *size
|
|
|
|
*/
|
|
|
|
static QueryOperand **
|
|
|
|
SortAndUniqItems(TSQuery q, int *size)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
char *operand = GETOPERAND(q);
|
|
|
|
QueryItem * item = GETQUERY(q);
|
|
|
|
QueryOperand **res,
|
2007-08-21 03:11:32 +02:00
|
|
|
**ptr,
|
|
|
|
**prevptr;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
ptr = res = (QueryOperand **) palloc(sizeof(QueryOperand *) * *size);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/* Collect all operands from the tree to res */
|
2007-08-21 03:11:32 +02:00
|
|
|
while ((*size)--)
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
if (item->type == QI_VAL)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
*ptr = (QueryOperand *) item;
|
2007-08-21 03:11:32 +02:00
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
item++;
|
|
|
|
}
|
|
|
|
|
|
|
|
*size = ptr - res;
|
|
|
|
if (*size < 2)
|
|
|
|
return res;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
qsort_arg(res, *size, sizeof(QueryOperand **), compareQueryOperand, (void *) operand);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
ptr = res + 1;
|
|
|
|
prevptr = res;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/* remove duplicates */
|
2007-08-21 03:11:32 +02:00
|
|
|
while (ptr - res < *size)
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
if (compareQueryOperand((void *) ptr, (void *) prevptr, (void *) operand) != 0)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
prevptr++;
|
|
|
|
*prevptr = *ptr;
|
|
|
|
}
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
*size = prevptr + 1 - res;
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/* A dummy WordEntryPos array to use when haspos is false */
|
2007-09-11 10:46:29 +02:00
|
|
|
static WordEntryPosVector POSNULL = {
|
2007-09-07 18:03:40 +02:00
|
|
|
1, /* Number of elements that follow */
|
2007-09-11 10:46:29 +02:00
|
|
|
{ 0 }
|
2007-08-21 03:11:32 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
static float
|
|
|
|
calc_rank_and(float *w, TSVector t, TSQuery q)
|
|
|
|
{
|
2007-09-11 10:46:29 +02:00
|
|
|
WordEntryPosVector **pos;
|
2007-08-21 03:11:32 +02:00
|
|
|
int i,
|
|
|
|
k,
|
|
|
|
l,
|
|
|
|
p;
|
|
|
|
WordEntry *entry;
|
|
|
|
WordEntryPos *post,
|
|
|
|
*ct;
|
|
|
|
int4 dimt,
|
|
|
|
lenct,
|
|
|
|
dist;
|
|
|
|
float res = -1.0;
|
2007-09-07 17:09:56 +02:00
|
|
|
QueryOperand **item;
|
2007-08-21 03:11:32 +02:00
|
|
|
int size = q->size;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
item = SortAndUniqItems(q, &size);
|
2007-08-21 03:11:32 +02:00
|
|
|
if (size < 2)
|
|
|
|
{
|
|
|
|
pfree(item);
|
|
|
|
return calc_rank_or(w, t, q);
|
|
|
|
}
|
2007-09-11 10:46:29 +02:00
|
|
|
pos = (WordEntryPosVector **) palloc0(sizeof(WordEntryPosVector *) * q->size);
|
|
|
|
WEP_SETPOS(POSNULL.pos[0], MAXENTRYPOS - 1);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
for (i = 0; i < size; i++)
|
|
|
|
{
|
|
|
|
entry = find_wordentry(t, q, item[i]);
|
|
|
|
if (!entry)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (entry->haspos)
|
2007-09-11 10:46:29 +02:00
|
|
|
pos[i] = _POSVECPTR(t, entry);
|
2007-08-21 03:11:32 +02:00
|
|
|
else
|
2007-09-11 10:46:29 +02:00
|
|
|
pos[i] = &POSNULL;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
|
2007-09-11 10:46:29 +02:00
|
|
|
dimt = pos[i]->npos;
|
|
|
|
post = pos[i]->pos;
|
2007-08-21 03:11:32 +02:00
|
|
|
for (k = 0; k < i; k++)
|
|
|
|
{
|
|
|
|
if (!pos[k])
|
|
|
|
continue;
|
2007-09-11 10:46:29 +02:00
|
|
|
lenct = pos[k]->npos;
|
|
|
|
ct = pos[k]->pos;
|
2007-08-21 03:11:32 +02:00
|
|
|
for (l = 0; l < dimt; l++)
|
|
|
|
{
|
|
|
|
for (p = 0; p < lenct; p++)
|
|
|
|
{
|
|
|
|
dist = Abs((int) WEP_GETPOS(post[l]) - (int) WEP_GETPOS(ct[p]));
|
2007-09-11 10:46:29 +02:00
|
|
|
if (dist || (dist == 0 && (pos[i] == &POSNULL || pos[k] == &POSNULL)))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
float curw;
|
|
|
|
|
|
|
|
if (!dist)
|
|
|
|
dist = MAXENTRYPOS;
|
|
|
|
curw = sqrt(wpos(post[l]) * wpos(ct[p]) * word_distance(dist));
|
|
|
|
res = (res < 0) ? curw : 1.0 - (1.0 - res) * (1.0 - curw);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
pfree(pos);
|
|
|
|
pfree(item);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
static float
|
|
|
|
calc_rank_or(float *w, TSVector t, TSQuery q)
|
|
|
|
{
|
|
|
|
WordEntry *entry;
|
|
|
|
WordEntryPos *post;
|
|
|
|
int4 dimt,
|
|
|
|
j,
|
|
|
|
i;
|
|
|
|
float res = 0.0;
|
2007-09-07 17:09:56 +02:00
|
|
|
QueryOperand **item;
|
2007-08-21 03:11:32 +02:00
|
|
|
int size = q->size;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
item = SortAndUniqItems(q, &size);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
for (i = 0; i < size; i++)
|
|
|
|
{
|
|
|
|
float resj,
|
|
|
|
wjm;
|
|
|
|
int4 jm;
|
|
|
|
|
|
|
|
entry = find_wordentry(t, q, item[i]);
|
|
|
|
if (!entry)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (entry->haspos)
|
|
|
|
{
|
|
|
|
dimt = POSDATALEN(t, entry);
|
|
|
|
post = POSDATAPTR(t, entry);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2007-09-11 10:46:29 +02:00
|
|
|
dimt = POSNULL.npos;
|
|
|
|
post = POSNULL.pos;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
resj = 0.0;
|
|
|
|
wjm = -1.0;
|
|
|
|
jm = 0;
|
|
|
|
for (j = 0; j < dimt; j++)
|
|
|
|
{
|
|
|
|
resj = resj + wpos(post[j]) / ((j + 1) * (j + 1));
|
|
|
|
if (wpos(post[j]) > wjm)
|
|
|
|
{
|
|
|
|
wjm = wpos(post[j]);
|
|
|
|
jm = j;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
limit (sum(i/i^2),i->inf) = pi^2/6
|
|
|
|
resj = sum(wi/i^2),i=1,noccurence,
|
|
|
|
wi - should be sorted desc,
|
|
|
|
don't sort for now, just choose maximum weight. This should be corrected
|
|
|
|
Oleg Bartunov
|
|
|
|
*/
|
|
|
|
res = res + (wjm + resj - wjm / ((jm + 1) * (jm + 1))) / 1.64493406685;
|
|
|
|
}
|
|
|
|
if (size > 0)
|
|
|
|
res = res / size;
|
|
|
|
pfree(item);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
static float
|
|
|
|
calc_rank(float *w, TSVector t, TSQuery q, int4 method)
|
|
|
|
{
|
|
|
|
QueryItem *item = GETQUERY(q);
|
|
|
|
float res = 0.0;
|
|
|
|
int len;
|
|
|
|
|
|
|
|
if (!t->size || !q->size)
|
|
|
|
return 0.0;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/* XXX: What about NOT? */
|
|
|
|
res = (item->type == QI_OPR && item->operator.oper == OP_AND) ?
|
2007-08-21 03:11:32 +02:00
|
|
|
calc_rank_and(w, t, q) : calc_rank_or(w, t, q);
|
|
|
|
|
|
|
|
if (res < 0)
|
|
|
|
res = 1e-20;
|
|
|
|
|
|
|
|
if ((method & RANK_NORM_LOGLENGTH) && t->size > 0)
|
|
|
|
res /= log((double) (cnt_length(t) + 1)) / log(2.0);
|
|
|
|
|
|
|
|
if (method & RANK_NORM_LENGTH)
|
|
|
|
{
|
|
|
|
len = cnt_length(t);
|
|
|
|
if (len > 0)
|
|
|
|
res /= (float) len;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((method & RANK_NORM_UNIQ) && t->size > 0)
|
|
|
|
res /= (float) (t->size);
|
|
|
|
|
|
|
|
if ((method & RANK_NORM_LOGUNIQ) && t->size > 0)
|
|
|
|
res /= log((double) (t->size + 1)) / log(2.0);
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
static float *
|
|
|
|
getWeights(ArrayType *win)
|
|
|
|
{
|
|
|
|
static float ws[lengthof(weights)];
|
|
|
|
int i;
|
|
|
|
float4 *arrdata;
|
|
|
|
|
|
|
|
if (win == 0)
|
|
|
|
return weights;
|
|
|
|
|
|
|
|
if (ARR_NDIM(win) != 1)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR),
|
|
|
|
errmsg("array of weight must be one-dimensional")));
|
|
|
|
|
|
|
|
if (ArrayGetNItems(ARR_NDIM(win), ARR_DIMS(win)) < lengthof(weights))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR),
|
|
|
|
errmsg("array of weight is too short")));
|
|
|
|
|
|
|
|
if (ARR_HASNULL(win))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
|
|
|
|
errmsg("array of weight must not contain nulls")));
|
|
|
|
|
|
|
|
arrdata = (float4 *) ARR_DATA_PTR(win);
|
|
|
|
for (i = 0; i < lengthof(weights); i++)
|
|
|
|
{
|
|
|
|
ws[i] = (arrdata[i] >= 0) ? arrdata[i] : weights[i];
|
|
|
|
if (ws[i] > 1.0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
|
|
|
errmsg("weight out of range")));
|
|
|
|
}
|
|
|
|
|
|
|
|
return ws;
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
ts_rank_wttf(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
ArrayType *win = (ArrayType *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0));
|
|
|
|
TSVector txt = PG_GETARG_TSVECTOR(1);
|
|
|
|
TSQuery query = PG_GETARG_TSQUERY(2);
|
|
|
|
int method = PG_GETARG_INT32(3);
|
|
|
|
float res;
|
|
|
|
|
|
|
|
res = calc_rank(getWeights(win), txt, query, method);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(win, 0);
|
|
|
|
PG_FREE_IF_COPY(txt, 1);
|
|
|
|
PG_FREE_IF_COPY(query, 2);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
ts_rank_wtt(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
ArrayType *win = (ArrayType *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0));
|
|
|
|
TSVector txt = PG_GETARG_TSVECTOR(1);
|
|
|
|
TSQuery query = PG_GETARG_TSQUERY(2);
|
|
|
|
float res;
|
|
|
|
|
|
|
|
res = calc_rank(getWeights(win), txt, query, DEF_NORM_METHOD);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(win, 0);
|
|
|
|
PG_FREE_IF_COPY(txt, 1);
|
|
|
|
PG_FREE_IF_COPY(query, 2);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
ts_rank_ttf(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSVector txt = PG_GETARG_TSVECTOR(0);
|
|
|
|
TSQuery query = PG_GETARG_TSQUERY(1);
|
|
|
|
int method = PG_GETARG_INT32(2);
|
|
|
|
float res;
|
|
|
|
|
|
|
|
res = calc_rank(getWeights(NULL), txt, query, method);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(txt, 0);
|
|
|
|
PG_FREE_IF_COPY(query, 1);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
ts_rank_tt(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSVector txt = PG_GETARG_TSVECTOR(0);
|
|
|
|
TSQuery query = PG_GETARG_TSQUERY(1);
|
|
|
|
float res;
|
|
|
|
|
|
|
|
res = calc_rank(getWeights(NULL), txt, query, DEF_NORM_METHOD);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(txt, 0);
|
|
|
|
PG_FREE_IF_COPY(query, 1);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
typedef struct
|
|
|
|
{
|
|
|
|
QueryItem **item;
|
|
|
|
int16 nitem;
|
|
|
|
uint8 wclass;
|
|
|
|
int32 pos;
|
|
|
|
} DocRepresentation;
|
|
|
|
|
|
|
|
static int
|
2007-09-11 10:46:29 +02:00
|
|
|
compareDocR(const void *va, const void *vb)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-11 10:46:29 +02:00
|
|
|
DocRepresentation *a = (DocRepresentation *) va;
|
|
|
|
DocRepresentation *b = (DocRepresentation *) vb;
|
|
|
|
|
|
|
|
if (a->pos == b->pos)
|
2007-08-21 03:11:32 +02:00
|
|
|
return 0;
|
2007-09-11 10:46:29 +02:00
|
|
|
return (a->pos > b->pos) ? 1 : -1;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
typedef struct
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-11 18:01:40 +02:00
|
|
|
TSQuery query;
|
|
|
|
bool *operandexist;
|
|
|
|
} QueryRepresentation;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
#define QR_GET_OPERAND_EXISTS(q, v) ( (q)->operandexist[ ((QueryItem*)(v)) - GETQUERY((q)->query) ] )
|
|
|
|
#define QR_SET_OPERAND_EXISTS(q, v) QR_GET_OPERAND_EXISTS(q,v) = true
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
static bool
|
|
|
|
checkcondition_QueryOperand(void *checkval, QueryOperand *val)
|
|
|
|
{
|
|
|
|
QueryRepresentation *qr = (QueryRepresentation*)checkval;
|
|
|
|
return QR_GET_OPERAND_EXISTS(qr, val);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
typedef struct
|
|
|
|
{
|
|
|
|
int pos;
|
|
|
|
int p;
|
|
|
|
int q;
|
|
|
|
DocRepresentation *begin;
|
|
|
|
DocRepresentation *end;
|
|
|
|
} Extention;
|
|
|
|
|
|
|
|
|
|
|
|
static bool
|
2007-09-11 18:01:40 +02:00
|
|
|
Cover(DocRepresentation *doc, int len, QueryRepresentation *qr, Extention *ext)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
DocRepresentation *ptr;
|
|
|
|
int lastpos = ext->pos;
|
|
|
|
int i;
|
|
|
|
bool found = false;
|
|
|
|
|
2007-09-07 17:35:11 +02:00
|
|
|
/* since this function recurses, it could be driven to stack overflow.
|
|
|
|
* (though any decent compiler will optimize away the tail-recursion. */
|
|
|
|
check_stack_depth();
|
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
memset( qr->operandexist, 0, sizeof(bool)*qr->query->size );
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
ext->p = 0x7fffffff;
|
|
|
|
ext->q = 0;
|
|
|
|
ptr = doc + ext->pos;
|
|
|
|
|
|
|
|
/* find upper bound of cover from current position, move up */
|
|
|
|
while (ptr - doc < len)
|
|
|
|
{
|
|
|
|
for (i = 0; i < ptr->nitem; i++)
|
2007-09-07 17:09:56 +02:00
|
|
|
{
|
|
|
|
if(ptr->item[i]->type == QI_VAL)
|
2007-09-11 18:01:40 +02:00
|
|
|
QR_SET_OPERAND_EXISTS(qr, ptr->item[i]);
|
2007-09-07 17:09:56 +02:00
|
|
|
}
|
2007-09-11 18:01:40 +02:00
|
|
|
if (TS_execute(GETQUERY(qr->query), (void*)qr, false, checkcondition_QueryOperand))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
if (ptr->pos > ext->q)
|
|
|
|
{
|
|
|
|
ext->q = ptr->pos;
|
|
|
|
ext->end = ptr;
|
|
|
|
lastpos = ptr - doc;
|
|
|
|
found = true;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!found)
|
|
|
|
return false;
|
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
memset( qr->operandexist, 0, sizeof(bool)*qr->query->size );
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
ptr = doc + lastpos;
|
|
|
|
|
2007-09-11 10:46:29 +02:00
|
|
|
/* find lower bound of cover from found upper bound, move down */
|
2007-08-21 03:11:32 +02:00
|
|
|
while (ptr >= doc + ext->pos)
|
|
|
|
{
|
|
|
|
for (i = 0; i < ptr->nitem; i++)
|
2007-09-11 10:46:29 +02:00
|
|
|
if(ptr->item[i]->type == QI_VAL)
|
2007-09-11 18:01:40 +02:00
|
|
|
QR_SET_OPERAND_EXISTS(qr, ptr->item[i]);
|
|
|
|
if (TS_execute(GETQUERY(qr->query), (void*)qr, true, checkcondition_QueryOperand))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
if (ptr->pos < ext->p)
|
|
|
|
{
|
|
|
|
ext->begin = ptr;
|
|
|
|
ext->p = ptr->pos;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
ptr--;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ext->p <= ext->q)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* set position for next try to next lexeme after begining of founded
|
|
|
|
* cover
|
|
|
|
*/
|
|
|
|
ext->pos = (ptr - doc) + 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
ext->pos++;
|
2007-09-11 18:01:40 +02:00
|
|
|
return Cover(doc, len, qr, ext);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static DocRepresentation *
|
2007-09-11 18:01:40 +02:00
|
|
|
get_docrep(TSVector txt, QueryRepresentation *qr, int *doclen)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-11 18:01:40 +02:00
|
|
|
QueryItem *item = GETQUERY(qr->query);
|
2007-08-21 03:11:32 +02:00
|
|
|
WordEntry *entry;
|
|
|
|
WordEntryPos *post;
|
|
|
|
int4 dimt,
|
|
|
|
j,
|
|
|
|
i;
|
2007-09-11 18:01:40 +02:00
|
|
|
int len = qr->query->size * 4,
|
2007-08-21 03:11:32 +02:00
|
|
|
cur = 0;
|
|
|
|
DocRepresentation *doc;
|
|
|
|
char *operand;
|
|
|
|
|
|
|
|
doc = (DocRepresentation *) palloc(sizeof(DocRepresentation) * len);
|
2007-09-11 18:01:40 +02:00
|
|
|
operand = GETOPERAND(qr->query);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
for (i = 0; i < qr->query->size; i++)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
QueryOperand *curoperand;
|
|
|
|
|
|
|
|
if (item[i].type != QI_VAL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
curoperand = &item[i].operand;
|
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
if(QR_GET_OPERAND_EXISTS(qr, &item[i]))
|
2007-08-21 03:11:32 +02:00
|
|
|
continue;
|
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
entry = find_wordentry(txt, qr->query, curoperand);
|
2007-08-21 03:11:32 +02:00
|
|
|
if (!entry)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (entry->haspos)
|
|
|
|
{
|
|
|
|
dimt = POSDATALEN(txt, entry);
|
|
|
|
post = POSDATAPTR(txt, entry);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2007-09-11 10:46:29 +02:00
|
|
|
dimt = POSNULL.npos;
|
|
|
|
post = POSNULL.pos;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
while (cur + dimt >= len)
|
|
|
|
{
|
|
|
|
len *= 2;
|
|
|
|
doc = (DocRepresentation *) repalloc(doc, sizeof(DocRepresentation) * len);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (j = 0; j < dimt; j++)
|
|
|
|
{
|
|
|
|
if (j == 0)
|
|
|
|
{
|
|
|
|
int k;
|
|
|
|
|
|
|
|
doc[cur].nitem = 0;
|
2007-09-11 18:01:40 +02:00
|
|
|
doc[cur].item = (QueryItem **) palloc(sizeof(QueryItem *) * qr->query->size);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
for (k = 0; k < qr->query->size; k++)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
QueryOperand *kptr = &item[k].operand;
|
|
|
|
QueryOperand *iptr = &item[i].operand;
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
if (k == i ||
|
2007-09-07 17:09:56 +02:00
|
|
|
(item[k].type == QI_VAL &&
|
|
|
|
compareQueryOperand(&kptr, &iptr, operand) == 0))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
/* if k == i, we've already checked above that it's type == Q_VAL */
|
2007-08-21 03:11:32 +02:00
|
|
|
doc[cur].item[doc[cur].nitem] = item + k;
|
|
|
|
doc[cur].nitem++;
|
2007-09-11 18:01:40 +02:00
|
|
|
QR_SET_OPERAND_EXISTS( qr, item+k );
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
doc[cur].nitem = doc[cur - 1].nitem;
|
|
|
|
doc[cur].item = doc[cur - 1].item;
|
|
|
|
}
|
|
|
|
doc[cur].pos = WEP_GETPOS(post[j]);
|
|
|
|
doc[cur].wclass = WEP_GETWEIGHT(post[j]);
|
|
|
|
cur++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
*doclen = cur;
|
|
|
|
|
|
|
|
if (cur > 0)
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
qsort((void *) doc, cur, sizeof(DocRepresentation), compareDocR);
|
2007-08-21 03:11:32 +02:00
|
|
|
return doc;
|
|
|
|
}
|
|
|
|
|
|
|
|
pfree(doc);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static float4
|
|
|
|
calc_rank_cd(float4 *arrdata, TSVector txt, TSQuery query, int method)
|
|
|
|
{
|
|
|
|
DocRepresentation *doc;
|
|
|
|
int len,
|
|
|
|
i,
|
|
|
|
doclen = 0;
|
|
|
|
Extention ext;
|
|
|
|
double Wdoc = 0.0;
|
|
|
|
double invws[lengthof(weights)];
|
|
|
|
double SumDist = 0.0,
|
|
|
|
PrevExtPos = 0.0,
|
|
|
|
CurExtPos = 0.0;
|
|
|
|
int NExtent = 0;
|
2007-09-11 18:01:40 +02:00
|
|
|
QueryRepresentation qr;
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
for (i = 0; i < lengthof(weights); i++)
|
|
|
|
{
|
|
|
|
invws[i] = ((double) ((arrdata[i] >= 0) ? arrdata[i] : weights[i]));
|
|
|
|
if (invws[i] > 1.0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
|
|
|
errmsg("weight out of range")));
|
|
|
|
invws[i] = 1.0 / invws[i];
|
|
|
|
}
|
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
qr.query = query;
|
|
|
|
qr.operandexist = (int*)palloc0(sizeof(bool) * query->size);
|
|
|
|
|
|
|
|
doc = get_docrep(txt, &qr, &doclen);
|
2007-08-21 03:11:32 +02:00
|
|
|
if (!doc)
|
2007-09-11 18:01:40 +02:00
|
|
|
{
|
|
|
|
pfree( qr.operandexist );
|
2007-08-21 03:11:32 +02:00
|
|
|
return 0.0;
|
2007-09-11 18:01:40 +02:00
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
MemSet(&ext, 0, sizeof(Extention));
|
2007-09-11 18:01:40 +02:00
|
|
|
while (Cover(doc, doclen, &qr, &ext))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
double Cpos = 0.0;
|
|
|
|
double InvSum = 0.0;
|
|
|
|
int nNoise;
|
|
|
|
DocRepresentation *ptr = ext.begin;
|
|
|
|
|
|
|
|
while (ptr <= ext.end)
|
|
|
|
{
|
|
|
|
InvSum += invws[ptr->wclass];
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
Cpos = ((double) (ext.end - ext.begin + 1)) / InvSum;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* if doc are big enough then ext.q may be equal to ext.p due to limit
|
|
|
|
* of posional information. In this case we approximate number of
|
|
|
|
* noise word as half cover's length
|
|
|
|
*/
|
|
|
|
nNoise = (ext.q - ext.p) - (ext.end - ext.begin);
|
|
|
|
if (nNoise < 0)
|
|
|
|
nNoise = (ext.end - ext.begin) / 2;
|
|
|
|
Wdoc += Cpos / ((double) (1 + nNoise));
|
|
|
|
|
|
|
|
CurExtPos = ((double) (ext.q + ext.p)) / 2.0;
|
|
|
|
if (NExtent > 0 && CurExtPos > PrevExtPos /* prevent devision by
|
|
|
|
* zero in a case of
|
|
|
|
multiple lexize */ )
|
|
|
|
SumDist += 1.0 / (CurExtPos - PrevExtPos);
|
|
|
|
|
|
|
|
PrevExtPos = CurExtPos;
|
|
|
|
NExtent++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((method & RANK_NORM_LOGLENGTH) && txt->size > 0)
|
|
|
|
Wdoc /= log((double) (cnt_length(txt) + 1));
|
|
|
|
|
|
|
|
if (method & RANK_NORM_LENGTH)
|
|
|
|
{
|
|
|
|
len = cnt_length(txt);
|
|
|
|
if (len > 0)
|
|
|
|
Wdoc /= (double) len;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((method & RANK_NORM_EXTDIST) && SumDist > 0)
|
|
|
|
Wdoc /= ((double) NExtent) / SumDist;
|
|
|
|
|
|
|
|
if ((method & RANK_NORM_UNIQ) && txt->size > 0)
|
|
|
|
Wdoc /= (double) (txt->size);
|
|
|
|
|
|
|
|
if ((method & RANK_NORM_LOGUNIQ) && txt->size > 0)
|
|
|
|
Wdoc /= log((double) (txt->size + 1)) / log(2.0);
|
|
|
|
|
|
|
|
pfree(doc);
|
|
|
|
|
2007-09-11 18:01:40 +02:00
|
|
|
pfree( qr.operandexist );
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
return (float4) Wdoc;
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
ts_rankcd_wttf(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
ArrayType *win = (ArrayType *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0));
|
|
|
|
TSVector txt = PG_GETARG_TSVECTOR(1);
|
2007-09-11 18:01:40 +02:00
|
|
|
TSQuery query = PG_GETARG_TSQUERY(2);
|
2007-08-21 03:11:32 +02:00
|
|
|
int method = PG_GETARG_INT32(3);
|
|
|
|
float res;
|
|
|
|
|
|
|
|
res = calc_rank_cd(getWeights(win), txt, query, method);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(win, 0);
|
|
|
|
PG_FREE_IF_COPY(txt, 1);
|
|
|
|
PG_FREE_IF_COPY(query, 2);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
ts_rankcd_wtt(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
ArrayType *win = (ArrayType *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0));
|
|
|
|
TSVector txt = PG_GETARG_TSVECTOR(1);
|
2007-09-11 18:01:40 +02:00
|
|
|
TSQuery query = PG_GETARG_TSQUERY(2);
|
2007-08-21 03:11:32 +02:00
|
|
|
float res;
|
|
|
|
|
|
|
|
res = calc_rank_cd(getWeights(win), txt, query, DEF_NORM_METHOD);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(win, 0);
|
|
|
|
PG_FREE_IF_COPY(txt, 1);
|
|
|
|
PG_FREE_IF_COPY(query, 2);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
ts_rankcd_ttf(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSVector txt = PG_GETARG_TSVECTOR(0);
|
2007-09-11 18:01:40 +02:00
|
|
|
TSQuery query = PG_GETARG_TSQUERY(1);
|
2007-08-21 03:11:32 +02:00
|
|
|
int method = PG_GETARG_INT32(2);
|
|
|
|
float res;
|
|
|
|
|
|
|
|
res = calc_rank_cd(getWeights(NULL), txt, query, method);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(txt, 0);
|
|
|
|
PG_FREE_IF_COPY(query, 1);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
ts_rankcd_tt(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSVector txt = PG_GETARG_TSVECTOR(0);
|
2007-09-11 18:01:40 +02:00
|
|
|
TSQuery query = PG_GETARG_TSQUERY(1);
|
2007-08-21 03:11:32 +02:00
|
|
|
float res;
|
|
|
|
|
|
|
|
res = calc_rank_cd(getWeights(NULL), txt, query, DEF_NORM_METHOD);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(txt, 0);
|
|
|
|
PG_FREE_IF_COPY(query, 1);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|