postgresql/contrib/tsearch/query.c

798 lines
16 KiB
C
Raw Normal View History

/*
* IO definitions for query_txt and mquery_txt. This type
* are identical, but for parsing mquery_txt used parser for text
* and also morphology is used.
* Internal structure:
* query tree, then string with original value.
* Query tree with plain view. It's means that in array of nodes
* right child is always next and left position = item+item->left
* Teodor Sigaev <teodor@stack.net>
*/
#include "postgres.h"
#include <float.h>
#include "access/gist.h"
#include "access/itup.h"
#include "access/rtree.h"
#include "utils/elog.h"
#include "utils/palloc.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "storage/bufpage.h"
#include "txtidx.h"
#include "crc32.h"
#include "query.h"
#include "morph.h"
#include "rewrite.h"
#include "deflex.h"
#include "parser.h"
PG_FUNCTION_INFO_V1(mqtxt_in);
Datum mqtxt_in(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(qtxt_in);
Datum qtxt_in(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(qtxt_out);
Datum qtxt_out(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(execqtxt);
Datum execqtxt(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(rexecqtxt);
Datum rexecqtxt(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(querytree);
Datum querytree(PG_FUNCTION_ARGS);
#define END 0
#define ERR 1
#define VAL 2
#define OPR 3
#define OPEN 4
#define CLOSE 5
#define VALTRUE 6 /* for stop words */
#define VALFALSE 7
/* parser's states */
#define WAITOPERAND 1
#define WAITOPERATOR 2
/*
* node of query tree, also used
* for storing polish notation in parser
*/
typedef struct NODE
{
int4 type;
int4 val;
int2 distance;
int2 length;
struct NODE *next;
} NODE;
typedef struct
{
char *buf;
int4 state;
int4 count;
/* reverse polish notation in list (for temprorary usage) */
NODE *str;
/* number in str */
int4 num;
/* user-friendly operand */
int4 lenop;
int4 sumlen;
char *op;
char *curop;
/* state for value's parser */
TI_IN_STATE valstate;
} QPRS_STATE;
/*
* get token from query string
*/
static int4
gettoken_query(QPRS_STATE * state, int4 *val, int4 *lenval, char **strval)
{
while (1)
{
switch (state->state)
{
case WAITOPERAND:
if (*(state->buf) == '!')
{
(state->buf)++;
*val = (int4) '!';
return OPR;
}
else if (*(state->buf) == '(')
{
state->count++;
(state->buf)++;
return OPEN;
}
else if (*(state->buf) != ' ')
{
state->valstate.prsbuf = state->buf;
state->state = WAITOPERATOR;
if (gettoken_txtidx(&(state->valstate)))
{
*strval = state->valstate.word;
*lenval = state->valstate.curpos - state->valstate.word;
state->buf = state->valstate.prsbuf;
return VAL;
}
else
elog(ERROR, "No operand");
}
break;
case WAITOPERATOR:
if (*(state->buf) == '&' || *(state->buf) == '|')
{
state->state = WAITOPERAND;
*val = (int4) *(state->buf);
(state->buf)++;
return OPR;
}
else if (*(state->buf) == ')')
{
(state->buf)++;
state->count--;
return (state->count < 0) ? ERR : CLOSE;
}
else if (*(state->buf) == '\0')
return (state->count) ? ERR : END;
else if (*(state->buf) != ' ')
return ERR;
break;
default:
return ERR;
break;
}
(state->buf)++;
}
return END;
}
/*
* push new one in polish notation reverse view
*/
static void
pushquery(QPRS_STATE * state, int4 type, int4 val, int4 distance, int4 lenval)
{
NODE *tmp = (NODE *) palloc(sizeof(NODE));
tmp->type = type;
tmp->val = val;
if (distance > 0xffff)
elog(ERROR, "Value is too big");
if (lenval > 0xffff)
elog(ERROR, "Operand is too long");
tmp->distance = distance;
tmp->length = lenval;
tmp->next = state->str;
state->str = tmp;
state->num++;
}
/*
* This function is used for query_txt parsing
*/
static void
pushval_asis(QPRS_STATE * state, int type, char *strval, int lenval)
{
if (lenval > 0xffff)
elog(ERROR, "Word is too long");
pushquery(state, type, crc32_sz((uint8 *) strval, lenval),
state->curop - state->op, lenval);
while (state->curop - state->op + lenval + 1 >= state->lenop)
{
int4 tmp = state->curop - state->op;
state->lenop *= 2;
state->op = (char *) repalloc((void *) state->op, state->lenop);
state->curop = state->op + tmp;
}
memcpy((void *) state->curop, (void *) strval, lenval);
state->curop += lenval;
*(state->curop) = '\0';
state->curop++;
state->sumlen += lenval + 1;
return;
}
/*
* This function is used for mquery_txt parsing
*/
static void
pushval_morph(QPRS_STATE * state, int typeval, char *strval, int lenval)
{
int4 type,
lenlemm;
int4 count = 0;
char *lemm;
start_parse_str(strval, lenval);
while ((type = tsearch_yylex()) != 0)
{
if (tokenlen > 0xffff)
{
end_parse();
elog(ERROR, "Word is too long");
}
lenlemm = tokenlen;
lemm = lemmatize(token, &lenlemm, type);
if (lemm)
{
pushval_asis(state, VAL, lemm, lenlemm);
if (lemm != token)
pfree(lemm);
}
else
pushval_asis(state, VALTRUE, 0, 0);
if (count)
pushquery(state, OPR, (int4) '&', 0, 0);
count++;
}
end_parse();
}
#define STACKDEPTH 32
/*
* make polish notaion of query
*/
static int4
makepol(QPRS_STATE * state, void (*pushval) (QPRS_STATE *, int, char *, int))
{
int4 val,
type;
int4 lenval;
char *strval;
int4 stack[STACKDEPTH];
int4 lenstack = 0;
while ((type = gettoken_query(state, &val, &lenval, &strval)) != END)
{
switch (type)
{
case VAL:
(*pushval) (state, VAL, strval, lenval);
while (lenstack && (stack[lenstack - 1] == (int4) '&' ||
stack[lenstack - 1] == (int4) '!'))
{
lenstack--;
pushquery(state, OPR, stack[lenstack], 0, 0);
}
break;
case OPR:
if (lenstack && val == (int4) '|')
pushquery(state, OPR, val, 0, 0);
else
{
if (lenstack == STACKDEPTH)
elog(ERROR, "Stack too short");
stack[lenstack] = val;
lenstack++;
}
break;
case OPEN:
if (makepol(state, pushval) == ERR)
return ERR;
if (lenstack && (stack[lenstack - 1] == (int4) '&' ||
stack[lenstack - 1] == (int4) '!'))
{
lenstack--;
pushquery(state, OPR, stack[lenstack], 0, 0);
}
break;
case CLOSE:
while (lenstack)
{
lenstack--;
pushquery(state, OPR, stack[lenstack], 0, 0);
};
return END;
break;
case ERR:
default:
elog(ERROR, "Syntax error");
return ERR;
}
}
while (lenstack)
{
lenstack--;
pushquery(state, OPR, stack[lenstack], 0, 0);
};
return END;
}
typedef struct
{
WordEntry *arrb;
WordEntry *arre;
char *values;
char *operand;
} CHKVAL;
/*
* compare 2 string values
*/
static int4
ValCompare(CHKVAL * chkval, WordEntry * ptr, ITEM * item)
{
if (ptr->len == item->length)
return strncmp(
&(chkval->values[ptr->pos]),
&(chkval->operand[item->distance]),
item->length);
return (ptr->len > item->length) ? 1 : -1;
}
/*
* is there value 'val' in array or not ?
*/
static bool
checkcondition_str(void *checkval, ITEM * val)
{
WordEntry *StopLow = ((CHKVAL *) checkval)->arrb;
WordEntry *StopHigh = ((CHKVAL *) checkval)->arre;
WordEntry *StopMiddle;
int difference;
/* Loop invariant: StopLow <= val < StopHigh */
while (StopLow < StopHigh)
{
StopMiddle = StopLow + (StopHigh - StopLow) / 2;
difference = ValCompare((CHKVAL *) checkval, StopMiddle, val);
if (difference == 0)
return (true);
else if (difference < 0)
StopLow = StopMiddle + 1;
else
StopHigh = StopMiddle;
}
return (false);
}
/*
* check for boolean condition
*/
bool
execute(ITEM * curitem, void *checkval, bool calcnot, bool (*chkcond) (void *checkval, ITEM * val))
{
if (curitem->type == VAL)
return (*chkcond) (checkval, curitem);
else if (curitem->val == (int4) '!')
{
return (calcnot) ?
((execute(curitem + 1, checkval, calcnot, chkcond)) ? false : true)
: true;
}
else if (curitem->val == (int4) '&')
{
if (execute(curitem + curitem->left, checkval, calcnot, chkcond))
return execute(curitem + 1, checkval, calcnot, chkcond);
else
return false;
}
else
{ /* |-operator */
if (execute(curitem + curitem->left, checkval, calcnot, chkcond))
return true;
else
return execute(curitem + 1, checkval, calcnot, chkcond);
}
return false;
}
/*
* boolean operations
*/
Datum
rexecqtxt(PG_FUNCTION_ARGS)
{
return DirectFunctionCall2(
execqtxt,
PG_GETARG_DATUM(1),
PG_GETARG_DATUM(0)
);
}
Datum
execqtxt(PG_FUNCTION_ARGS)
{
txtidx *val = (txtidx *) DatumGetPointer(PG_DETOAST_DATUM(PG_GETARG_DATUM(0)));
QUERYTYPE *query = (QUERYTYPE *) DatumGetPointer(PG_DETOAST_DATUM(PG_GETARG_DATUM(1)));
CHKVAL chkval;
bool result;
if (!val->size)
{
PG_FREE_IF_COPY(val, 0);
PG_FREE_IF_COPY(query, 1);
PG_RETURN_BOOL(false);
}
chkval.arrb = ARRPTR(val);
chkval.arre = chkval.arrb + val->size;
chkval.values = STRPTR(val);
chkval.operand = GETOPERAND(query);
result = execute(
GETQUERY(query),
&chkval,
true,
checkcondition_str
);
PG_FREE_IF_COPY(val, 0);
PG_FREE_IF_COPY(query, 1);
PG_RETURN_BOOL(result);
}
/*
* find left operand in polish notation view
*/
static void
findoprnd(ITEM * ptr, int4 *pos)
{
#ifdef BS_DEBUG
elog(NOTICE, (ptr[*pos].type == OPR) ?
"%d %c" : "%d %d ", *pos, ptr[*pos].val);
#endif
if (ptr[*pos].type == VAL || ptr[*pos].type == VALTRUE)
{
ptr[*pos].left = 0;
(*pos)++;
}
else if (ptr[*pos].val == (int4) '!')
{
ptr[*pos].left = 1;
(*pos)++;
findoprnd(ptr, pos);
}
else
{
ITEM *curitem = &ptr[*pos];
int4 tmp = *pos;
(*pos)++;
findoprnd(ptr, pos);
curitem->left = *pos - tmp;
findoprnd(ptr, pos);
}
}
/*
* input
*/
static QUERYTYPE *
queryin(char *buf, void (*pushval) (QPRS_STATE *, int, char *, int))
{
QPRS_STATE state;
int4 i;
QUERYTYPE *query;
int4 commonlen;
ITEM *ptr;
NODE *tmp;
int4 pos = 0;
#ifdef BS_DEBUG
char pbuf[16384],
*cur;
#endif
/* init state */
state.buf = buf;
state.state = WAITOPERAND;
state.count = 0;
state.num = 0;
state.str = NULL;
/* init value parser's state */
state.valstate.oprisdelim = true;
state.valstate.len = 32;
state.valstate.word = (char *) palloc(state.valstate.len);
/* init list of operand */
state.sumlen = 0;
state.lenop = 64;
state.curop = state.op = (char *) palloc(state.lenop);
*(state.curop) = '\0';
/* parse query & make polish notation (postfix, but in reverse order) */
makepol(&state, pushval);
pfree(state.valstate.word);
if (!state.num)
elog(ERROR, "Empty query");
/* make finish struct */
commonlen = COMPUTESIZE(state.num, state.sumlen);
query = (QUERYTYPE *) palloc(commonlen);
query->len = commonlen;
query->size = state.num;
ptr = GETQUERY(query);
/* set item in polish notation */
for (i = 0; i < state.num; i++)
{
ptr[i].type = state.str->type;
ptr[i].val = state.str->val;
ptr[i].distance = state.str->distance;
ptr[i].length = state.str->length;
tmp = state.str->next;
pfree(state.str);
state.str = tmp;
}
/* set user friendly-operand view */
memcpy((void *) GETOPERAND(query), (void *) state.op, state.sumlen);
pfree(state.op);
/* set left operand's position for every operator */
pos = 0;
findoprnd(ptr, &pos);
#ifdef BS_DEBUG
cur = pbuf;
*cur = '\0';
for (i = 0; i < query->size; i++)
{
if (ptr[i].type == OPR)
sprintf(cur, "%c(%d) ", ptr[i].val, ptr[i].left);
else
sprintf(cur, "%d(%s) ", ptr[i].val, GETOPERAND(query) + ptr[i].distance);
cur = strchr(cur, '\0');
}
elog(NOTICE, "POR: %s", pbuf);
#endif
return query;
}
/*
* in without morphology
*/
Datum
qtxt_in(PG_FUNCTION_ARGS)
{
PG_RETURN_POINTER(queryin((char *) PG_GETARG_POINTER(0), pushval_asis));
}
/*
* in with morphology
*/
Datum
mqtxt_in(PG_FUNCTION_ARGS)
{
QUERYTYPE *query;
ITEM *res;
int4 len;
#ifdef BS_DEBUG
ITEM *ptr;
int4 i;
char pbuf[16384],
*cur;
#endif
initmorph();
query = queryin((char *) PG_GETARG_POINTER(0), pushval_morph);
res = clean_fakeval(GETQUERY(query), &len);
if (!res)
{
pfree(query);
PG_RETURN_NULL();
}
memcpy((void *) GETQUERY(query), (void *) res, len * sizeof(ITEM));
#ifdef BS_DEBUG
cur = pbuf;
*cur = '\0';
ptr = GETQUERY(query);
for (i = 0; i < len; i++)
{
if (ptr[i].type == OPR)
sprintf(cur, "%c(%d) ", ptr[i].val, ptr[i].left);
else
sprintf(cur, "%d(%s) ", ptr[i].val, GETOPERAND(query) + ptr[i].distance);
cur = strchr(cur, '\0');
}
elog(NOTICE, "POR: %s", pbuf);
#endif
pfree(res);
PG_RETURN_POINTER(query);
}
/*
* out function
*/
typedef struct
{
ITEM *curpol;
char *buf;
char *cur;
char *op;
int4 buflen;
} INFIX;
#define RESIZEBUF(inf,addsize) \
while( ( inf->cur - inf->buf ) + addsize + 1 >= inf->buflen ) \
{ \
int4 len = inf->cur - inf->buf; \
inf->buflen *= 2; \
inf->buf = (char*) repalloc( (void*)inf->buf, inf->buflen ); \
inf->cur = inf->buf + len; \
}
/*
* recursive walk on tree and print it in
* infix (human-readable) view
*/
static void
infix(INFIX * in, bool first)
{
if (in->curpol->type == VAL)
{
char *op = in->op + in->curpol->distance;
RESIZEBUF(in, in->curpol->length * 2 + 2);
*(in->cur) = '\'';
in->cur++;
while (*op)
{
if (*op == '\'')
{
*(in->cur) = '\\';
in->cur++;
}
*(in->cur) = *op;
op++;
in->cur++;
}
*(in->cur) = '\'';
in->cur++;
*(in->cur) = '\0';
in->curpol++;
}
else if (in->curpol->val == (int4) '!')
{
bool isopr = false;
RESIZEBUF(in, 1);
*(in->cur) = '!';
in->cur++;
*(in->cur) = '\0';
in->curpol++;
if (in->curpol->type == OPR)
{
isopr = true;
RESIZEBUF(in, 2);
sprintf(in->cur, "( ");
in->cur = strchr(in->cur, '\0');
}
infix(in, isopr);
if (isopr)
{
RESIZEBUF(in, 2);
sprintf(in->cur, " )");
in->cur = strchr(in->cur, '\0');
}
}
else
{
int4 op = in->curpol->val;
INFIX nrm;
in->curpol++;
if (op == (int4) '|' && !first)
{
RESIZEBUF(in, 2);
sprintf(in->cur, "( ");
in->cur = strchr(in->cur, '\0');
}
nrm.curpol = in->curpol;
nrm.op = in->op;
nrm.buflen = 16;
nrm.cur = nrm.buf = (char *) palloc(sizeof(char) * nrm.buflen);
/* get right operand */
infix(&nrm, false);
/* get & print left operand */
in->curpol = nrm.curpol;
infix(in, false);
/* print operator & right operand */
RESIZEBUF(in, 3 + (nrm.cur - nrm.buf));
sprintf(in->cur, " %c %s", op, nrm.buf);
in->cur = strchr(in->cur, '\0');
pfree(nrm.buf);
if (op == (int4) '|' && !first)
{
RESIZEBUF(in, 2);
sprintf(in->cur, " )");
in->cur = strchr(in->cur, '\0');
}
}
}
Datum
qtxt_out(PG_FUNCTION_ARGS)
{
QUERYTYPE *query = (QUERYTYPE *) DatumGetPointer(PG_DETOAST_DATUM(PG_GETARG_DATUM(0)));
INFIX nrm;
if (query->size == 0)
elog(ERROR, "Empty");
nrm.curpol = GETQUERY(query);
nrm.buflen = 32;
nrm.cur = nrm.buf = (char *) palloc(sizeof(char) * nrm.buflen);
*(nrm.cur) = '\0';
nrm.op = GETOPERAND(query);
infix(&nrm, true);
PG_FREE_IF_COPY(query, 0);
PG_RETURN_POINTER(nrm.buf);
}
/*
* debug function, used only for view query
* which will be executed in non-leaf pages in index
*/
Datum
querytree(PG_FUNCTION_ARGS)
{
QUERYTYPE *query = (QUERYTYPE *) DatumGetPointer(PG_DETOAST_DATUM(PG_GETARG_DATUM(0)));
INFIX nrm;
text *res;
ITEM *q;
int4 len;
if (query->size == 0)
elog(ERROR, "Empty");
q = clean_NOT(GETQUERY(query), &len);
if (!q)
{
res = (text *) palloc(1 + VARHDRSZ);
VARATT_SIZEP(res) = 1 + VARHDRSZ;
*((char *) VARDATA(res)) = 'T';
}
else
{
nrm.curpol = q;
nrm.buflen = 32;
nrm.cur = nrm.buf = (char *) palloc(sizeof(char) * nrm.buflen);
*(nrm.cur) = '\0';
nrm.op = GETOPERAND(query);
infix(&nrm, true);
res = (text *) palloc(nrm.cur - nrm.buf + VARHDRSZ);
VARATT_SIZEP(res) = nrm.cur - nrm.buf + VARHDRSZ;
strncpy(VARDATA(res), nrm.buf, nrm.cur - nrm.buf);
pfree(q);
}
PG_FREE_IF_COPY(query, 0);
PG_RETURN_POINTER(res);
}