2007-08-21 03:11:32 +02:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* tsquery.c
|
|
|
|
* I/O functions for tsquery
|
|
|
|
*
|
2022-01-08 01:04:57 +01:00
|
|
|
* Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
|
2007-08-21 03:11:32 +02:00
|
|
|
*
|
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
2010-09-20 22:08:53 +02:00
|
|
|
* src/backend/utils/adt/tsquery.c
|
2007-08-21 03:11:32 +02:00
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include "libpq/pqformat.h"
|
2007-08-31 04:26:29 +02:00
|
|
|
#include "miscadmin.h"
|
2007-08-21 03:11:32 +02:00
|
|
|
#include "tsearch/ts_locale.h"
|
2019-11-12 04:00:16 +01:00
|
|
|
#include "tsearch/ts_type.h"
|
2007-08-21 03:11:32 +02:00
|
|
|
#include "tsearch/ts_utils.h"
|
2008-03-25 23:42:46 +01:00
|
|
|
#include "utils/builtins.h"
|
2007-08-21 03:11:32 +02:00
|
|
|
#include "utils/memutils.h"
|
2015-04-14 16:03:42 +02:00
|
|
|
#include "utils/pg_crc.h"
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2016-04-08 19:11:30 +02:00
|
|
|
/* FTS operator priorities, see ts_type.h */
|
|
|
|
const int tsearch_op_priority[OP_COUNT] =
|
|
|
|
{
|
2016-06-27 19:55:24 +02:00
|
|
|
4, /* OP_NOT */
|
2016-04-08 19:11:30 +02:00
|
|
|
2, /* OP_AND */
|
|
|
|
1, /* OP_OR */
|
2016-06-27 19:55:24 +02:00
|
|
|
3 /* OP_PHRASE */
|
2016-04-08 19:11:30 +02:00
|
|
|
};
|
2007-08-31 04:26:29 +02:00
|
|
|
|
2018-04-05 18:55:11 +02:00
|
|
|
/*
|
|
|
|
* parser's states
|
|
|
|
*/
|
|
|
|
typedef enum
|
|
|
|
{
|
|
|
|
WAITOPERAND = 1,
|
|
|
|
WAITOPERATOR = 2,
|
|
|
|
WAITFIRSTOPERAND = 3
|
|
|
|
} ts_parserstate;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* token types for parsing
|
|
|
|
*/
|
|
|
|
typedef enum
|
|
|
|
{
|
|
|
|
PT_END = 0,
|
|
|
|
PT_ERR = 1,
|
|
|
|
PT_VAL = 2,
|
|
|
|
PT_OPR = 3,
|
|
|
|
PT_OPEN = 4,
|
|
|
|
PT_CLOSE = 5
|
|
|
|
} ts_tokentype;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* get token from query string
|
|
|
|
*
|
|
|
|
* *operator is filled in with OP_* when return values is PT_OPR,
|
|
|
|
* but *weight could contain a distance value in case of phrase operator.
|
|
|
|
* *strval, *lenval and *weight are filled in when return value is PT_VAL
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
typedef ts_tokentype (*ts_tokenizer) (TSQueryParserState state, int8 *operator,
|
|
|
|
int *lenval, char **strval,
|
|
|
|
int16 *weight, bool *prefix);
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
struct TSQueryParserStateData
|
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
/* Tokenizer used for parsing tsquery */
|
|
|
|
ts_tokenizer gettoken;
|
|
|
|
|
|
|
|
/* State of tokenizer function */
|
2007-09-07 17:09:56 +02:00
|
|
|
char *buffer; /* entire string we are scanning */
|
|
|
|
char *buf; /* current scan point */
|
|
|
|
int count; /* nesting count, incremented by (,
|
|
|
|
* decremented by ) */
|
2018-04-05 18:55:11 +02:00
|
|
|
ts_parserstate state;
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
/* polish (prefix) notation in list, filled in by push* functions */
|
|
|
|
List *polstr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Strings from operands are collected in op. curop is a pointer to the
|
|
|
|
* end of used space of op.
|
|
|
|
*/
|
|
|
|
char *op;
|
|
|
|
char *curop;
|
|
|
|
int lenop; /* allocated size of op */
|
|
|
|
int sumlen; /* used size of op */
|
|
|
|
|
|
|
|
/* state for value's parser */
|
|
|
|
TSVectorParseState valstate;
|
|
|
|
};
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
/*
|
2008-05-16 18:31:02 +02:00
|
|
|
* subroutine to parse the modifiers (weight and prefix flag currently)
|
2016-04-07 17:44:18 +02:00
|
|
|
* part, like ':AB*' of a query.
|
2007-08-21 03:11:32 +02:00
|
|
|
*/
|
|
|
|
static char *
|
2008-05-16 18:31:02 +02:00
|
|
|
get_modifiers(char *buf, int16 *weight, bool *prefix)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
*weight = 0;
|
2008-05-16 18:31:02 +02:00
|
|
|
*prefix = false;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
if (!t_iseq(buf, ':'))
|
|
|
|
return buf;
|
|
|
|
|
|
|
|
buf++;
|
|
|
|
while (*buf && pg_mblen(buf) == 1)
|
|
|
|
{
|
|
|
|
switch (*buf)
|
|
|
|
{
|
|
|
|
case 'a':
|
|
|
|
case 'A':
|
|
|
|
*weight |= 1 << 3;
|
|
|
|
break;
|
|
|
|
case 'b':
|
|
|
|
case 'B':
|
|
|
|
*weight |= 1 << 2;
|
|
|
|
break;
|
|
|
|
case 'c':
|
|
|
|
case 'C':
|
|
|
|
*weight |= 1 << 1;
|
|
|
|
break;
|
|
|
|
case 'd':
|
|
|
|
case 'D':
|
|
|
|
*weight |= 1;
|
|
|
|
break;
|
2008-05-16 18:31:02 +02:00
|
|
|
case '*':
|
|
|
|
*prefix = true;
|
|
|
|
break;
|
2007-08-21 03:11:32 +02:00
|
|
|
default:
|
|
|
|
return buf;
|
|
|
|
}
|
|
|
|
buf++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return buf;
|
|
|
|
}
|
|
|
|
|
2016-04-07 17:44:18 +02:00
|
|
|
/*
|
|
|
|
* Parse phrase operator. The operator
|
|
|
|
* may take the following forms:
|
|
|
|
*
|
2017-06-26 16:31:10 +02:00
|
|
|
* a <N> b (distance is exactly N lexemes)
|
2016-04-07 17:44:18 +02:00
|
|
|
* a <-> b (default distance = 1)
|
|
|
|
*
|
|
|
|
* The buffer should begin with '<' char
|
|
|
|
*/
|
2018-04-05 18:55:11 +02:00
|
|
|
static bool
|
|
|
|
parse_phrase_operator(TSQueryParserState pstate, int16 *distance)
|
2016-04-07 17:44:18 +02:00
|
|
|
{
|
|
|
|
enum
|
|
|
|
{
|
|
|
|
PHRASE_OPEN = 0,
|
|
|
|
PHRASE_DIST,
|
|
|
|
PHRASE_CLOSE,
|
|
|
|
PHRASE_FINISH
|
|
|
|
} state = PHRASE_OPEN;
|
2018-04-05 18:55:11 +02:00
|
|
|
char *ptr = pstate->buf;
|
2016-04-07 17:44:18 +02:00
|
|
|
char *endptr;
|
2017-06-26 16:31:10 +02:00
|
|
|
long l = 1; /* default distance */
|
2016-04-07 17:44:18 +02:00
|
|
|
|
|
|
|
while (*ptr)
|
|
|
|
{
|
|
|
|
switch (state)
|
|
|
|
{
|
|
|
|
case PHRASE_OPEN:
|
2018-04-05 18:55:11 +02:00
|
|
|
if (t_iseq(ptr, '<'))
|
|
|
|
{
|
|
|
|
state = PHRASE_DIST;
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
return false;
|
2016-04-07 17:44:18 +02:00
|
|
|
break;
|
|
|
|
|
|
|
|
case PHRASE_DIST:
|
|
|
|
if (t_iseq(ptr, '-'))
|
|
|
|
{
|
|
|
|
state = PHRASE_CLOSE;
|
|
|
|
ptr++;
|
2018-04-05 18:55:11 +02:00
|
|
|
continue;
|
2016-04-07 17:44:18 +02:00
|
|
|
}
|
2018-04-05 18:55:11 +02:00
|
|
|
|
2017-06-26 16:31:10 +02:00
|
|
|
if (!t_isdigit(ptr))
|
2018-04-05 18:55:11 +02:00
|
|
|
return false;
|
2016-04-07 17:44:18 +02:00
|
|
|
|
2017-06-26 16:31:10 +02:00
|
|
|
errno = 0;
|
2016-04-07 17:44:18 +02:00
|
|
|
l = strtol(ptr, &endptr, 10);
|
|
|
|
if (ptr == endptr)
|
2018-04-05 18:55:11 +02:00
|
|
|
return false;
|
2017-06-26 16:31:10 +02:00
|
|
|
else if (errno == ERANGE || l < 0 || l > MAXENTRYPOS)
|
2016-04-07 17:44:18 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
2021-08-25 04:43:56 +02:00
|
|
|
errmsg("distance in phrase operator must be an integer value between zero and %d inclusive",
|
2016-04-07 17:44:18 +02:00
|
|
|
MAXENTRYPOS)));
|
|
|
|
else
|
|
|
|
{
|
|
|
|
state = PHRASE_CLOSE;
|
|
|
|
ptr = endptr;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case PHRASE_CLOSE:
|
|
|
|
if (t_iseq(ptr, '>'))
|
|
|
|
{
|
|
|
|
state = PHRASE_FINISH;
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
else
|
2018-04-05 18:55:11 +02:00
|
|
|
return false;
|
2016-04-07 17:44:18 +02:00
|
|
|
break;
|
|
|
|
|
|
|
|
case PHRASE_FINISH:
|
|
|
|
*distance = (int16) l;
|
2018-04-05 18:55:11 +02:00
|
|
|
pstate->buf = ptr;
|
|
|
|
return true;
|
2016-04-07 17:44:18 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-05 18:55:11 +02:00
|
|
|
return false;
|
2016-04-07 17:44:18 +02:00
|
|
|
}
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/*
|
2018-04-05 18:55:11 +02:00
|
|
|
* Parse OR operator used in websearch_to_tsquery(), returns true if we
|
|
|
|
* believe that "OR" literal could be an operator OR
|
2007-09-07 17:09:56 +02:00
|
|
|
*/
|
2018-04-05 18:55:11 +02:00
|
|
|
static bool
|
|
|
|
parse_or_operator(TSQueryParserState pstate)
|
2007-09-07 17:09:56 +02:00
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
char *ptr = pstate->buf;
|
|
|
|
|
|
|
|
/* it should begin with "OR" literal */
|
|
|
|
if (pg_strncasecmp(ptr, "or", 2) != 0)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
ptr += 2;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* it shouldn't be a part of any word but somewhere later it should be
|
|
|
|
* some operand
|
|
|
|
*/
|
|
|
|
if (*ptr == '\0') /* no operand */
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* it shouldn't be a part of any word */
|
|
|
|
if (t_iseq(ptr, '-') || t_iseq(ptr, '_') || t_isalpha(ptr) || t_isdigit(ptr))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
for (;;)
|
|
|
|
{
|
|
|
|
ptr += pg_mblen(ptr);
|
|
|
|
|
|
|
|
if (*ptr == '\0') /* got end of string without operand */
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Suppose, we found an operand, but could be a not correct operand.
|
|
|
|
* So we still treat OR literal as operation with possibly incorrect
|
2022-04-11 10:49:41 +02:00
|
|
|
* operand and will not search it as lexeme
|
2018-04-05 18:55:11 +02:00
|
|
|
*/
|
|
|
|
if (!t_isspace(ptr))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
pstate->buf += 2;
|
|
|
|
return true;
|
|
|
|
}
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
static ts_tokentype
|
2018-04-05 18:55:11 +02:00
|
|
|
gettoken_query_standard(TSQueryParserState state, int8 *operator,
|
|
|
|
int *lenval, char **strval,
|
|
|
|
int16 *weight, bool *prefix)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2008-05-16 18:31:02 +02:00
|
|
|
*weight = 0;
|
|
|
|
*prefix = false;
|
|
|
|
|
2018-04-05 18:55:11 +02:00
|
|
|
while (true)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
switch (state->state)
|
|
|
|
{
|
|
|
|
case WAITFIRSTOPERAND:
|
|
|
|
case WAITOPERAND:
|
|
|
|
if (t_iseq(state->buf, '!'))
|
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
state->buf++;
|
2007-08-21 03:11:32 +02:00
|
|
|
state->state = WAITOPERAND;
|
2018-04-05 18:55:11 +02:00
|
|
|
*operator = OP_NOT;
|
2007-09-07 17:09:56 +02:00
|
|
|
return PT_OPR;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
else if (t_iseq(state->buf, '('))
|
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
state->buf++;
|
2007-08-21 03:11:32 +02:00
|
|
|
state->state = WAITOPERAND;
|
2018-04-05 18:55:11 +02:00
|
|
|
state->count++;
|
2007-09-07 17:09:56 +02:00
|
|
|
return PT_OPEN;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
else if (t_iseq(state->buf, ':'))
|
|
|
|
{
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2007-10-22 00:29:56 +02:00
|
|
|
errmsg("syntax error in tsquery: \"%s\"",
|
2007-08-21 03:11:32 +02:00
|
|
|
state->buffer)));
|
|
|
|
}
|
|
|
|
else if (!t_isspace(state->buf))
|
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
/*
|
|
|
|
* We rely on the tsvector parser to parse the value for
|
|
|
|
* us
|
|
|
|
*/
|
2007-09-07 17:09:56 +02:00
|
|
|
reset_tsvector_parser(state->valstate, state->buf);
|
2018-04-05 18:55:11 +02:00
|
|
|
if (gettoken_tsvector(state->valstate, strval, lenval,
|
|
|
|
NULL, NULL, &state->buf))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2008-05-16 18:31:02 +02:00
|
|
|
state->buf = get_modifiers(state->buf, weight, prefix);
|
2007-08-21 03:11:32 +02:00
|
|
|
state->state = WAITOPERATOR;
|
2007-09-07 17:09:56 +02:00
|
|
|
return PT_VAL;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
else if (state->state == WAITFIRSTOPERAND)
|
2018-04-05 18:55:11 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
return PT_END;
|
2018-04-05 18:55:11 +02:00
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2007-10-22 00:29:56 +02:00
|
|
|
errmsg("no operand in tsquery: \"%s\"",
|
2007-08-21 03:11:32 +02:00
|
|
|
state->buffer)));
|
|
|
|
}
|
|
|
|
break;
|
2018-04-05 18:55:11 +02:00
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
case WAITOPERATOR:
|
2007-09-07 17:09:56 +02:00
|
|
|
if (t_iseq(state->buf, '&'))
|
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
state->buf++;
|
2007-09-07 17:09:56 +02:00
|
|
|
state->state = WAITOPERAND;
|
|
|
|
*operator = OP_AND;
|
|
|
|
return PT_OPR;
|
|
|
|
}
|
2016-04-07 17:44:18 +02:00
|
|
|
else if (t_iseq(state->buf, '|'))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
state->buf++;
|
2007-08-21 03:11:32 +02:00
|
|
|
state->state = WAITOPERAND;
|
2007-09-07 17:09:56 +02:00
|
|
|
*operator = OP_OR;
|
|
|
|
return PT_OPR;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
2018-04-05 18:55:11 +02:00
|
|
|
else if (parse_phrase_operator(state, weight))
|
2016-04-07 17:44:18 +02:00
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
/* weight var is used as storage for distance */
|
2016-04-07 17:44:18 +02:00
|
|
|
state->state = WAITOPERAND;
|
|
|
|
*operator = OP_PHRASE;
|
|
|
|
return PT_OPR;
|
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
else if (t_iseq(state->buf, ')'))
|
|
|
|
{
|
2018-04-05 18:55:11 +02:00
|
|
|
state->buf++;
|
2007-08-21 03:11:32 +02:00
|
|
|
state->count--;
|
2007-09-07 17:09:56 +02:00
|
|
|
return (state->count < 0) ? PT_ERR : PT_CLOSE;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
2018-04-05 18:55:11 +02:00
|
|
|
else if (*state->buf == '\0')
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
return (state->count) ? PT_ERR : PT_END;
|
2018-04-05 18:55:11 +02:00
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
else if (!t_isspace(state->buf))
|
2018-04-05 18:55:11 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
return PT_ERR;
|
2018-04-05 18:55:11 +02:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
state->buf += pg_mblen(state->buf);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static ts_tokentype
|
|
|
|
gettoken_query_websearch(TSQueryParserState state, int8 *operator,
|
|
|
|
int *lenval, char **strval,
|
|
|
|
int16 *weight, bool *prefix)
|
|
|
|
{
|
|
|
|
*weight = 0;
|
|
|
|
*prefix = false;
|
|
|
|
|
|
|
|
while (true)
|
|
|
|
{
|
|
|
|
switch (state->state)
|
|
|
|
{
|
|
|
|
case WAITFIRSTOPERAND:
|
|
|
|
case WAITOPERAND:
|
|
|
|
if (t_iseq(state->buf, '-'))
|
|
|
|
{
|
|
|
|
state->buf++;
|
|
|
|
state->state = WAITOPERAND;
|
|
|
|
|
|
|
|
*operator = OP_NOT;
|
|
|
|
return PT_OPR;
|
|
|
|
}
|
|
|
|
else if (t_iseq(state->buf, '"'))
|
|
|
|
{
|
2021-05-03 02:58:03 +02:00
|
|
|
/* Everything in quotes is processed as a single token */
|
|
|
|
|
|
|
|
/* skip opening quote */
|
2018-04-05 18:55:11 +02:00
|
|
|
state->buf++;
|
2021-05-03 02:58:03 +02:00
|
|
|
*strval = state->buf;
|
2018-04-05 18:55:11 +02:00
|
|
|
|
2021-05-03 02:58:03 +02:00
|
|
|
/* iterate to the closing quote or end of the string */
|
|
|
|
while (*state->buf != '\0' && !t_iseq(state->buf, '"'))
|
|
|
|
state->buf++;
|
|
|
|
*lenval = state->buf - *strval;
|
2018-04-05 18:55:11 +02:00
|
|
|
|
2021-05-03 02:58:03 +02:00
|
|
|
/* skip closing quote if not end of the string */
|
|
|
|
if (*state->buf != '\0')
|
|
|
|
state->buf++;
|
2018-04-05 18:55:11 +02:00
|
|
|
|
2021-05-03 02:58:03 +02:00
|
|
|
state->state = WAITOPERATOR;
|
|
|
|
state->count++;
|
|
|
|
return PT_VAL;
|
2018-04-05 18:55:11 +02:00
|
|
|
}
|
|
|
|
else if (ISOPERATOR(state->buf))
|
|
|
|
{
|
|
|
|
/* or else gettoken_tsvector() will raise an error */
|
|
|
|
state->buf++;
|
|
|
|
state->state = WAITOPERAND;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
else if (!t_isspace(state->buf))
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We rely on the tsvector parser to parse the value for
|
|
|
|
* us
|
|
|
|
*/
|
|
|
|
reset_tsvector_parser(state->valstate, state->buf);
|
|
|
|
if (gettoken_tsvector(state->valstate, strval, lenval,
|
|
|
|
NULL, NULL, &state->buf))
|
|
|
|
{
|
|
|
|
state->state = WAITOPERATOR;
|
|
|
|
return PT_VAL;
|
|
|
|
}
|
|
|
|
else if (state->state == WAITFIRSTOPERAND)
|
|
|
|
{
|
|
|
|
return PT_END;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* finally, we have to provide an operand */
|
|
|
|
pushStop(state);
|
|
|
|
return PT_END;
|
|
|
|
}
|
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
break;
|
2018-04-05 18:55:11 +02:00
|
|
|
|
|
|
|
case WAITOPERATOR:
|
|
|
|
if (t_iseq(state->buf, '"'))
|
|
|
|
{
|
2021-05-03 02:58:03 +02:00
|
|
|
/*
|
|
|
|
* put implicit AND after an operand and handle this quote
|
|
|
|
* in WAITOPERAND
|
|
|
|
*/
|
|
|
|
state->state = WAITOPERAND;
|
|
|
|
*operator = OP_AND;
|
|
|
|
return PT_OPR;
|
2018-04-05 18:55:11 +02:00
|
|
|
}
|
|
|
|
else if (parse_or_operator(state))
|
|
|
|
{
|
|
|
|
state->state = WAITOPERAND;
|
|
|
|
*operator = OP_OR;
|
|
|
|
return PT_OPR;
|
|
|
|
}
|
|
|
|
else if (*state->buf == '\0')
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
return PT_END;
|
2018-04-05 18:55:11 +02:00
|
|
|
}
|
|
|
|
else if (!t_isspace(state->buf))
|
|
|
|
{
|
2021-05-03 02:58:03 +02:00
|
|
|
/* put implicit AND after an operand */
|
|
|
|
*operator = OP_AND;
|
2018-04-05 18:55:11 +02:00
|
|
|
state->state = WAITOPERAND;
|
|
|
|
return PT_OPR;
|
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
break;
|
|
|
|
}
|
2018-04-05 18:55:11 +02:00
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
state->buf += pg_mblen(state->buf);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-05 18:55:11 +02:00
|
|
|
static ts_tokentype
|
|
|
|
gettoken_query_plain(TSQueryParserState state, int8 *operator,
|
|
|
|
int *lenval, char **strval,
|
|
|
|
int16 *weight, bool *prefix)
|
|
|
|
{
|
|
|
|
*weight = 0;
|
|
|
|
*prefix = false;
|
|
|
|
|
|
|
|
if (*state->buf == '\0')
|
|
|
|
return PT_END;
|
|
|
|
|
|
|
|
*strval = state->buf;
|
|
|
|
*lenval = strlen(state->buf);
|
|
|
|
state->buf += *lenval;
|
|
|
|
state->count++;
|
|
|
|
return PT_VAL;
|
|
|
|
}
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
/*
|
2007-09-07 17:09:56 +02:00
|
|
|
* Push an operator to state->polstr
|
2007-08-21 03:11:32 +02:00
|
|
|
*/
|
|
|
|
void
|
2016-04-07 17:44:18 +02:00
|
|
|
pushOperator(TSQueryParserState state, int8 oper, int16 distance)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
QueryOperator *tmp;
|
|
|
|
|
2016-04-07 17:44:18 +02:00
|
|
|
Assert(oper == OP_NOT || oper == OP_AND || oper == OP_OR || oper == OP_PHRASE);
|
2007-11-15 22:14:46 +01:00
|
|
|
|
2008-04-12 00:52:05 +02:00
|
|
|
tmp = (QueryOperator *) palloc0(sizeof(QueryOperator));
|
2007-09-07 17:09:56 +02:00
|
|
|
tmp->type = QI_OPR;
|
|
|
|
tmp->oper = oper;
|
2016-04-07 17:44:18 +02:00
|
|
|
tmp->distance = (oper == OP_PHRASE) ? distance : 0;
|
2007-09-07 17:09:56 +02:00
|
|
|
/* left is filled in later with findoprnd */
|
|
|
|
|
|
|
|
state->polstr = lcons(tmp, state->polstr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2008-05-16 18:31:02 +02:00
|
|
|
pushValue_internal(TSQueryParserState state, pg_crc32 valcrc, int distance, int lenval, int weight, bool prefix)
|
2007-09-07 17:09:56 +02:00
|
|
|
{
|
|
|
|
QueryOperand *tmp;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
if (distance >= MAXSTRPOS)
|
|
|
|
ereport(ERROR,
|
2007-11-28 22:56:30 +01:00
|
|
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
2007-10-22 00:29:56 +02:00
|
|
|
errmsg("value is too big in tsquery: \"%s\"",
|
2007-08-21 03:11:32 +02:00
|
|
|
state->buffer)));
|
|
|
|
if (lenval >= MAXSTRLEN)
|
|
|
|
ereport(ERROR,
|
2007-11-28 22:56:30 +01:00
|
|
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
2007-10-22 00:29:56 +02:00
|
|
|
errmsg("operand is too long in tsquery: \"%s\"",
|
2007-08-21 03:11:32 +02:00
|
|
|
state->buffer)));
|
2007-09-07 17:09:56 +02:00
|
|
|
|
2008-04-12 00:52:05 +02:00
|
|
|
tmp = (QueryOperand *) palloc0(sizeof(QueryOperand));
|
2007-09-07 17:09:56 +02:00
|
|
|
tmp->type = QI_VAL;
|
|
|
|
tmp->weight = weight;
|
2008-05-16 18:31:02 +02:00
|
|
|
tmp->prefix = prefix;
|
2007-09-07 17:09:56 +02:00
|
|
|
tmp->valcrc = (int32) valcrc;
|
2007-08-21 03:11:32 +02:00
|
|
|
tmp->length = lenval;
|
2007-09-07 17:09:56 +02:00
|
|
|
tmp->distance = distance;
|
|
|
|
|
|
|
|
state->polstr = lcons(tmp, state->polstr);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2007-09-07 17:09:56 +02:00
|
|
|
* Push an operand to state->polstr.
|
|
|
|
*
|
|
|
|
* strval must point to a string equal to state->curop. lenval is the length
|
|
|
|
* of the string.
|
2007-08-21 03:11:32 +02:00
|
|
|
*/
|
|
|
|
void
|
2012-06-25 00:51:46 +02:00
|
|
|
pushValue(TSQueryParserState state, char *strval, int lenval, int16 weight, bool prefix)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
pg_crc32 valcrc;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
if (lenval >= MAXSTRLEN)
|
|
|
|
ereport(ERROR,
|
2007-11-28 22:56:30 +01:00
|
|
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
2007-10-22 00:29:56 +02:00
|
|
|
errmsg("word is too long in tsquery: \"%s\"",
|
2007-08-21 03:11:32 +02:00
|
|
|
state->buffer)));
|
|
|
|
|
Switch to CRC-32C in WAL and other places.
The old algorithm was found to not be the usual CRC-32 algorithm, used by
Ethernet et al. We were using a non-reflected lookup table with code meant
for a reflected lookup table. That's a strange combination that AFAICS does
not correspond to any bit-wise CRC calculation, which makes it difficult to
reason about its properties. Although it has worked well in practice, seems
safer to use a well-known algorithm.
Since we're changing the algorithm anyway, we might as well choose a
different polynomial. The Castagnoli polynomial has better error-correcting
properties than the traditional CRC-32 polynomial, even if we had
implemented it correctly. Another reason for picking that is that some new
CPUs have hardware support for calculating CRC-32C, but not CRC-32, let
alone our strange variant of it. This patch doesn't add any support for such
hardware, but a future patch could now do that.
The old algorithm is kept around for tsquery and pg_trgm, which use the
values in indexes that need to remain compatible so that pg_upgrade works.
While we're at it, share the old lookup table for CRC-32 calculation
between hstore, ltree and core. They all use the same table, so might as
well.
2014-11-04 10:35:15 +01:00
|
|
|
INIT_LEGACY_CRC32(valcrc);
|
|
|
|
COMP_LEGACY_CRC32(valcrc, strval, lenval);
|
|
|
|
FIN_LEGACY_CRC32(valcrc);
|
2008-05-16 18:31:02 +02:00
|
|
|
pushValue_internal(state, valcrc, state->curop - state->op, lenval, weight, prefix);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/* append the value string to state.op, enlarging buffer if needed first */
|
2007-08-21 03:11:32 +02:00
|
|
|
while (state->curop - state->op + lenval + 1 >= state->lenop)
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
int used = state->curop - state->op;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
state->lenop *= 2;
|
|
|
|
state->op = (char *) repalloc((void *) state->op, state->lenop);
|
2007-09-07 17:09:56 +02:00
|
|
|
state->curop = state->op + used;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
memcpy((void *) state->curop, (void *) strval, lenval);
|
|
|
|
state->curop += lenval;
|
|
|
|
*(state->curop) = '\0';
|
|
|
|
state->curop++;
|
|
|
|
state->sumlen += lenval + 1 /* \0 */ ;
|
|
|
|
}
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Push a stopword placeholder to state->polstr
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
pushStop(TSQueryParserState state)
|
|
|
|
{
|
|
|
|
QueryOperand *tmp;
|
|
|
|
|
2008-04-12 00:52:05 +02:00
|
|
|
tmp = (QueryOperand *) palloc0(sizeof(QueryOperand));
|
2007-09-07 17:09:56 +02:00
|
|
|
tmp->type = QI_VALSTOP;
|
|
|
|
|
|
|
|
state->polstr = lcons(tmp, state->polstr);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
#define STACKDEPTH 32
|
2007-08-31 04:26:29 +02:00
|
|
|
|
2016-06-27 19:55:24 +02:00
|
|
|
typedef struct OperatorElement
|
|
|
|
{
|
|
|
|
int8 op;
|
|
|
|
int16 distance;
|
|
|
|
} OperatorElement;
|
|
|
|
|
|
|
|
static void
|
|
|
|
pushOpStack(OperatorElement *stack, int *lenstack, int8 op, int16 distance)
|
|
|
|
{
|
|
|
|
if (*lenstack == STACKDEPTH) /* internal error */
|
|
|
|
elog(ERROR, "tsquery stack too small");
|
|
|
|
|
|
|
|
stack[*lenstack].op = op;
|
|
|
|
stack[*lenstack].distance = distance;
|
|
|
|
|
|
|
|
(*lenstack)++;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
cleanOpStack(TSQueryParserState state,
|
|
|
|
OperatorElement *stack, int *lenstack, int8 op)
|
|
|
|
{
|
|
|
|
int opPriority = OP_PRIORITY(op);
|
|
|
|
|
|
|
|
while (*lenstack)
|
|
|
|
{
|
2016-07-15 19:01:41 +02:00
|
|
|
/* NOT is right associative unlike to others */
|
|
|
|
if ((op != OP_NOT && opPriority > OP_PRIORITY(stack[*lenstack - 1].op)) ||
|
|
|
|
(op == OP_NOT && opPriority >= OP_PRIORITY(stack[*lenstack - 1].op)))
|
2016-06-27 19:55:24 +02:00
|
|
|
break;
|
|
|
|
|
|
|
|
(*lenstack)--;
|
|
|
|
pushOperator(state, stack[*lenstack].op,
|
|
|
|
stack[*lenstack].distance);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
/*
|
2007-09-07 17:09:56 +02:00
|
|
|
* Make polish (prefix) notation of query.
|
|
|
|
*
|
|
|
|
* See parse_tsquery for explanation of pushval.
|
2007-08-21 03:11:32 +02:00
|
|
|
*/
|
2007-09-07 17:09:56 +02:00
|
|
|
static void
|
|
|
|
makepol(TSQueryParserState state,
|
|
|
|
PushFunction pushval,
|
2007-09-10 14:36:41 +02:00
|
|
|
Datum opaque)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2016-04-07 17:44:18 +02:00
|
|
|
int8 operator = 0;
|
|
|
|
ts_tokentype type;
|
|
|
|
int lenval = 0;
|
|
|
|
char *strval = NULL;
|
2016-06-27 19:55:24 +02:00
|
|
|
OperatorElement opstack[STACKDEPTH];
|
2016-04-07 17:44:18 +02:00
|
|
|
int lenstack = 0;
|
|
|
|
int16 weight = 0;
|
|
|
|
bool prefix;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-08-31 04:26:29 +02:00
|
|
|
/* since this function recurses, it could be driven to stack overflow */
|
|
|
|
check_stack_depth();
|
|
|
|
|
2018-04-05 18:55:11 +02:00
|
|
|
while ((type = state->gettoken(state, &operator,
|
|
|
|
&lenval, &strval,
|
|
|
|
&weight, &prefix)) != PT_END)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
switch (type)
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
case PT_VAL:
|
2008-05-16 18:31:02 +02:00
|
|
|
pushval(opaque, state, strval, lenval, weight, prefix);
|
2007-08-21 03:11:32 +02:00
|
|
|
break;
|
2007-09-07 17:09:56 +02:00
|
|
|
case PT_OPR:
|
2016-06-27 19:55:24 +02:00
|
|
|
cleanOpStack(state, opstack, &lenstack, operator);
|
|
|
|
pushOpStack(opstack, &lenstack, operator, weight);
|
2007-08-21 03:11:32 +02:00
|
|
|
break;
|
2007-09-07 17:09:56 +02:00
|
|
|
case PT_OPEN:
|
|
|
|
makepol(state, pushval, opaque);
|
2007-08-21 03:11:32 +02:00
|
|
|
break;
|
2007-09-07 17:09:56 +02:00
|
|
|
case PT_CLOSE:
|
2016-06-27 19:55:24 +02:00
|
|
|
cleanOpStack(state, opstack, &lenstack, OP_OR /* lowest */ );
|
2007-09-07 17:09:56 +02:00
|
|
|
return;
|
|
|
|
case PT_ERR:
|
2007-08-21 03:11:32 +02:00
|
|
|
default:
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2007-10-22 00:29:56 +02:00
|
|
|
errmsg("syntax error in tsquery: \"%s\"",
|
2007-08-21 03:11:32 +02:00
|
|
|
state->buffer)));
|
|
|
|
}
|
|
|
|
}
|
2016-06-27 19:55:24 +02:00
|
|
|
|
|
|
|
cleanOpStack(state, opstack, &lenstack, OP_OR /* lowest */ );
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2016-04-07 17:44:18 +02:00
|
|
|
findoprnd_recurse(QueryItem *ptr, uint32 *pos, int nnodes, bool *needcleanup)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
/* since this function recurses, it could be driven to stack overflow. */
|
|
|
|
check_stack_depth();
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
if (*pos >= nnodes)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "malformed tsquery: operand not found");
|
2007-09-07 18:03:40 +02:00
|
|
|
|
2016-04-07 17:44:18 +02:00
|
|
|
if (ptr[*pos].type == QI_VAL)
|
|
|
|
{
|
|
|
|
(*pos)++;
|
|
|
|
}
|
|
|
|
else if (ptr[*pos].type == QI_VALSTOP)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2016-04-07 17:44:18 +02:00
|
|
|
*needcleanup = true; /* we'll have to remove stop words */
|
2007-08-21 03:11:32 +02:00
|
|
|
(*pos)++;
|
|
|
|
}
|
2007-09-07 17:09:56 +02:00
|
|
|
else
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
Assert(ptr[*pos].type == QI_OPR);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2009-07-16 08:33:46 +02:00
|
|
|
if (ptr[*pos].qoperator.oper == OP_NOT)
|
2007-09-07 17:09:56 +02:00
|
|
|
{
|
2016-04-07 17:44:18 +02:00
|
|
|
ptr[*pos].qoperator.left = 1; /* fixed offset */
|
2007-09-07 17:09:56 +02:00
|
|
|
(*pos)++;
|
2016-04-07 17:44:18 +02:00
|
|
|
|
|
|
|
/* process the only argument */
|
|
|
|
findoprnd_recurse(ptr, pos, nnodes, needcleanup);
|
2007-09-07 17:09:56 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2016-04-07 17:44:18 +02:00
|
|
|
QueryOperator *curitem = &ptr[*pos].qoperator;
|
|
|
|
int tmp = *pos; /* save current position */
|
|
|
|
|
|
|
|
Assert(curitem->oper == OP_AND ||
|
|
|
|
curitem->oper == OP_OR ||
|
|
|
|
curitem->oper == OP_PHRASE);
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
(*pos)++;
|
2016-04-07 17:44:18 +02:00
|
|
|
|
|
|
|
/* process RIGHT argument */
|
|
|
|
findoprnd_recurse(ptr, pos, nnodes, needcleanup);
|
Fix strange behavior (and possible crashes) in full text phrase search.
In an attempt to simplify the tsquery matching engine, the original
phrase search patch invented rewrite rules that would rearrange a
tsquery so that no AND/OR/NOT operator appeared below a PHRASE operator.
But this approach had numerous problems. The rearrangement step was
missed by ts_rewrite (and perhaps other places), allowing tsqueries
to be created that would cause Assert failures or perhaps crashes at
execution, as reported by Andreas Seltenreich. The rewrite rules
effectively defined semantics for operators underneath PHRASE that were
buggy, or at least unintuitive. And because rewriting was done in
tsqueryin() rather than at execution, the rearrangement was user-visible,
which is not very desirable --- for example, it might cause unexpected
matches or failures to match in ts_rewrite.
As a somewhat independent problem, the behavior of nested PHRASE operators
was only sane for left-deep trees; queries like "x <-> (y <-> z)" did not
behave intuitively at all.
To fix, get rid of the rewrite logic altogether, and instead teach the
tsquery execution engine to manage AND/OR/NOT below a PHRASE operator
by explicitly computing the match location(s) and match widths for these
operators.
This requires introducing some additional fields into the publicly visible
ExecPhraseData struct; but since there's no way for third-party code to
pass such a struct to TS_phrase_execute, it shouldn't create an ABI problem
as long as we don't move the offsets of the existing fields.
Another related problem was that index searches supposed that "!x <-> y"
could be lossily approximated as "!x & y", which isn't correct because
the latter will reject, say, "x q y" which the query itself accepts.
This required some tweaking in TS_execute_ternary along with the main
tsquery engine.
Back-patch to 9.6 where phrase operators were introduced. While this
could be argued to change behavior more than we'd like in a stable branch,
we have to do something about the crash hazards and index-vs-seqscan
inconsistency, and it doesn't seem desirable to let the unintuitive
behaviors induced by the rewriting implementation stand as precedent.
Discussion: https://postgr.es/m/28215.1481999808@sss.pgh.pa.us
Discussion: https://postgr.es/m/26706.1482087250@sss.pgh.pa.us
2016-12-21 21:18:25 +01:00
|
|
|
|
2016-04-07 17:44:18 +02:00
|
|
|
curitem->left = *pos - tmp; /* set LEFT arg's offset */
|
|
|
|
|
|
|
|
/* process LEFT argument */
|
|
|
|
findoprnd_recurse(ptr, pos, nnodes, needcleanup);
|
2007-09-07 17:09:56 +02:00
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
/*
|
Fix strange behavior (and possible crashes) in full text phrase search.
In an attempt to simplify the tsquery matching engine, the original
phrase search patch invented rewrite rules that would rearrange a
tsquery so that no AND/OR/NOT operator appeared below a PHRASE operator.
But this approach had numerous problems. The rearrangement step was
missed by ts_rewrite (and perhaps other places), allowing tsqueries
to be created that would cause Assert failures or perhaps crashes at
execution, as reported by Andreas Seltenreich. The rewrite rules
effectively defined semantics for operators underneath PHRASE that were
buggy, or at least unintuitive. And because rewriting was done in
tsqueryin() rather than at execution, the rearrangement was user-visible,
which is not very desirable --- for example, it might cause unexpected
matches or failures to match in ts_rewrite.
As a somewhat independent problem, the behavior of nested PHRASE operators
was only sane for left-deep trees; queries like "x <-> (y <-> z)" did not
behave intuitively at all.
To fix, get rid of the rewrite logic altogether, and instead teach the
tsquery execution engine to manage AND/OR/NOT below a PHRASE operator
by explicitly computing the match location(s) and match widths for these
operators.
This requires introducing some additional fields into the publicly visible
ExecPhraseData struct; but since there's no way for third-party code to
pass such a struct to TS_phrase_execute, it shouldn't create an ABI problem
as long as we don't move the offsets of the existing fields.
Another related problem was that index searches supposed that "!x <-> y"
could be lossily approximated as "!x & y", which isn't correct because
the latter will reject, say, "x q y" which the query itself accepts.
This required some tweaking in TS_execute_ternary along with the main
tsquery engine.
Back-patch to 9.6 where phrase operators were introduced. While this
could be argued to change behavior more than we'd like in a stable branch,
we have to do something about the crash hazards and index-vs-seqscan
inconsistency, and it doesn't seem desirable to let the unintuitive
behaviors induced by the rewriting implementation stand as precedent.
Discussion: https://postgr.es/m/28215.1481999808@sss.pgh.pa.us
Discussion: https://postgr.es/m/26706.1482087250@sss.pgh.pa.us
2016-12-21 21:18:25 +01:00
|
|
|
* Fill in the left-fields previously left unfilled.
|
|
|
|
* The input QueryItems must be in polish (prefix) notation.
|
|
|
|
* Also, set *needcleanup to true if there are any QI_VALSTOP nodes.
|
2007-09-07 18:03:40 +02:00
|
|
|
*/
|
|
|
|
static void
|
2016-04-07 17:44:18 +02:00
|
|
|
findoprnd(QueryItem *ptr, int size, bool *needcleanup)
|
2007-09-07 18:03:40 +02:00
|
|
|
{
|
|
|
|
uint32 pos;
|
|
|
|
|
2016-04-07 17:44:18 +02:00
|
|
|
*needcleanup = false;
|
2007-09-07 18:03:40 +02:00
|
|
|
pos = 0;
|
2016-04-07 17:44:18 +02:00
|
|
|
findoprnd_recurse(ptr, &pos, size, needcleanup);
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
if (pos != size)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "malformed tsquery: extra nodes");
|
2007-09-07 18:03:40 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
/*
|
2014-10-20 16:23:40 +02:00
|
|
|
* Each value (operand) in the query is passed to pushval. pushval can
|
2007-09-07 17:09:56 +02:00
|
|
|
* transform the simple value to an arbitrarily complex expression using
|
|
|
|
* pushValue and pushOperator. It must push a single value with pushValue,
|
2014-10-20 16:23:40 +02:00
|
|
|
* a complete expression with all operands, or a stopword placeholder
|
2007-09-07 17:09:56 +02:00
|
|
|
* with pushStop, otherwise the prefix notation representation will be broken,
|
|
|
|
* having an operator with no operand.
|
|
|
|
*
|
|
|
|
* opaque is passed on to pushval as is, pushval can use it to store its
|
|
|
|
* private state.
|
2007-08-21 03:11:32 +02:00
|
|
|
*/
|
|
|
|
TSQuery
|
2007-09-07 17:09:56 +02:00
|
|
|
parse_tsquery(char *buf,
|
|
|
|
PushFunction pushval,
|
2007-09-10 14:36:41 +02:00
|
|
|
Datum opaque,
|
2018-04-05 18:55:11 +02:00
|
|
|
int flags)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
struct TSQueryParserStateData state;
|
|
|
|
int i;
|
2007-08-21 03:11:32 +02:00
|
|
|
TSQuery query;
|
2007-09-07 17:09:56 +02:00
|
|
|
int commonlen;
|
2007-08-21 03:11:32 +02:00
|
|
|
QueryItem *ptr;
|
2007-09-07 17:09:56 +02:00
|
|
|
ListCell *cell;
|
2016-04-07 17:44:18 +02:00
|
|
|
bool needcleanup;
|
2018-04-05 18:55:11 +02:00
|
|
|
int tsv_flags = P_TSV_OPR_IS_DELIM | P_TSV_IS_TSQUERY;
|
|
|
|
|
|
|
|
/* plain should not be used with web */
|
|
|
|
Assert((flags & (P_TSQ_PLAIN | P_TSQ_WEB)) != (P_TSQ_PLAIN | P_TSQ_WEB));
|
|
|
|
|
|
|
|
/* select suitable tokenizer */
|
|
|
|
if (flags & P_TSQ_PLAIN)
|
|
|
|
state.gettoken = gettoken_query_plain;
|
|
|
|
else if (flags & P_TSQ_WEB)
|
|
|
|
{
|
|
|
|
state.gettoken = gettoken_query_websearch;
|
|
|
|
tsv_flags |= P_TSV_IS_WEB;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
state.gettoken = gettoken_query_standard;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
/* init state */
|
|
|
|
state.buffer = buf;
|
|
|
|
state.buf = buf;
|
|
|
|
state.count = 0;
|
2018-04-05 18:55:11 +02:00
|
|
|
state.state = WAITFIRSTOPERAND;
|
2007-09-07 17:09:56 +02:00
|
|
|
state.polstr = NIL;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
/* init value parser's state */
|
2018-04-05 18:55:11 +02:00
|
|
|
state.valstate = init_tsvector_parser(state.buffer, tsv_flags);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
/* init list of operand */
|
|
|
|
state.sumlen = 0;
|
|
|
|
state.lenop = 64;
|
|
|
|
state.curop = state.op = (char *) palloc(state.lenop);
|
|
|
|
*(state.curop) = '\0';
|
|
|
|
|
|
|
|
/* parse query & make polish notation (postfix, but in reverse order) */
|
2007-09-07 17:09:56 +02:00
|
|
|
makepol(&state, pushval, opaque);
|
|
|
|
|
|
|
|
close_tsvector_parser(state.valstate);
|
|
|
|
|
|
|
|
if (list_length(state.polstr) == 0)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
ereport(NOTICE,
|
2007-12-27 14:02:48 +01:00
|
|
|
(errmsg("text-search query doesn't contain lexemes: \"%s\"",
|
2007-08-21 03:11:32 +02:00
|
|
|
state.buffer)));
|
|
|
|
query = (TSQuery) palloc(HDRSIZETQ);
|
|
|
|
SET_VARSIZE(query, HDRSIZETQ);
|
|
|
|
query->size = 0;
|
|
|
|
return query;
|
|
|
|
}
|
|
|
|
|
2014-02-17 15:33:31 +01:00
|
|
|
if (TSQUERY_TOO_BIG(list_length(state.polstr), state.sumlen))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
|
|
|
errmsg("tsquery is too large")));
|
2007-09-07 17:09:56 +02:00
|
|
|
commonlen = COMPUTESIZE(list_length(state.polstr), state.sumlen);
|
2014-02-17 15:33:31 +01:00
|
|
|
|
|
|
|
/* Pack the QueryItems in the final TSQuery struct to return to caller */
|
2007-09-07 17:09:56 +02:00
|
|
|
query = (TSQuery) palloc0(commonlen);
|
2007-08-21 03:11:32 +02:00
|
|
|
SET_VARSIZE(query, commonlen);
|
2007-09-07 17:09:56 +02:00
|
|
|
query->size = list_length(state.polstr);
|
2007-08-21 03:11:32 +02:00
|
|
|
ptr = GETQUERY(query);
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/* Copy QueryItems to TSQuery */
|
|
|
|
i = 0;
|
|
|
|
foreach(cell, state.polstr)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2016-04-07 17:44:18 +02:00
|
|
|
QueryItem *item = (QueryItem *) lfirst(cell);
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
switch (item->type)
|
|
|
|
{
|
|
|
|
case QI_VAL:
|
|
|
|
memcpy(&ptr[i], item, sizeof(QueryOperand));
|
|
|
|
break;
|
|
|
|
case QI_VALSTOP:
|
|
|
|
ptr[i].type = QI_VALSTOP;
|
|
|
|
break;
|
|
|
|
case QI_OPR:
|
|
|
|
memcpy(&ptr[i], item, sizeof(QueryOperator));
|
|
|
|
break;
|
|
|
|
default:
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "unrecognized QueryItem type: %d", item->type);
|
2007-09-07 17:09:56 +02:00
|
|
|
}
|
|
|
|
i++;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/* Copy all the operand strings to TSQuery */
|
2007-08-21 03:11:32 +02:00
|
|
|
memcpy((void *) GETOPERAND(query), (void *) state.op, state.sumlen);
|
|
|
|
pfree(state.op);
|
|
|
|
|
Fix strange behavior (and possible crashes) in full text phrase search.
In an attempt to simplify the tsquery matching engine, the original
phrase search patch invented rewrite rules that would rearrange a
tsquery so that no AND/OR/NOT operator appeared below a PHRASE operator.
But this approach had numerous problems. The rearrangement step was
missed by ts_rewrite (and perhaps other places), allowing tsqueries
to be created that would cause Assert failures or perhaps crashes at
execution, as reported by Andreas Seltenreich. The rewrite rules
effectively defined semantics for operators underneath PHRASE that were
buggy, or at least unintuitive. And because rewriting was done in
tsqueryin() rather than at execution, the rearrangement was user-visible,
which is not very desirable --- for example, it might cause unexpected
matches or failures to match in ts_rewrite.
As a somewhat independent problem, the behavior of nested PHRASE operators
was only sane for left-deep trees; queries like "x <-> (y <-> z)" did not
behave intuitively at all.
To fix, get rid of the rewrite logic altogether, and instead teach the
tsquery execution engine to manage AND/OR/NOT below a PHRASE operator
by explicitly computing the match location(s) and match widths for these
operators.
This requires introducing some additional fields into the publicly visible
ExecPhraseData struct; but since there's no way for third-party code to
pass such a struct to TS_phrase_execute, it shouldn't create an ABI problem
as long as we don't move the offsets of the existing fields.
Another related problem was that index searches supposed that "!x <-> y"
could be lossily approximated as "!x & y", which isn't correct because
the latter will reject, say, "x q y" which the query itself accepts.
This required some tweaking in TS_execute_ternary along with the main
tsquery engine.
Back-patch to 9.6 where phrase operators were introduced. While this
could be argued to change behavior more than we'd like in a stable branch,
we have to do something about the crash hazards and index-vs-seqscan
inconsistency, and it doesn't seem desirable to let the unintuitive
behaviors induced by the rewriting implementation stand as precedent.
Discussion: https://postgr.es/m/28215.1481999808@sss.pgh.pa.us
Discussion: https://postgr.es/m/26706.1482087250@sss.pgh.pa.us
2016-12-21 21:18:25 +01:00
|
|
|
/*
|
|
|
|
* Set left operand pointers for every operator. While we're at it,
|
|
|
|
* detect whether there are any QI_VALSTOP nodes.
|
|
|
|
*/
|
2016-04-07 17:44:18 +02:00
|
|
|
findoprnd(ptr, query->size, &needcleanup);
|
|
|
|
|
|
|
|
/*
|
Fix strange behavior (and possible crashes) in full text phrase search.
In an attempt to simplify the tsquery matching engine, the original
phrase search patch invented rewrite rules that would rearrange a
tsquery so that no AND/OR/NOT operator appeared below a PHRASE operator.
But this approach had numerous problems. The rearrangement step was
missed by ts_rewrite (and perhaps other places), allowing tsqueries
to be created that would cause Assert failures or perhaps crashes at
execution, as reported by Andreas Seltenreich. The rewrite rules
effectively defined semantics for operators underneath PHRASE that were
buggy, or at least unintuitive. And because rewriting was done in
tsqueryin() rather than at execution, the rearrangement was user-visible,
which is not very desirable --- for example, it might cause unexpected
matches or failures to match in ts_rewrite.
As a somewhat independent problem, the behavior of nested PHRASE operators
was only sane for left-deep trees; queries like "x <-> (y <-> z)" did not
behave intuitively at all.
To fix, get rid of the rewrite logic altogether, and instead teach the
tsquery execution engine to manage AND/OR/NOT below a PHRASE operator
by explicitly computing the match location(s) and match widths for these
operators.
This requires introducing some additional fields into the publicly visible
ExecPhraseData struct; but since there's no way for third-party code to
pass such a struct to TS_phrase_execute, it shouldn't create an ABI problem
as long as we don't move the offsets of the existing fields.
Another related problem was that index searches supposed that "!x <-> y"
could be lossily approximated as "!x & y", which isn't correct because
the latter will reject, say, "x q y" which the query itself accepts.
This required some tweaking in TS_execute_ternary along with the main
tsquery engine.
Back-patch to 9.6 where phrase operators were introduced. While this
could be argued to change behavior more than we'd like in a stable branch,
we have to do something about the crash hazards and index-vs-seqscan
inconsistency, and it doesn't seem desirable to let the unintuitive
behaviors induced by the rewriting implementation stand as precedent.
Discussion: https://postgr.es/m/28215.1481999808@sss.pgh.pa.us
Discussion: https://postgr.es/m/26706.1482087250@sss.pgh.pa.us
2016-12-21 21:18:25 +01:00
|
|
|
* If there are QI_VALSTOP nodes, delete them and simplify the tree.
|
2016-04-07 17:44:18 +02:00
|
|
|
*/
|
|
|
|
if (needcleanup)
|
Fix strange behavior (and possible crashes) in full text phrase search.
In an attempt to simplify the tsquery matching engine, the original
phrase search patch invented rewrite rules that would rearrange a
tsquery so that no AND/OR/NOT operator appeared below a PHRASE operator.
But this approach had numerous problems. The rearrangement step was
missed by ts_rewrite (and perhaps other places), allowing tsqueries
to be created that would cause Assert failures or perhaps crashes at
execution, as reported by Andreas Seltenreich. The rewrite rules
effectively defined semantics for operators underneath PHRASE that were
buggy, or at least unintuitive. And because rewriting was done in
tsqueryin() rather than at execution, the rearrangement was user-visible,
which is not very desirable --- for example, it might cause unexpected
matches or failures to match in ts_rewrite.
As a somewhat independent problem, the behavior of nested PHRASE operators
was only sane for left-deep trees; queries like "x <-> (y <-> z)" did not
behave intuitively at all.
To fix, get rid of the rewrite logic altogether, and instead teach the
tsquery execution engine to manage AND/OR/NOT below a PHRASE operator
by explicitly computing the match location(s) and match widths for these
operators.
This requires introducing some additional fields into the publicly visible
ExecPhraseData struct; but since there's no way for third-party code to
pass such a struct to TS_phrase_execute, it shouldn't create an ABI problem
as long as we don't move the offsets of the existing fields.
Another related problem was that index searches supposed that "!x <-> y"
could be lossily approximated as "!x & y", which isn't correct because
the latter will reject, say, "x q y" which the query itself accepts.
This required some tweaking in TS_execute_ternary along with the main
tsquery engine.
Back-patch to 9.6 where phrase operators were introduced. While this
could be argued to change behavior more than we'd like in a stable branch,
we have to do something about the crash hazards and index-vs-seqscan
inconsistency, and it doesn't seem desirable to let the unintuitive
behaviors induced by the rewriting implementation stand as precedent.
Discussion: https://postgr.es/m/28215.1481999808@sss.pgh.pa.us
Discussion: https://postgr.es/m/26706.1482087250@sss.pgh.pa.us
2016-12-21 21:18:25 +01:00
|
|
|
query = cleanup_tsquery_stopwords(query);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
return query;
|
|
|
|
}
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
static void
|
2007-09-10 14:36:41 +02:00
|
|
|
pushval_asis(Datum opaque, TSQueryParserState state, char *strval, int lenval,
|
2008-05-16 18:31:02 +02:00
|
|
|
int16 weight, bool prefix)
|
2007-09-07 17:09:56 +02:00
|
|
|
{
|
2008-05-16 18:31:02 +02:00
|
|
|
pushValue(state, strval, lenval, weight, prefix);
|
2007-09-07 17:09:56 +02:00
|
|
|
}
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
/*
|
|
|
|
* in without morphology
|
|
|
|
*/
|
|
|
|
Datum
|
|
|
|
tsqueryin(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
char *in = PG_GETARG_CSTRING(0);
|
|
|
|
|
2018-04-05 18:55:11 +02:00
|
|
|
PG_RETURN_TSQUERY(parse_tsquery(in, pushval_asis, PointerGetDatum(NULL), 0));
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* out function
|
|
|
|
*/
|
|
|
|
typedef struct
|
|
|
|
{
|
|
|
|
QueryItem *curpol;
|
|
|
|
char *buf;
|
|
|
|
char *cur;
|
|
|
|
char *op;
|
2007-09-07 17:09:56 +02:00
|
|
|
int buflen;
|
2007-08-21 03:11:32 +02:00
|
|
|
} INFIX;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/* Makes sure inf->buf is large enough for adding 'addsize' bytes */
|
|
|
|
#define RESIZEBUF(inf, addsize) \
|
2007-08-21 03:11:32 +02:00
|
|
|
while( ( (inf)->cur - (inf)->buf ) + (addsize) + 1 >= (inf)->buflen ) \
|
|
|
|
{ \
|
2007-09-07 17:09:56 +02:00
|
|
|
int len = (inf)->cur - (inf)->buf; \
|
2007-08-21 03:11:32 +02:00
|
|
|
(inf)->buflen *= 2; \
|
|
|
|
(inf)->buf = (char*) repalloc( (void*)(inf)->buf, (inf)->buflen ); \
|
|
|
|
(inf)->cur = (inf)->buf + len; \
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2016-04-07 17:44:18 +02:00
|
|
|
* recursively traverse the tree and
|
|
|
|
* print it in infix (human-readable) form
|
2007-08-21 03:11:32 +02:00
|
|
|
*/
|
|
|
|
static void
|
2016-06-27 19:55:24 +02:00
|
|
|
infix(INFIX *in, int parentPriority, bool rightPhraseOp)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
/* since this function recurses, it could be driven to stack overflow. */
|
|
|
|
check_stack_depth();
|
|
|
|
|
|
|
|
if (in->curpol->type == QI_VAL)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2009-07-16 08:33:46 +02:00
|
|
|
QueryOperand *curpol = &in->curpol->qoperand;
|
2007-09-07 17:09:56 +02:00
|
|
|
char *op = in->op + curpol->distance;
|
2007-08-21 03:11:32 +02:00
|
|
|
int clen;
|
|
|
|
|
2008-05-16 18:31:02 +02:00
|
|
|
RESIZEBUF(in, curpol->length * (pg_database_encoding_max_length() + 1) + 2 + 6);
|
2007-08-21 03:11:32 +02:00
|
|
|
*(in->cur) = '\'';
|
|
|
|
in->cur++;
|
|
|
|
while (*op)
|
|
|
|
{
|
|
|
|
if (t_iseq(op, '\''))
|
|
|
|
{
|
|
|
|
*(in->cur) = '\'';
|
|
|
|
in->cur++;
|
|
|
|
}
|
2007-11-16 16:05:59 +01:00
|
|
|
else if (t_iseq(op, '\\'))
|
|
|
|
{
|
|
|
|
*(in->cur) = '\\';
|
|
|
|
in->cur++;
|
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
COPYCHAR(in->cur, op);
|
|
|
|
|
|
|
|
clen = pg_mblen(op);
|
|
|
|
op += clen;
|
|
|
|
in->cur += clen;
|
|
|
|
}
|
|
|
|
*(in->cur) = '\'';
|
|
|
|
in->cur++;
|
2008-05-16 18:31:02 +02:00
|
|
|
if (curpol->weight || curpol->prefix)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
*(in->cur) = ':';
|
|
|
|
in->cur++;
|
2008-05-16 18:31:02 +02:00
|
|
|
if (curpol->prefix)
|
|
|
|
{
|
|
|
|
*(in->cur) = '*';
|
|
|
|
in->cur++;
|
|
|
|
}
|
2007-09-07 17:09:56 +02:00
|
|
|
if (curpol->weight & (1 << 3))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
*(in->cur) = 'A';
|
|
|
|
in->cur++;
|
|
|
|
}
|
2007-09-07 17:09:56 +02:00
|
|
|
if (curpol->weight & (1 << 2))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
*(in->cur) = 'B';
|
|
|
|
in->cur++;
|
|
|
|
}
|
2007-09-07 17:09:56 +02:00
|
|
|
if (curpol->weight & (1 << 1))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
*(in->cur) = 'C';
|
|
|
|
in->cur++;
|
|
|
|
}
|
2007-09-07 17:09:56 +02:00
|
|
|
if (curpol->weight & 1)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
*(in->cur) = 'D';
|
|
|
|
in->cur++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*(in->cur) = '\0';
|
|
|
|
in->curpol++;
|
|
|
|
}
|
2009-07-16 08:33:46 +02:00
|
|
|
else if (in->curpol->qoperator.oper == OP_NOT)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2016-06-27 19:55:24 +02:00
|
|
|
int priority = QO_PRIORITY(in->curpol);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2016-04-07 17:44:18 +02:00
|
|
|
if (priority < parentPriority)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
RESIZEBUF(in, 2);
|
|
|
|
sprintf(in->cur, "( ");
|
|
|
|
in->cur = strchr(in->cur, '\0');
|
|
|
|
}
|
2016-04-07 17:44:18 +02:00
|
|
|
RESIZEBUF(in, 1);
|
|
|
|
*(in->cur) = '!';
|
|
|
|
in->cur++;
|
|
|
|
*(in->cur) = '\0';
|
|
|
|
in->curpol++;
|
2007-09-07 17:09:56 +02:00
|
|
|
|
2016-06-27 19:55:24 +02:00
|
|
|
infix(in, priority, false);
|
2016-04-07 17:44:18 +02:00
|
|
|
if (priority < parentPriority)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
RESIZEBUF(in, 2);
|
|
|
|
sprintf(in->cur, " )");
|
|
|
|
in->cur = strchr(in->cur, '\0');
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2009-07-16 08:33:46 +02:00
|
|
|
int8 op = in->curpol->qoperator.oper;
|
2016-06-27 19:55:24 +02:00
|
|
|
int priority = QO_PRIORITY(in->curpol);
|
2016-04-07 17:44:18 +02:00
|
|
|
int16 distance = in->curpol->qoperator.distance;
|
2007-08-21 03:11:32 +02:00
|
|
|
INFIX nrm;
|
2016-04-07 17:44:18 +02:00
|
|
|
bool needParenthesis = false;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
in->curpol++;
|
2016-04-07 17:44:18 +02:00
|
|
|
if (priority < parentPriority ||
|
2016-06-27 19:55:24 +02:00
|
|
|
/* phrase operator depends on order */
|
|
|
|
(op == OP_PHRASE && rightPhraseOp))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2016-04-07 17:44:18 +02:00
|
|
|
needParenthesis = true;
|
2007-08-21 03:11:32 +02:00
|
|
|
RESIZEBUF(in, 2);
|
|
|
|
sprintf(in->cur, "( ");
|
|
|
|
in->cur = strchr(in->cur, '\0');
|
|
|
|
}
|
|
|
|
|
|
|
|
nrm.curpol = in->curpol;
|
|
|
|
nrm.op = in->op;
|
|
|
|
nrm.buflen = 16;
|
|
|
|
nrm.cur = nrm.buf = (char *) palloc(sizeof(char) * nrm.buflen);
|
|
|
|
|
|
|
|
/* get right operand */
|
2016-06-27 19:55:24 +02:00
|
|
|
infix(&nrm, priority, (op == OP_PHRASE));
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
/* get & print left operand */
|
|
|
|
in->curpol = nrm.curpol;
|
2016-06-27 19:55:24 +02:00
|
|
|
infix(in, priority, false);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
/* print operator & right operand */
|
2016-04-07 17:44:18 +02:00
|
|
|
RESIZEBUF(in, 3 + (2 + 10 /* distance */ ) + (nrm.cur - nrm.buf));
|
2007-09-07 17:09:56 +02:00
|
|
|
switch (op)
|
|
|
|
{
|
|
|
|
case OP_OR:
|
|
|
|
sprintf(in->cur, " | %s", nrm.buf);
|
|
|
|
break;
|
|
|
|
case OP_AND:
|
|
|
|
sprintf(in->cur, " & %s", nrm.buf);
|
|
|
|
break;
|
2016-04-07 17:44:18 +02:00
|
|
|
case OP_PHRASE:
|
|
|
|
if (distance != 1)
|
|
|
|
sprintf(in->cur, " <%d> %s", distance, nrm.buf);
|
|
|
|
else
|
|
|
|
sprintf(in->cur, " <-> %s", nrm.buf);
|
|
|
|
break;
|
2007-09-07 17:09:56 +02:00
|
|
|
default:
|
|
|
|
/* OP_NOT is handled in above if-branch */
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "unrecognized operator type: %d", op);
|
2007-09-07 17:09:56 +02:00
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
in->cur = strchr(in->cur, '\0');
|
|
|
|
pfree(nrm.buf);
|
|
|
|
|
2016-04-07 17:44:18 +02:00
|
|
|
if (needParenthesis)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
RESIZEBUF(in, 2);
|
|
|
|
sprintf(in->cur, " )");
|
|
|
|
in->cur = strchr(in->cur, '\0');
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
tsqueryout(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSQuery query = PG_GETARG_TSQUERY(0);
|
|
|
|
INFIX nrm;
|
|
|
|
|
|
|
|
if (query->size == 0)
|
|
|
|
{
|
|
|
|
char *b = palloc(1);
|
|
|
|
|
|
|
|
*b = '\0';
|
|
|
|
PG_RETURN_POINTER(b);
|
|
|
|
}
|
|
|
|
nrm.curpol = GETQUERY(query);
|
|
|
|
nrm.buflen = 32;
|
|
|
|
nrm.cur = nrm.buf = (char *) palloc(sizeof(char) * nrm.buflen);
|
|
|
|
*(nrm.cur) = '\0';
|
|
|
|
nrm.op = GETOPERAND(query);
|
2016-06-27 19:55:24 +02:00
|
|
|
infix(&nrm, -1 /* lowest priority */ , false);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
PG_FREE_IF_COPY(query, 0);
|
|
|
|
PG_RETURN_CSTRING(nrm.buf);
|
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/*
|
|
|
|
* Binary Input / Output functions. The binary format is as follows:
|
|
|
|
*
|
|
|
|
* uint32 number of operators/operands in the query
|
|
|
|
*
|
|
|
|
* Followed by the operators and operands, in prefix notation. For each
|
|
|
|
* operand:
|
|
|
|
*
|
|
|
|
* uint8 type, QI_VAL
|
|
|
|
* uint8 weight
|
|
|
|
* operand text in client encoding, null-terminated
|
2008-05-16 18:31:02 +02:00
|
|
|
* uint8 prefix
|
2007-09-07 18:03:40 +02:00
|
|
|
*
|
|
|
|
* For each operator:
|
|
|
|
* uint8 type, QI_OPR
|
2016-04-07 17:44:18 +02:00
|
|
|
* uint8 operator, one of OP_AND, OP_PHRASE OP_OR, OP_NOT.
|
|
|
|
* uint16 distance (only for OP_PHRASE)
|
2007-09-07 18:03:40 +02:00
|
|
|
*/
|
2007-08-21 03:11:32 +02:00
|
|
|
Datum
|
|
|
|
tsquerysend(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSQuery query = PG_GETARG_TSQUERY(0);
|
|
|
|
StringInfoData buf;
|
|
|
|
int i;
|
|
|
|
QueryItem *item = GETQUERY(query);
|
|
|
|
|
|
|
|
pq_begintypsend(&buf);
|
|
|
|
|
2017-10-12 06:00:46 +02:00
|
|
|
pq_sendint32(&buf, query->size);
|
2007-08-21 03:11:32 +02:00
|
|
|
for (i = 0; i < query->size; i++)
|
|
|
|
{
|
2017-10-12 06:00:46 +02:00
|
|
|
pq_sendint8(&buf, item->type);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
switch (item->type)
|
|
|
|
{
|
|
|
|
case QI_VAL:
|
2017-10-12 06:00:46 +02:00
|
|
|
pq_sendint8(&buf, item->qoperand.weight);
|
|
|
|
pq_sendint8(&buf, item->qoperand.prefix);
|
2009-07-16 08:33:46 +02:00
|
|
|
pq_sendstring(&buf, GETOPERAND(query) + item->qoperand.distance);
|
2007-09-07 17:09:56 +02:00
|
|
|
break;
|
|
|
|
case QI_OPR:
|
2017-10-12 06:00:46 +02:00
|
|
|
pq_sendint8(&buf, item->qoperator.oper);
|
2016-04-07 17:44:18 +02:00
|
|
|
if (item->qoperator.oper == OP_PHRASE)
|
2017-10-12 06:00:46 +02:00
|
|
|
pq_sendint16(&buf, item->qoperator.distance);
|
2007-09-07 17:09:56 +02:00
|
|
|
break;
|
|
|
|
default:
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "unrecognized tsquery node type: %d", item->type);
|
2007-09-07 17:09:56 +02:00
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
item++;
|
|
|
|
}
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(query, 0);
|
|
|
|
|
|
|
|
PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
tsqueryrecv(PG_FUNCTION_ARGS)
|
|
|
|
{
|
2016-04-07 17:44:18 +02:00
|
|
|
StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
|
|
|
|
TSQuery query;
|
|
|
|
int i,
|
|
|
|
len;
|
|
|
|
QueryItem *item;
|
|
|
|
int datalen;
|
|
|
|
char *ptr;
|
|
|
|
uint32 size;
|
|
|
|
const char **operands;
|
|
|
|
bool needcleanup;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
size = pq_getmsgint(buf, sizeof(uint32));
|
2007-09-07 18:03:40 +02:00
|
|
|
if (size > (MaxAllocSize / sizeof(QueryItem)))
|
2007-08-21 03:11:32 +02:00
|
|
|
elog(ERROR, "invalid size of tsquery");
|
2007-09-07 17:09:56 +02:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/* Allocate space to temporarily hold operand strings */
|
|
|
|
operands = palloc(size * sizeof(char *));
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/* Allocate space for all the QueryItems. */
|
|
|
|
len = HDRSIZETQ + sizeof(QueryItem) * size;
|
2007-09-07 17:35:11 +02:00
|
|
|
query = (TSQuery) palloc0(len);
|
2007-08-21 03:11:32 +02:00
|
|
|
query->size = size;
|
|
|
|
item = GETQUERY(query);
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
datalen = 0;
|
2007-08-21 03:11:32 +02:00
|
|
|
for (i = 0; i < size; i++)
|
|
|
|
{
|
|
|
|
item->type = (int8) pq_getmsgint(buf, sizeof(int8));
|
2007-09-07 17:09:56 +02:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
if (item->type == QI_VAL)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 18:03:40 +02:00
|
|
|
size_t val_len; /* length after recoding to server
|
|
|
|
* encoding */
|
|
|
|
uint8 weight;
|
2008-05-16 18:31:02 +02:00
|
|
|
uint8 prefix;
|
2007-09-07 18:03:40 +02:00
|
|
|
const char *val;
|
|
|
|
pg_crc32 valcrc;
|
|
|
|
|
|
|
|
weight = (uint8) pq_getmsgint(buf, sizeof(uint8));
|
2008-05-16 18:31:02 +02:00
|
|
|
prefix = (uint8) pq_getmsgint(buf, sizeof(uint8));
|
2007-09-07 18:03:40 +02:00
|
|
|
val = pq_getmsgstring(buf);
|
|
|
|
val_len = strlen(val);
|
|
|
|
|
|
|
|
/* Sanity checks */
|
|
|
|
|
|
|
|
if (weight > 0xF)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "invalid tsquery: invalid weight bitmap");
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
if (val_len > MAXSTRLEN)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "invalid tsquery: operand too long");
|
2007-11-15 22:14:46 +01:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
if (datalen > MAXSTRPOS)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "invalid tsquery: total operand length exceeded");
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
/* Looks valid. */
|
|
|
|
|
Switch to CRC-32C in WAL and other places.
The old algorithm was found to not be the usual CRC-32 algorithm, used by
Ethernet et al. We were using a non-reflected lookup table with code meant
for a reflected lookup table. That's a strange combination that AFAICS does
not correspond to any bit-wise CRC calculation, which makes it difficult to
reason about its properties. Although it has worked well in practice, seems
safer to use a well-known algorithm.
Since we're changing the algorithm anyway, we might as well choose a
different polynomial. The Castagnoli polynomial has better error-correcting
properties than the traditional CRC-32 polynomial, even if we had
implemented it correctly. Another reason for picking that is that some new
CPUs have hardware support for calculating CRC-32C, but not CRC-32, let
alone our strange variant of it. This patch doesn't add any support for such
hardware, but a future patch could now do that.
The old algorithm is kept around for tsquery and pg_trgm, which use the
values in indexes that need to remain compatible so that pg_upgrade works.
While we're at it, share the old lookup table for CRC-32 calculation
between hstore, ltree and core. They all use the same table, so might as
well.
2014-11-04 10:35:15 +01:00
|
|
|
INIT_LEGACY_CRC32(valcrc);
|
|
|
|
COMP_LEGACY_CRC32(valcrc, val, val_len);
|
|
|
|
FIN_LEGACY_CRC32(valcrc);
|
2007-09-07 18:03:40 +02:00
|
|
|
|
2009-07-16 08:33:46 +02:00
|
|
|
item->qoperand.weight = weight;
|
|
|
|
item->qoperand.prefix = (prefix) ? true : false;
|
|
|
|
item->qoperand.valcrc = (int32) valcrc;
|
|
|
|
item->qoperand.length = val_len;
|
|
|
|
item->qoperand.distance = datalen;
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Operand strings are copied to the final struct after this loop;
|
|
|
|
* here we just collect them to an array
|
|
|
|
*/
|
|
|
|
operands[i] = val;
|
|
|
|
|
|
|
|
datalen += val_len + 1; /* + 1 for the '\0' terminator */
|
|
|
|
}
|
|
|
|
else if (item->type == QI_OPR)
|
|
|
|
{
|
|
|
|
int8 oper;
|
2007-11-15 22:14:46 +01:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
oper = (int8) pq_getmsgint(buf, sizeof(int8));
|
2016-04-07 17:44:18 +02:00
|
|
|
if (oper != OP_NOT && oper != OP_OR && oper != OP_AND && oper != OP_PHRASE)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "invalid tsquery: unrecognized operator type %d",
|
|
|
|
(int) oper);
|
2007-09-07 18:03:40 +02:00
|
|
|
if (i == size - 1)
|
|
|
|
elog(ERROR, "invalid pointer to right operand");
|
|
|
|
|
2009-07-16 08:33:46 +02:00
|
|
|
item->qoperator.oper = oper;
|
2016-04-07 17:44:18 +02:00
|
|
|
if (oper == OP_PHRASE)
|
|
|
|
item->qoperator.distance = (int16) pq_getmsgint(buf, sizeof(int16));
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
2007-09-07 18:03:40 +02:00
|
|
|
else
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "unrecognized tsquery node type: %d", item->type);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
item++;
|
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/* Enlarge buffer to make room for the operand values. */
|
2007-08-21 03:11:32 +02:00
|
|
|
query = (TSQuery) repalloc(query, len + datalen);
|
|
|
|
item = GETQUERY(query);
|
|
|
|
ptr = GETOPERAND(query);
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Fill in the left-pointers. Checks that the tree is well-formed as a
|
|
|
|
* side-effect.
|
|
|
|
*/
|
2016-04-07 17:44:18 +02:00
|
|
|
findoprnd(item, size, &needcleanup);
|
2007-09-07 18:03:40 +02:00
|
|
|
|
Fix strange behavior (and possible crashes) in full text phrase search.
In an attempt to simplify the tsquery matching engine, the original
phrase search patch invented rewrite rules that would rearrange a
tsquery so that no AND/OR/NOT operator appeared below a PHRASE operator.
But this approach had numerous problems. The rearrangement step was
missed by ts_rewrite (and perhaps other places), allowing tsqueries
to be created that would cause Assert failures or perhaps crashes at
execution, as reported by Andreas Seltenreich. The rewrite rules
effectively defined semantics for operators underneath PHRASE that were
buggy, or at least unintuitive. And because rewriting was done in
tsqueryin() rather than at execution, the rearrangement was user-visible,
which is not very desirable --- for example, it might cause unexpected
matches or failures to match in ts_rewrite.
As a somewhat independent problem, the behavior of nested PHRASE operators
was only sane for left-deep trees; queries like "x <-> (y <-> z)" did not
behave intuitively at all.
To fix, get rid of the rewrite logic altogether, and instead teach the
tsquery execution engine to manage AND/OR/NOT below a PHRASE operator
by explicitly computing the match location(s) and match widths for these
operators.
This requires introducing some additional fields into the publicly visible
ExecPhraseData struct; but since there's no way for third-party code to
pass such a struct to TS_phrase_execute, it shouldn't create an ABI problem
as long as we don't move the offsets of the existing fields.
Another related problem was that index searches supposed that "!x <-> y"
could be lossily approximated as "!x & y", which isn't correct because
the latter will reject, say, "x q y" which the query itself accepts.
This required some tweaking in TS_execute_ternary along with the main
tsquery engine.
Back-patch to 9.6 where phrase operators were introduced. While this
could be argued to change behavior more than we'd like in a stable branch,
we have to do something about the crash hazards and index-vs-seqscan
inconsistency, and it doesn't seem desirable to let the unintuitive
behaviors induced by the rewriting implementation stand as precedent.
Discussion: https://postgr.es/m/28215.1481999808@sss.pgh.pa.us
Discussion: https://postgr.es/m/26706.1482087250@sss.pgh.pa.us
2016-12-21 21:18:25 +01:00
|
|
|
/* Can't have found any QI_VALSTOP nodes */
|
|
|
|
Assert(!needcleanup);
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/* Copy operands to output struct */
|
2007-08-21 03:11:32 +02:00
|
|
|
for (i = 0; i < size; i++)
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
if (item->type == QI_VAL)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2009-07-16 08:33:46 +02:00
|
|
|
memcpy(ptr, operands[i], item->qoperand.length + 1);
|
|
|
|
ptr += item->qoperand.length + 1;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
item++;
|
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
pfree(operands);
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
Assert(ptr - GETOPERAND(query) == datalen);
|
|
|
|
|
|
|
|
SET_VARSIZE(query, len + datalen);
|
|
|
|
|
2016-04-07 17:44:18 +02:00
|
|
|
PG_RETURN_TSQUERY(query);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* debug function, used only for view query
|
|
|
|
* which will be executed in non-leaf pages in index
|
|
|
|
*/
|
|
|
|
Datum
|
|
|
|
tsquerytree(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSQuery query = PG_GETARG_TSQUERY(0);
|
|
|
|
INFIX nrm;
|
|
|
|
text *res;
|
|
|
|
QueryItem *q;
|
2007-09-07 17:09:56 +02:00
|
|
|
int len;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
if (query->size == 0)
|
|
|
|
{
|
|
|
|
res = (text *) palloc(VARHDRSZ);
|
|
|
|
SET_VARSIZE(res, VARHDRSZ);
|
|
|
|
PG_RETURN_POINTER(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
q = clean_NOT(GETQUERY(query), &len);
|
|
|
|
|
|
|
|
if (!q)
|
|
|
|
{
|
2008-03-25 23:42:46 +01:00
|
|
|
res = cstring_to_text("T");
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
nrm.curpol = q;
|
|
|
|
nrm.buflen = 32;
|
|
|
|
nrm.cur = nrm.buf = (char *) palloc(sizeof(char) * nrm.buflen);
|
|
|
|
*(nrm.cur) = '\0';
|
|
|
|
nrm.op = GETOPERAND(query);
|
2016-06-27 19:55:24 +02:00
|
|
|
infix(&nrm, -1, false);
|
2008-03-25 23:42:46 +01:00
|
|
|
res = cstring_to_text_with_len(nrm.buf, nrm.cur - nrm.buf);
|
2007-08-21 03:11:32 +02:00
|
|
|
pfree(q);
|
|
|
|
}
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(query, 0);
|
|
|
|
|
2008-03-25 23:42:46 +01:00
|
|
|
PG_RETURN_TEXT_P(res);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|