2007-08-21 03:11:32 +02:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* tsvector.c
|
|
|
|
* I/O functions for tsvector
|
|
|
|
*
|
2016-01-02 19:33:40 +01:00
|
|
|
* Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
|
2007-08-21 03:11:32 +02:00
|
|
|
*
|
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
2010-09-20 22:08:53 +02:00
|
|
|
* src/backend/utils/adt/tsvector.c
|
2007-08-21 03:11:32 +02:00
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include "libpq/pqformat.h"
|
|
|
|
#include "tsearch/ts_locale.h"
|
|
|
|
#include "tsearch/ts_utils.h"
|
|
|
|
#include "utils/memutils.h"
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
typedef struct
|
|
|
|
{
|
2007-10-23 02:51:23 +02:00
|
|
|
WordEntry entry; /* must be first! */
|
2007-09-07 17:09:56 +02:00
|
|
|
WordEntryPos *pos;
|
|
|
|
int poslen; /* number of elements in pos */
|
2007-11-15 23:25:18 +01:00
|
|
|
} WordEntryIN;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
|
|
|
|
/* Compare two WordEntryPos values for qsort */
|
2016-04-07 17:44:18 +02:00
|
|
|
int
|
2016-04-08 11:02:45 +02:00
|
|
|
compareWordEntryPos(const void *a, const void *b)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-11-15 22:14:46 +01:00
|
|
|
int apos = WEP_GETPOS(*(const WordEntryPos *) a);
|
|
|
|
int bpos = WEP_GETPOS(*(const WordEntryPos *) b);
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
if (apos == bpos)
|
2007-08-21 03:11:32 +02:00
|
|
|
return 0;
|
2007-09-07 17:09:56 +02:00
|
|
|
return (apos > bpos) ? 1 : -1;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
/*
|
|
|
|
* Removes duplicate pos entries. If there's two entries with same pos
|
|
|
|
* but different weight, the higher weight is retained.
|
|
|
|
*
|
|
|
|
* Returns new length.
|
|
|
|
*/
|
2007-08-21 03:11:32 +02:00
|
|
|
static int
|
2007-11-15 23:25:18 +01:00
|
|
|
uniquePos(WordEntryPos *a, int l)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
WordEntryPos *ptr,
|
|
|
|
*res;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
if (l <= 1)
|
2007-08-21 03:11:32 +02:00
|
|
|
return l;
|
|
|
|
|
2016-04-08 11:02:45 +02:00
|
|
|
qsort((void *) a, l, sizeof(WordEntryPos), compareWordEntryPos);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
res = a;
|
2007-08-21 03:11:32 +02:00
|
|
|
ptr = a + 1;
|
|
|
|
while (ptr - a < l)
|
|
|
|
{
|
|
|
|
if (WEP_GETPOS(*ptr) != WEP_GETPOS(*res))
|
|
|
|
{
|
|
|
|
res++;
|
|
|
|
*res = *ptr;
|
2007-10-23 02:51:23 +02:00
|
|
|
if (res - a >= MAXNUMPOS - 1 ||
|
|
|
|
WEP_GETPOS(*res) == MAXENTRYPOS - 1)
|
2007-08-21 03:11:32 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
else if (WEP_GETWEIGHT(*ptr) > WEP_GETWEIGHT(*res))
|
|
|
|
WEP_SETWEIGHT(*res, WEP_GETWEIGHT(*ptr));
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return res + 1 - a;
|
|
|
|
}
|
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
/* Compare two WordEntryIN values for qsort */
|
2007-08-21 03:11:32 +02:00
|
|
|
static int
|
2007-09-07 18:03:40 +02:00
|
|
|
compareentry(const void *va, const void *vb, void *arg)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-10-23 02:51:23 +02:00
|
|
|
const WordEntryIN *a = (const WordEntryIN *) va;
|
|
|
|
const WordEntryIN *b = (const WordEntryIN *) vb;
|
2007-08-21 03:11:32 +02:00
|
|
|
char *BufferStr = (char *) arg;
|
|
|
|
|
2009-06-11 16:49:15 +02:00
|
|
|
return tsCompareString(&BufferStr[a->entry.pos], a->entry.len,
|
|
|
|
&BufferStr[b->entry.pos], b->entry.len,
|
|
|
|
false);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
/*
|
|
|
|
* Sort an array of WordEntryIN, remove duplicates.
|
|
|
|
* *outbuflen receives the amount of space needed for strings and positions.
|
|
|
|
*/
|
2007-08-21 03:11:32 +02:00
|
|
|
static int
|
2007-11-15 23:25:18 +01:00
|
|
|
uniqueentry(WordEntryIN *a, int l, char *buf, int *outbuflen)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-11-15 22:14:46 +01:00
|
|
|
int buflen;
|
2007-08-21 03:11:32 +02:00
|
|
|
WordEntryIN *ptr,
|
|
|
|
*res;
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
Assert(l >= 1);
|
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
if (l > 1)
|
|
|
|
qsort_arg((void *) a, l, sizeof(WordEntryIN), compareentry,
|
|
|
|
(void *) buf);
|
2007-09-07 18:03:40 +02:00
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
buflen = 0;
|
2007-09-07 17:09:56 +02:00
|
|
|
res = a;
|
2007-08-21 03:11:32 +02:00
|
|
|
ptr = a + 1;
|
|
|
|
while (ptr - a < l)
|
|
|
|
{
|
|
|
|
if (!(ptr->entry.len == res->entry.len &&
|
2007-10-23 02:51:23 +02:00
|
|
|
strncmp(&buf[ptr->entry.pos], &buf[res->entry.pos],
|
|
|
|
res->entry.len) == 0))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-10-23 02:51:23 +02:00
|
|
|
/* done accumulating data into *res, count space needed */
|
|
|
|
buflen += res->entry.len;
|
2007-08-21 03:11:32 +02:00
|
|
|
if (res->entry.haspos)
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
res->poslen = uniquePos(res->pos, res->poslen);
|
2007-10-23 02:51:23 +02:00
|
|
|
buflen = SHORTALIGN(buflen);
|
|
|
|
buflen += res->poslen * sizeof(WordEntryPos) + sizeof(uint16);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
res++;
|
2014-03-07 12:13:33 +01:00
|
|
|
if (res != ptr)
|
|
|
|
memcpy(res, ptr, sizeof(WordEntryIN));
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
else if (ptr->entry.haspos)
|
|
|
|
{
|
|
|
|
if (res->entry.haspos)
|
|
|
|
{
|
2007-10-23 02:51:23 +02:00
|
|
|
/* append ptr's positions to res's positions */
|
2007-11-15 22:14:46 +01:00
|
|
|
int newlen = ptr->poslen + res->poslen;
|
2007-09-07 17:09:56 +02:00
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
res->pos = (WordEntryPos *)
|
|
|
|
repalloc(res->pos, newlen * sizeof(WordEntryPos));
|
|
|
|
memcpy(&res->pos[res->poslen], ptr->pos,
|
|
|
|
ptr->poslen * sizeof(WordEntryPos));
|
2007-09-07 17:09:56 +02:00
|
|
|
res->poslen = newlen;
|
2007-08-21 03:11:32 +02:00
|
|
|
pfree(ptr->pos);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2007-10-23 02:51:23 +02:00
|
|
|
/* just give ptr's positions to pos */
|
2007-08-21 03:11:32 +02:00
|
|
|
res->entry.haspos = 1;
|
|
|
|
res->pos = ptr->pos;
|
2007-10-23 02:51:23 +02:00
|
|
|
res->poslen = ptr->poslen;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
ptr++;
|
|
|
|
}
|
2007-09-07 18:03:40 +02:00
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
/* count space needed for last item */
|
|
|
|
buflen += res->entry.len;
|
2007-08-21 03:11:32 +02:00
|
|
|
if (res->entry.haspos)
|
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
res->poslen = uniquePos(res->pos, res->poslen);
|
2007-10-23 02:51:23 +02:00
|
|
|
buflen = SHORTALIGN(buflen);
|
|
|
|
buflen += res->poslen * sizeof(WordEntryPos) + sizeof(uint16);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
*outbuflen = buflen;
|
2007-08-21 03:11:32 +02:00
|
|
|
return res + 1 - a;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2007-11-15 23:25:18 +01:00
|
|
|
WordEntryCMP(WordEntry *a, WordEntry *b, char *buf)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
return compareentry(a, b, buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Datum
|
|
|
|
tsvectorin(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
char *buf = PG_GETARG_CSTRING(0);
|
|
|
|
TSVectorParseState state;
|
|
|
|
WordEntryIN *arr;
|
2007-09-07 17:09:56 +02:00
|
|
|
int totallen;
|
2007-11-15 22:14:46 +01:00
|
|
|
int arrlen; /* allocated size of arr */
|
2007-08-21 03:11:32 +02:00
|
|
|
WordEntry *inarr;
|
2007-09-07 17:09:56 +02:00
|
|
|
int len = 0;
|
2007-08-21 03:11:32 +02:00
|
|
|
TSVector in;
|
2007-09-07 17:09:56 +02:00
|
|
|
int i;
|
|
|
|
char *token;
|
|
|
|
int toklen;
|
|
|
|
WordEntryPos *pos;
|
|
|
|
int poslen;
|
2007-10-23 02:51:23 +02:00
|
|
|
char *strbuf;
|
|
|
|
int stroff;
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
/*
|
2007-11-15 22:14:46 +01:00
|
|
|
* Tokens are appended to tmpbuf, cur is a pointer to the end of used
|
|
|
|
* space in tmpbuf.
|
2007-09-07 17:09:56 +02:00
|
|
|
*/
|
|
|
|
char *tmpbuf;
|
|
|
|
char *cur;
|
2007-11-15 22:14:46 +01:00
|
|
|
int buflen = 256; /* allocated size of tmpbuf */
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-10-22 00:29:56 +02:00
|
|
|
state = init_tsvector_parser(buf, false, false);
|
2007-11-15 22:14:46 +01:00
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
arrlen = 64;
|
|
|
|
arr = (WordEntryIN *) palloc(sizeof(WordEntryIN) * arrlen);
|
2007-08-21 03:11:32 +02:00
|
|
|
cur = tmpbuf = (char *) palloc(buflen);
|
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
while (gettoken_tsvector(state, &token, &toklen, &pos, &poslen, NULL))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 17:09:56 +02:00
|
|
|
if (toklen >= MAXSTRLEN)
|
2007-08-21 03:11:32 +02:00
|
|
|
ereport(ERROR,
|
2007-10-23 02:51:23 +02:00
|
|
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
2007-08-21 03:45:33 +02:00
|
|
|
errmsg("word is too long (%ld bytes, max %ld bytes)",
|
2007-09-07 17:09:56 +02:00
|
|
|
(long) toklen,
|
2007-11-15 22:14:46 +01:00
|
|
|
(long) (MAXSTRLEN - 1))));
|
2007-09-07 17:09:56 +02:00
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
if (cur - tmpbuf > MAXSTRPOS)
|
|
|
|
ereport(ERROR,
|
2007-10-23 02:51:23 +02:00
|
|
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
2009-06-11 16:49:15 +02:00
|
|
|
errmsg("string is too long for tsvector (%ld bytes, max %ld bytes)",
|
2008-03-10 13:57:05 +01:00
|
|
|
(long) (cur - tmpbuf), (long) MAXSTRPOS)));
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Enlarge buffers if needed
|
|
|
|
*/
|
|
|
|
if (len >= arrlen)
|
|
|
|
{
|
|
|
|
arrlen *= 2;
|
2007-10-23 02:51:23 +02:00
|
|
|
arr = (WordEntryIN *)
|
|
|
|
repalloc((void *) arr, sizeof(WordEntryIN) * arrlen);
|
2007-09-07 17:09:56 +02:00
|
|
|
}
|
|
|
|
while ((cur - tmpbuf) + toklen >= buflen)
|
|
|
|
{
|
2007-11-15 22:14:46 +01:00
|
|
|
int dist = cur - tmpbuf;
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
buflen *= 2;
|
|
|
|
tmpbuf = (char *) repalloc((void *) tmpbuf, buflen);
|
|
|
|
cur = tmpbuf + dist;
|
|
|
|
}
|
|
|
|
arr[len].entry.len = toklen;
|
2007-08-21 03:11:32 +02:00
|
|
|
arr[len].entry.pos = cur - tmpbuf;
|
2007-09-07 17:09:56 +02:00
|
|
|
memcpy((void *) cur, (void *) token, toklen);
|
|
|
|
cur += toklen;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 17:09:56 +02:00
|
|
|
if (poslen != 0)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
arr[len].entry.haspos = 1;
|
2007-09-07 17:09:56 +02:00
|
|
|
arr[len].pos = pos;
|
|
|
|
arr[len].poslen = poslen;
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
else
|
2007-10-23 02:51:23 +02:00
|
|
|
{
|
2007-08-21 03:11:32 +02:00
|
|
|
arr[len].entry.haspos = 0;
|
2007-10-23 02:51:23 +02:00
|
|
|
arr[len].pos = NULL;
|
|
|
|
arr[len].poslen = 0;
|
|
|
|
}
|
2007-08-21 03:11:32 +02:00
|
|
|
len++;
|
|
|
|
}
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
close_tsvector_parser(state);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
if (len > 0)
|
|
|
|
len = uniqueentry(arr, len, tmpbuf, &buflen);
|
|
|
|
else
|
|
|
|
buflen = 0;
|
2007-10-23 02:51:23 +02:00
|
|
|
|
|
|
|
if (buflen > MAXSTRPOS)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
2008-03-05 16:50:37 +01:00
|
|
|
errmsg("string is too long for tsvector (%d bytes, max %d bytes)", buflen, MAXSTRPOS)));
|
2007-10-23 02:51:23 +02:00
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
totallen = CALCDATASIZE(len, buflen);
|
|
|
|
in = (TSVector) palloc0(totallen);
|
|
|
|
SET_VARSIZE(in, totallen);
|
|
|
|
in->size = len;
|
|
|
|
inarr = ARRPTR(in);
|
2007-10-23 02:51:23 +02:00
|
|
|
strbuf = STRPTR(in);
|
|
|
|
stroff = 0;
|
2007-08-21 03:11:32 +02:00
|
|
|
for (i = 0; i < len; i++)
|
|
|
|
{
|
2007-10-23 02:51:23 +02:00
|
|
|
memcpy(strbuf + stroff, &tmpbuf[arr[i].entry.pos], arr[i].entry.len);
|
|
|
|
arr[i].entry.pos = stroff;
|
|
|
|
stroff += arr[i].entry.len;
|
2007-08-21 03:11:32 +02:00
|
|
|
if (arr[i].entry.haspos)
|
|
|
|
{
|
2007-10-23 02:51:23 +02:00
|
|
|
if (arr[i].poslen > 0xFFFF)
|
2007-09-07 17:09:56 +02:00
|
|
|
elog(ERROR, "positions array too long");
|
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
/* Copy number of positions */
|
|
|
|
stroff = SHORTALIGN(stroff);
|
|
|
|
*(uint16 *) (strbuf + stroff) = (uint16) arr[i].poslen;
|
|
|
|
stroff += sizeof(uint16);
|
2007-09-07 17:09:56 +02:00
|
|
|
|
|
|
|
/* Copy positions */
|
2007-10-23 02:51:23 +02:00
|
|
|
memcpy(strbuf + stroff, arr[i].pos, arr[i].poslen * sizeof(WordEntryPos));
|
|
|
|
stroff += arr[i].poslen * sizeof(WordEntryPos);
|
2007-09-07 17:09:56 +02:00
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
pfree(arr[i].pos);
|
|
|
|
}
|
|
|
|
inarr[i] = arr[i].entry;
|
|
|
|
}
|
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
Assert((strbuf + stroff - (char *) in) == totallen);
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
PG_RETURN_TSVECTOR(in);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
tsvectorout(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSVector out = PG_GETARG_TSVECTOR(0);
|
|
|
|
char *outbuf;
|
2012-06-25 00:51:46 +02:00
|
|
|
int32 i,
|
2007-08-21 03:11:32 +02:00
|
|
|
lenbuf = 0,
|
|
|
|
pp;
|
|
|
|
WordEntry *ptr = ARRPTR(out);
|
|
|
|
char *curbegin,
|
|
|
|
*curin,
|
|
|
|
*curout;
|
|
|
|
|
|
|
|
lenbuf = out->size * 2 /* '' */ + out->size - 1 /* space */ + 2 /* \0 */ ;
|
|
|
|
for (i = 0; i < out->size; i++)
|
|
|
|
{
|
|
|
|
lenbuf += ptr[i].len * 2 * pg_database_encoding_max_length() /* for escape */ ;
|
|
|
|
if (ptr[i].haspos)
|
|
|
|
lenbuf += 1 /* : */ + 7 /* int2 + , + weight */ * POSDATALEN(out, &(ptr[i]));
|
|
|
|
}
|
|
|
|
|
|
|
|
curout = outbuf = (char *) palloc(lenbuf);
|
|
|
|
for (i = 0; i < out->size; i++)
|
|
|
|
{
|
|
|
|
curbegin = curin = STRPTR(out) + ptr->pos;
|
|
|
|
if (i != 0)
|
|
|
|
*curout++ = ' ';
|
|
|
|
*curout++ = '\'';
|
|
|
|
while (curin - curbegin < ptr->len)
|
|
|
|
{
|
|
|
|
int len = pg_mblen(curin);
|
|
|
|
|
|
|
|
if (t_iseq(curin, '\''))
|
|
|
|
*curout++ = '\'';
|
2007-11-16 16:05:59 +01:00
|
|
|
else if (t_iseq(curin, '\\'))
|
|
|
|
*curout++ = '\\';
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
while (len--)
|
|
|
|
*curout++ = *curin++;
|
|
|
|
}
|
|
|
|
|
|
|
|
*curout++ = '\'';
|
|
|
|
if ((pp = POSDATALEN(out, ptr)) != 0)
|
|
|
|
{
|
|
|
|
WordEntryPos *wptr;
|
|
|
|
|
|
|
|
*curout++ = ':';
|
|
|
|
wptr = POSDATAPTR(out, ptr);
|
|
|
|
while (pp)
|
|
|
|
{
|
|
|
|
curout += sprintf(curout, "%d", WEP_GETPOS(*wptr));
|
|
|
|
switch (WEP_GETWEIGHT(*wptr))
|
|
|
|
{
|
|
|
|
case 3:
|
|
|
|
*curout++ = 'A';
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
*curout++ = 'B';
|
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
*curout++ = 'C';
|
|
|
|
break;
|
|
|
|
case 0:
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pp > 1)
|
|
|
|
*curout++ = ',';
|
|
|
|
pp--;
|
|
|
|
wptr++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
*curout = '\0';
|
|
|
|
PG_FREE_IF_COPY(out, 0);
|
|
|
|
PG_RETURN_CSTRING(outbuf);
|
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/*
|
|
|
|
* Binary Input / Output functions. The binary format is as follows:
|
|
|
|
*
|
|
|
|
* uint32 number of lexemes
|
2007-11-15 22:14:46 +01:00
|
|
|
*
|
2007-09-07 18:03:40 +02:00
|
|
|
* for each lexeme:
|
|
|
|
* lexeme text in client encoding, null-terminated
|
2007-11-15 22:14:46 +01:00
|
|
|
* uint16 number of positions
|
|
|
|
* for each position:
|
2007-09-07 18:03:40 +02:00
|
|
|
* uint16 WordEntryPos
|
|
|
|
*/
|
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
Datum
|
|
|
|
tsvectorsend(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
TSVector vec = PG_GETARG_TSVECTOR(0);
|
|
|
|
StringInfoData buf;
|
|
|
|
int i,
|
|
|
|
j;
|
|
|
|
WordEntry *weptr = ARRPTR(vec);
|
|
|
|
|
|
|
|
pq_begintypsend(&buf);
|
|
|
|
|
|
|
|
pq_sendint(&buf, vec->size, sizeof(int32));
|
|
|
|
for (i = 0; i < vec->size; i++)
|
|
|
|
{
|
2007-11-15 22:14:46 +01:00
|
|
|
uint16 npos;
|
2007-09-07 18:03:40 +02:00
|
|
|
|
2007-11-15 22:14:46 +01:00
|
|
|
/*
|
|
|
|
* the strings in the TSVector array are not null-terminated, so we
|
|
|
|
* have to send the null-terminator separately
|
2007-08-21 03:11:32 +02:00
|
|
|
*/
|
2007-09-07 18:03:40 +02:00
|
|
|
pq_sendtext(&buf, STRPTR(vec) + weptr->pos, weptr->len);
|
|
|
|
pq_sendbyte(&buf, '\0');
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
npos = POSDATALEN(vec, weptr);
|
|
|
|
pq_sendint(&buf, npos, sizeof(uint16));
|
|
|
|
|
2007-11-15 22:14:46 +01:00
|
|
|
if (npos > 0)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
WordEntryPos *wepptr = POSDATAPTR(vec, weptr);
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
for (j = 0; j < npos; j++)
|
2007-08-21 03:11:32 +02:00
|
|
|
pq_sendint(&buf, wepptr[j], sizeof(WordEntryPos));
|
|
|
|
}
|
|
|
|
weptr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
tsvectorrecv(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
|
|
|
|
TSVector vec;
|
2007-09-07 17:09:56 +02:00
|
|
|
int i;
|
2007-09-07 18:03:40 +02:00
|
|
|
int32 nentries;
|
2007-11-15 22:14:46 +01:00
|
|
|
int datalen; /* number of bytes used in the variable size
|
|
|
|
* area after fixed size TSVector header and
|
|
|
|
* WordEntries */
|
2007-09-07 18:03:40 +02:00
|
|
|
Size hdrlen;
|
2007-11-15 22:14:46 +01:00
|
|
|
Size len; /* allocated size of vec */
|
2009-05-21 22:09:36 +02:00
|
|
|
bool needSort = false;
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
nentries = pq_getmsgint(buf, sizeof(int32));
|
|
|
|
if (nentries < 0 || nentries > (MaxAllocSize / sizeof(WordEntry)))
|
2007-08-21 03:11:32 +02:00
|
|
|
elog(ERROR, "invalid size of tsvector");
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
hdrlen = DATAHDRSIZE + sizeof(WordEntry) * nentries;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-11-15 22:14:46 +01:00
|
|
|
len = hdrlen * 2; /* times two to make room for lexemes */
|
2007-08-21 03:11:32 +02:00
|
|
|
vec = (TSVector) palloc0(len);
|
2007-09-07 18:03:40 +02:00
|
|
|
vec->size = nentries;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
datalen = 0;
|
|
|
|
for (i = 0; i < nentries; i++)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 18:03:40 +02:00
|
|
|
const char *lexeme;
|
2007-11-15 22:14:46 +01:00
|
|
|
uint16 npos;
|
|
|
|
size_t lex_len;
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
lexeme = pq_getmsgstring(buf);
|
|
|
|
npos = (uint16) pq_getmsgint(buf, sizeof(uint16));
|
|
|
|
|
|
|
|
/* sanity checks */
|
|
|
|
|
|
|
|
lex_len = strlen(lexeme);
|
2009-05-21 14:54:27 +02:00
|
|
|
if (lex_len > MAXSTRLEN)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "invalid tsvector: lexeme too long");
|
2007-09-07 18:03:40 +02:00
|
|
|
|
|
|
|
if (datalen > MAXSTRPOS)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "invalid tsvector: maximum total lexeme length exceeded");
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
if (npos > MAXNUMPOS)
|
2007-11-28 22:56:30 +01:00
|
|
|
elog(ERROR, "unexpected number of tsvector positions");
|
2007-08-21 03:11:32 +02:00
|
|
|
|
|
|
|
/*
|
2007-09-07 18:03:40 +02:00
|
|
|
* Looks valid. Fill the WordEntry struct, and copy lexeme.
|
|
|
|
*
|
|
|
|
* But make sure the buffer is large enough first.
|
2007-08-21 03:11:32 +02:00
|
|
|
*/
|
2007-09-07 18:03:40 +02:00
|
|
|
while (hdrlen + SHORTALIGN(datalen + lex_len) +
|
|
|
|
(npos + 1) * sizeof(WordEntryPos) >= len)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
|
|
|
len *= 2;
|
|
|
|
vec = (TSVector) repalloc(vec, len);
|
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
vec->entries[i].haspos = (npos > 0) ? 1 : 0;
|
|
|
|
vec->entries[i].len = lex_len;
|
|
|
|
vec->entries[i].pos = datalen;
|
|
|
|
|
|
|
|
memcpy(STRPTR(vec) + datalen, lexeme, lex_len);
|
|
|
|
|
|
|
|
datalen += lex_len;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-10-23 02:51:23 +02:00
|
|
|
if (i > 0 && WordEntryCMP(&vec->entries[i],
|
|
|
|
&vec->entries[i - 1],
|
|
|
|
STRPTR(vec)) <= 0)
|
2009-05-21 22:09:36 +02:00
|
|
|
needSort = true;
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/* Receive positions */
|
|
|
|
if (npos > 0)
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 18:03:40 +02:00
|
|
|
uint16 j;
|
2007-08-21 03:11:32 +02:00
|
|
|
WordEntryPos *wepptr;
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
/*
|
|
|
|
* Pad to 2-byte alignment if necessary. Though we used palloc0
|
2007-11-15 22:14:46 +01:00
|
|
|
* for the initial allocation, subsequent repalloc'd memory areas
|
|
|
|
* are not initialized to zero.
|
2007-09-07 18:03:40 +02:00
|
|
|
*/
|
|
|
|
if (datalen != SHORTALIGN(datalen))
|
2007-08-21 03:11:32 +02:00
|
|
|
{
|
2007-09-07 18:03:40 +02:00
|
|
|
*(STRPTR(vec) + datalen) = '\0';
|
|
|
|
datalen = SHORTALIGN(datalen);
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
memcpy(STRPTR(vec) + datalen, &npos, sizeof(uint16));
|
|
|
|
|
|
|
|
wepptr = POSDATAPTR(vec, &vec->entries[i]);
|
2007-08-21 03:11:32 +02:00
|
|
|
for (j = 0; j < npos; j++)
|
|
|
|
{
|
2007-09-07 18:03:40 +02:00
|
|
|
wepptr[j] = (WordEntryPos) pq_getmsgint(buf, sizeof(WordEntryPos));
|
2007-08-21 03:11:32 +02:00
|
|
|
if (j > 0 && WEP_GETPOS(wepptr[j]) <= WEP_GETPOS(wepptr[j - 1]))
|
2007-10-22 00:29:56 +02:00
|
|
|
elog(ERROR, "position information is misordered");
|
2007-08-21 03:11:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
datalen += (npos + 1) * sizeof(WordEntry);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-09-07 18:03:40 +02:00
|
|
|
SET_VARSIZE(vec, hdrlen + datalen);
|
2007-08-21 03:11:32 +02:00
|
|
|
|
2009-05-21 22:09:36 +02:00
|
|
|
if (needSort)
|
2009-06-11 16:49:15 +02:00
|
|
|
qsort_arg((void *) ARRPTR(vec), vec->size, sizeof(WordEntry),
|
|
|
|
compareentry, (void *) STRPTR(vec));
|
2009-05-21 22:09:36 +02:00
|
|
|
|
2007-08-21 03:11:32 +02:00
|
|
|
PG_RETURN_TSVECTOR(vec);
|
|
|
|
}
|