postgresql/contrib/tsearch/txtidx.c

/*
 * In/Out definitions for txtidx type
 * Internal structure:
 * string of values, array of position lexem in string and it's length
 * Teodor Sigaev <teodor@stack.net>
 */
#include "postgres.h"

#include "access/gist.h"
#include "access/itup.h"
#include "utils/elog.h"
#include "utils/palloc.h"
#include "utils/builtins.h"
#include "storage/bufpage.h"
#include "executor/spi.h"
#include "commands/trigger.h"

#include "utils/pg_locale.h"

#include <ctype.h> /* tolower */
#include "txtidx.h" 
#include "query.h" 

#include "deflex.h"
#include "parser.h"

#include "morph.h"

PG_FUNCTION_INFO_V1(txtidx_in);
Datum   txtidx_in(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(txtidx_out);
Datum   txtidx_out(PG_FUNCTION_ARGS);

PG_FUNCTION_INFO_V1(txt2txtidx);
Datum   txt2txtidx(PG_FUNCTION_ARGS);

PG_FUNCTION_INFO_V1(tsearch);
Datum tsearch(PG_FUNCTION_ARGS);

PG_FUNCTION_INFO_V1(txtidxsize);
Datum txtidxsize(PG_FUNCTION_ARGS);

/*
 * in/out text index type
 */
static char *BufferStr;
static int
compareentry( const void * a, const void * b ) {
	if ( ((WordEntry*)a)->len == ((WordEntry*)b)->len ) {
		return strncmp( 
			&BufferStr[((WordEntry*)a)->pos],
			&BufferStr[((WordEntry*)b)->pos],
			((WordEntry*)b)->len );
	}
	return ( ((WordEntry*)a)->len > ((WordEntry*)b)->len ) ? 1 : -1;
}

static int
uniqueentry( WordEntry* a, int4 l, char *buf, int4 *outbuflen ) {
	WordEntry *ptr, *res;

	res = a;
	*outbuflen = res->len;
	if ( l == 1 )
		return l;

	ptr = a+1;
	BufferStr = buf;
	qsort((void*)a, l, sizeof(int4), compareentry );
	*outbuflen = res->len;

	while (ptr - a < l) {
		if ( ! (ptr->len == res->len && 
				strncmp(&buf[ ptr->pos ], &buf[ res->pos ],res->len) == 0 ) ) {
			res++;
			res->len = ptr->len;
			res->pos = ptr->pos;
			*outbuflen += res->len;

		}
		ptr++;
	}
	return res + 1 - a;
} 

#define WAITWORD	1
#define WAITENDWORD	2
#define WAITNEXTCHAR	3
#define WAITENDCMPLX	4

#define RESIZEPRSBUF if ( state->curpos - state->word == state->len ) { \
        int4 clen = state->curpos - state->word; \
        state->len *= 2; \
        state->word = (char*)repalloc( (void*)state->word, state->len ); \
        state->curpos = state->word + clen; \
}

int4
gettoken_txtidx( TI_IN_STATE *state ) {
	int4 oldstate = 0;
	state->curpos = state->word;
	state->state = WAITWORD;

	while( 1 ) {
		if ( state->state == WAITWORD ) {
			if ( *(state->prsbuf) == '\0' ) {
				return 0;
			} else if ( *(state->prsbuf) == '\'' ) {
				state->state = WAITENDCMPLX; 
			} else if ( *(state->prsbuf) == '\\' ) {
				state->state = WAITNEXTCHAR;
				oldstate = WAITENDWORD;
			} else if ( state->oprisdelim && ISOPERATOR( *(state->prsbuf) ) ) {
				elog(ERROR, "Syntax error");
			} else if ( *(state->prsbuf) != ' ' ) {
				*(state->curpos) = *(state->prsbuf);
				state->curpos++;
				state->state = WAITENDWORD;
			}
		} else if ( state->state == WAITNEXTCHAR ) {
			if ( *(state->prsbuf) == '\0' ) {
				elog(ERROR,"There is no escaped character");
			} else {
				RESIZEPRSBUF;
				*(state->curpos) = *(state->prsbuf);
				state->curpos++;
				state->state = oldstate;
			}
		} else if ( state->state == WAITENDWORD ) {
			if ( *(state->prsbuf) == '\\' ) {
				state->state = WAITNEXTCHAR;
				oldstate = WAITENDWORD;
			} else if ( *(state->prsbuf) == ' ' || *(state->prsbuf) == '\0' || 
					( state->oprisdelim && ISOPERATOR( *(state->prsbuf) ) ) ) {
				RESIZEPRSBUF;
				if ( state->curpos == state->word ) 
					elog(ERROR, "Syntax error");
				*(state->curpos) = '\0';
				return 1;
			} else {
				RESIZEPRSBUF;
				*(state->curpos) = *(state->prsbuf);
				state->curpos++;
			}
		} else if ( state->state == WAITENDCMPLX ) {
			if ( *(state->prsbuf) == '\'' ) {
				RESIZEPRSBUF;
				*(state->curpos) = '\0';
				if ( state->curpos == state->word ) 
					elog(ERROR, "Syntax error");
				state->prsbuf++;
				return 1;
			} else if ( *(state->prsbuf) == '\\' ) {
				state->state = WAITNEXTCHAR;
				oldstate = WAITENDCMPLX;
			} else if ( *(state->prsbuf) == '\0' ) {
				elog(ERROR,"Syntax error");
			} else {
				RESIZEPRSBUF;
				*(state->curpos) = *(state->prsbuf);
				state->curpos++;
			}
		} else {
			elog(ERROR, "Inner bug :(");
		} 
		state->prsbuf++;
	}

	return 0;
}

Datum
txtidx_in(PG_FUNCTION_ARGS) {
	char *buf = (char*)PG_GETARG_POINTER(0);
	TI_IN_STATE state;
	WordEntry *arr;
	int4 len=0, totallen = 64;
	txtidx *in;
	char *tmpbuf, *cur;
	int4 i,buflen = 256;

	state.prsbuf = buf;
	state.len=32;
	state.word = (char*)palloc( state.len );
	state.oprisdelim = false;

	arr = (WordEntry*)palloc( sizeof(WordEntry) * totallen );
	cur = tmpbuf = (char*)palloc( buflen ); 
	while( gettoken_txtidx( &state ) ) {
		if ( len == totallen ) {
			totallen *= 2;
			arr = (WordEntry*)repalloc( (void*)arr, sizeof(int4)*totallen );
		}
		while ( cur-tmpbuf + state.curpos - state.word >= buflen ) {
			int4 dist = cur-tmpbuf;
			buflen *= 2;
			tmpbuf = (char*)repalloc( (void*)tmpbuf, buflen );
			cur = tmpbuf+dist;
		}
		if ( state.curpos - state.word > 0xffff )
			elog(ERROR,"Word is too long");
		arr[len].len = state.curpos - state.word;
		if ( cur - tmpbuf > 0xffff )
			elog(ERROR,"Too long value");
		arr[len].pos = cur - tmpbuf;
		memcpy( (void*)cur, (void*)state.word, arr[len].len );
		cur += arr[len].len;
		len++;
	}
	pfree(state.word);

	if ( !len ) 
		elog(ERROR,"Void value");

	len = uniqueentry( arr, len, tmpbuf, &buflen );
	totallen = CALCDATASIZE( len, buflen );
	in = (txtidx*)palloc( totallen );
	in->len = totallen;
	in->size = len;
	cur = STRPTR(in);
	for(i=0;i<len;i++) {
		memcpy( (void*)cur, (void*)&tmpbuf[ arr[i].pos ], arr[i].len );
		arr[i].pos = cur - STRPTR(in);
		cur += arr[i].len;
	}
	pfree(tmpbuf);
	memcpy( (void*)ARRPTR(in), (void*)arr, sizeof(int4)*len );
	pfree( arr );
	PG_RETURN_POINTER( in );
}

Datum
txtidxsize(PG_FUNCTION_ARGS) {
	txtidx	*in=(txtidx*)DatumGetPointer(PG_DETOAST_DATUM(PG_GETARG_DATUM(0)));
	int4	ret = in->size;
	PG_FREE_IF_COPY(in,0);
	PG_RETURN_INT32( ret );
}

Datum
txtidx_out(PG_FUNCTION_ARGS) {
	txtidx	*out=(txtidx*)DatumGetPointer(PG_DETOAST_DATUM(PG_GETARG_DATUM(0)));
	char *outbuf;
	int4 i,j,lenbuf = STRSIZE(out) + 1 /* \0 */ + out->size*2 /* '' */ + out->size - 1 /* space */;
	WordEntry	*ptr = ARRPTR(out);
	char *curin, *curout;

	curout = outbuf = (char*) palloc( lenbuf );
	for(i=0;i<out->size;i++) {
		curin = STRPTR(out) + ptr->pos;
		if ( i!= 0 )
			*curout++ = ' ';
		*curout++ = '\'';
		j = ptr->len;
		while( j-- ) {
			if ( *curin == '\'' ) {
				int4 pos = curout - outbuf;
				outbuf = (char*)repalloc((void*)outbuf, ++lenbuf );
				curout = outbuf + pos;
				*curout++ = '\\';
			}
			*curout++ = *curin++;
		}
		*curout++ = '\'';
		ptr++;
	}
	outbuf[ lenbuf-1 ] = '\0';
	PG_FREE_IF_COPY(out,0);
	PG_RETURN_POINTER( outbuf );
}

typedef struct {
	uint16	len;
	char*	word;
} WORD;

typedef struct {
	WORD	*words;
	int4 lenwords;
	int4 curwords;
} PRSTEXT;

/*
 * Parse text to lexems
 */
static void
parsetext( PRSTEXT *prs, char *buf, int4 buflen ) {
	int type,lenlemm;
	char *ptr,*ptrw;
	char *lemm;

	start_parse_str( buf, buflen );
	while( (type=tsearch_yylex()) != 0 ) {
		if ( prs->curwords == prs->lenwords ) {
			prs->lenwords *= 2;
			prs->words = (WORD*)repalloc( (void*)prs->words, prs->lenwords * sizeof(WORD) );
		}
		if ( tokenlen>0xffff ) {
			end_parse();
			elog(ERROR, "Word is too long");
		}

		lenlemm = tokenlen;
		lemm = lemmatize( token, &lenlemm, type );
	
		if ( ! lemm )
			continue;

		if ( lemm != token ) {
			prs->words[ prs->curwords ].len = lenlemm;
			prs->words[ prs->curwords ].word = lemm;
		} else {
			prs->words[ prs->curwords ].len = lenlemm;
			ptrw = prs->words[ prs->curwords ].word = (char*)palloc( lenlemm );
			ptr = token;
			while( ptr-token < lenlemm ) {
				*ptrw = tolower( (unsigned char) *ptr );
				ptr++; ptrw++;
			}
		}
		prs->curwords++; 
	}
	end_parse();
}

static int
compareWORD( const void * a, const void * b ) {
	if ( ((WORD*)a)->len == ((WORD*)b)->len )
		return strncmp( 
			((WORD*)a)->word,
			((WORD*)b)->word,
			((WORD*)b)->len );
	return ( ((WORD*)a)->len > ((WORD*)b)->len ) ? 1 : -1;
}

static int
uniqueWORD( WORD* a, int4 l ) {
	WORD *ptr, *res;

	if ( l == 1 )
		return l;

	res = a;
	ptr = a + 1;

	qsort((void*)a, l, sizeof(WORD), compareWORD );

	while (ptr - a < l) {
		if ( ! (ptr->len == res->len && 
				strncmp(ptr->word, res->word ,res->len) == 0 ) ) {
			res++;
			res->len  = ptr->len;
			res->word = ptr->word;
		} else {
			pfree(ptr->word);
		}
		ptr++;
	}

	return res + 1 - a;
} 

/*
 * make value of txtidx
 */
static txtidx *
makevalue( PRSTEXT *prs ) {
	int4	i, lenstr=0, totallen;
	txtidx *in;
	WordEntry *ptr;
	char *str,*cur;

	prs->curwords = uniqueWORD( prs->words, prs->curwords );
	for(i=0;i<prs->curwords;i++)
		lenstr += prs->words[i].len;

	totallen = CALCDATASIZE( prs->curwords, lenstr );
	in = (txtidx*)palloc( totallen );
	in->len = totallen;
	in->size = prs->curwords;

	ptr = ARRPTR(in);
	cur = str = STRPTR(in);
	for(i=0;i<prs->curwords;i++) {
		ptr->len = prs->words[i].len;
		if ( cur-str > 0xffff )
			elog(ERROR,"Value is too big");
		ptr->pos = cur-str;
		ptr++;
		memcpy( (void*)cur, (void*)prs->words[i].word, prs->words[i].len );
		pfree(prs->words[i].word);
		cur += prs->words[i].len;	
	}
	pfree(prs->words);
	return in;
}

Datum
txt2txtidx(PG_FUNCTION_ARGS) {
	text	*in = (text*)DatumGetPointer(PG_DETOAST_DATUM(PG_GETARG_DATUM(0)));
	PRSTEXT	prs;
	txtidx *out = NULL;
	
	prs.lenwords = 32;	
	prs.curwords = 0;	
	prs.words = (WORD*)palloc(sizeof(WORD)*prs.lenwords);

	initmorph();
	parsetext( &prs, VARDATA(in), VARSIZE(in) - VARHDRSZ );
	PG_FREE_IF_COPY(in,0);
	
	if ( prs.curwords ) {
		out = makevalue( &prs );
		PG_RETURN_POINTER( out );
	}
	pfree(prs.words);
	PG_RETURN_NULL();
} 

/*
 * Trigger
 */
Datum
tsearch(PG_FUNCTION_ARGS) {
	TriggerData *trigdata;
	Trigger    *trigger;
	Relation        rel;
	HeapTuple       rettuple = NULL;
	int numidxattr,i;
	PRSTEXT	prs;
	Datum datum = (Datum)0;


	if (!CALLED_AS_TRIGGER(fcinfo))
		elog(ERROR, "TSearch: Not fired by trigger manager");

	trigdata = (TriggerData *) fcinfo->context;
	if (TRIGGER_FIRED_FOR_STATEMENT(trigdata->tg_event))
		elog(ERROR, "TSearch: Can't process STATEMENT events");
	if (TRIGGER_FIRED_AFTER(trigdata->tg_event))
		elog(ERROR, "TSearch: Must be fired BEFORE event");

	if (TRIGGER_FIRED_BY_INSERT(trigdata->tg_event))
		rettuple = trigdata->tg_trigtuple;
	else if (TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event))
		rettuple = trigdata->tg_newtuple;
	else 
		elog(ERROR, "TSearch: Unknown event");

	trigger = trigdata->tg_trigger;
	rel = trigdata->tg_relation;

	if ( trigger->tgnargs < 2 )
		elog(ERROR,"TSearch: format tsearch(txtidx_field, text_field1,...)");

	numidxattr = SPI_fnumber(rel->rd_att, trigger->tgargs[0]);
	if ( numidxattr < 0 )
		elog(ERROR,"TSearch: Can not find txtidx_field");

	prs.lenwords = 32;	
	prs.curwords = 0;	
	prs.words = (WORD*)palloc(sizeof(WORD)*prs.lenwords);

	initmorph();
	/* find all words in indexable column */ 
	for(i=1; i<trigger->tgnargs; i++) {
		int4 numattr;
		text *txt_toasted, *txt;
		bool isnull;
		Oid oidtype;

		numattr = SPI_fnumber(rel->rd_att, trigger->tgargs[i]);
		oidtype = SPI_gettypeid(rel->rd_att, numattr);
		if ( numattr<0 || ( ! ( oidtype==TEXTOID || oidtype==VARCHAROID ) ) ) {
			elog(NOTICE, "TSearch: can not find field '%s'", trigger->tgargs[i]);
			continue;
		}
		txt_toasted = (text*)DatumGetPointer( SPI_getbinval(rettuple, rel->rd_att, numattr, &isnull ) );
		if ( isnull )
			continue;
		txt = (text*)DatumGetPointer( PG_DETOAST_DATUM( PointerGetDatum ( txt_toasted ) ) );

		parsetext( &prs, VARDATA(txt), VARSIZE(txt) - VARHDRSZ );
		if ( txt != txt_toasted )
			pfree(txt);
	}

	/* make txtidx value */
	if (prs.curwords) {
		datum = PointerGetDatum( makevalue( &prs ) );
		rettuple = SPI_modifytuple( rel, rettuple, 1, &numidxattr,
			&datum, NULL );
		pfree(DatumGetPointer(datum));
	} else {
		char nulls = 'n'; 
		pfree( prs.words );
		rettuple = SPI_modifytuple( rel, rettuple, 1, &numidxattr, 
			&datum, &nulls );
	}
 
	if (rettuple == NULL)
		elog(ERROR, "TSearch: %d returned by SPI_modifytuple", SPI_result);

	return PointerGetDatum( rettuple );
}