postgresql/contrib/tsearch/morph.c

/*
 * morphology module
 * New dictionary is include in dict.h. For languages which
 * use latin charset it may be need to modify mapdict table.
 * Teodor Sigaev <teodor@stack.net>
 */ 
#include "postgres.h"

#include "utils/elog.h"
#include "utils/palloc.h"
#include "utils/builtins.h"
#include "catalog/pg_control.h"
#include "utils/pg_locale.h"

#include "morph.h"
#include "deflex.h"

/*
 * Struct for calling dictionaries
 * All of this methods are optional, but
 * if all methods are NULL, then dictionary does nothing :)
 * Return value of lemmatize must be palloced or the same.
 * Return value of init must be malloced in other case 
 * it will be free in end of transaction! 
 */
typedef struct {
	char    localename[LOCALE_NAME_BUFLEN];
	/* init dictionary */
	void*	(*init)(void);
	/* close dictionary */
	void	(*close)(void*);
	/* find in dictionary */
	char*	(*lemmatize)(void*,char*,int*);
	int	(*is_stoplemm)(void*,char*,int);
	int	(*is_stemstoplemm)(void*,char*,int);
} DICT;

/* insert all dictionaries */
#define DICT_BODY
#include "dict.h"
#undef  DICT_BODY

/* fill dictionary's structure */  
#define DICT_TABLE
DICT dicts[] = {
	{
		"C",NULL,NULL,NULL,NULL,NULL	/* fake dictionary */
	}
#include "dict.h"
};
#undef DICT_TABLE

/* array for storing dictinary's objects (if needed) */
void*	dictobjs[ lengthof(dicts) ];

#define STOPLEXEM	-2
#define BYLOCALE	-1
#define	NODICT		0
#define DEFAULTDICT	1 
	
#define MAXNDICT	2
typedef int2	MAPDICT[MAXNDICT];
#define GETDICT(x,i)	*( ((int2*)(x)) + (i) )

/* map dictionaries for lexem type */
static MAPDICT mapdict[] = {
	{NODICT,	NODICT},	/* not used 		*/
	{DEFAULTDICT,	NODICT},	/* LATWORD		*/
	{BYLOCALE,	NODICT},	/* NONLATINWORD		*/
	{BYLOCALE,	DEFAULTDICT},	/* UWORD		*/
	{NODICT,	NODICT},	/* EMAIL		*/
	{NODICT,	NODICT},	/* FURL			*/
	{NODICT,	NODICT},	/* HOST			*/
	{NODICT,	NODICT},	/* FLOAT		*/
	{NODICT,	NODICT},	/* FINT			*/
	{BYLOCALE,	DEFAULTDICT},	/* PARTWORD		*/
	{BYLOCALE,	NODICT},	/* NONLATINPARTWORD	*/
	{DEFAULTDICT,	NODICT},	/* LATPARTWORD		*/
	{STOPLEXEM,	NODICT},	/* SPACE		*/
	{STOPLEXEM,	NODICT},	/* SYMTAG		*/
	{STOPLEXEM,	NODICT},	/* HTTP			*/
	{BYLOCALE,	DEFAULTDICT},	/* DEFISWORD		*/
	{DEFAULTDICT,	NODICT},	/* DEFISLATWORD		*/
	{BYLOCALE,	NODICT},	/* DEFISNONLATINWORD	*/
	{NODICT,	NODICT},	/* URI			*/
	{NODICT,	NODICT}		/* FILEPATH		*/
};

static bool inited=false;

void initmorph(void) {
	int i,j,k;
	MAPDICT	*md;
	bool 	needinit[ lengthof(dicts) ];
#ifdef USE_LOCALE
	PG_LocaleCategories     lc;

	int	bylocaledict = NODICT;
#endif

	if ( inited ) return;
	for(i=1; i<lengthof(dicts);i++)
		needinit[i] = false;
	
#ifdef USE_LOCALE
	PGLC_current(&lc);
	for(i=1;i<lengthof(dicts);i++)
		if (strcmp( dicts[i].localename, lc.lang ) == 0) {
			bylocaledict = i;
			break;
		}
	PGLC_free_categories(&lc);
#endif

	for(i=1; i<lengthof(mapdict);i++) {
		k=0;
		md = &mapdict[i];
		for(j=0;j<MAXNDICT;j++) {
			GETDICT(md,k) = GETDICT(md,j);
			if ( GETDICT(md,k) == NODICT ) {
				break;
			} else if ( GETDICT(md,k) == BYLOCALE ) {
#ifdef USE_LOCALE
				if ( bylocaledict == NODICT )
					continue;
				GETDICT(md,k) = bylocaledict;
#else
				continue;
#endif
			}
			if ( GETDICT(md,k) >= (int2)lengthof(dicts) )
				continue;
			needinit[ GETDICT(md,k) ] = true;
			k++; 
		}
		for(;k<MAXNDICT;k++)
			if ( GETDICT(md,k) != STOPLEXEM )
				GETDICT(md,k) = NODICT;
	}

	for(i=1; i<lengthof(dicts);i++)
		if ( needinit[i] && dicts[i].init )
			dictobjs[i] = (*(dicts[i].init))();
			 
	inited = true;
	return;
}

char* lemmatize( char* word, int *len, int type ) {
	int2	nd;
	int i;
	DICT	*dict;

	for(i=0;i<MAXNDICT;i++) {
		nd = GETDICT( &mapdict[type], i );
		if ( nd == NODICT ) {
			/* there is no dictionary */ 
			return word;
		} else if ( nd == STOPLEXEM ) {
			/* word is stopword */
			return NULL;
		} else {
			dict = &dicts[ nd ];
			if ( dict->is_stoplemm && (*(dict->is_stoplemm))(dictobjs[nd], word, *len) )
				return NULL;
			if ( dict->lemmatize ) {
				int oldlen = *len;
				char *newword = (*(dict->lemmatize))(dictobjs[nd], word, len);
				/* word is recognized by distionary */
				if ( newword != word || *len != oldlen ) {
					if ( dict->is_stemstoplemm && 
							(*(dict->is_stemstoplemm))(dictobjs[nd], word, *len) ) {
						if ( newword != word && newword)
							pfree(newword);
						return NULL;
					}
					return newword;
				}
			}
		}
	}

	return word;
}

bool is_stoptype(int type) {
	return ( GETDICT( &mapdict[type], 0 ) == STOPLEXEM ) ? true : false;
}