2008-05-17 03:28:26 +02:00
|
|
|
/*
|
2010-09-20 22:08:53 +02:00
|
|
|
* contrib/pg_trgm/trgm_op.c
|
2008-05-17 03:28:26 +02:00
|
|
|
*/
|
2010-12-04 06:16:21 +01:00
|
|
|
#include "postgres.h"
|
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
#include <ctype.h>
|
2010-12-04 06:16:21 +01:00
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
#include "catalog/pg_type.h"
|
2024-02-16 21:05:36 +01:00
|
|
|
#include "common/int.h"
|
2019-11-07 04:51:04 +01:00
|
|
|
#include "lib/qunique.h"
|
2022-09-05 11:10:57 +02:00
|
|
|
#include "miscadmin.h"
|
2019-10-23 05:56:22 +02:00
|
|
|
#include "trgm.h"
|
2008-11-12 14:43:54 +01:00
|
|
|
#include "tsearch/ts_locale.h"
|
Split up guc.c for better build speed and ease of maintenance.
guc.c has grown to be one of our largest .c files, making it
a bottleneck for compilation. It's also acquired a bunch of
knowledge that'd be better kept elsewhere, because of our not
very good habit of putting variable-specific check hooks here.
Hence, split it up along these lines:
* guc.c itself retains just the core GUC housekeeping mechanisms.
* New file guc_funcs.c contains the SET/SHOW interfaces and some
SQL-accessible functions for GUC manipulation.
* New file guc_tables.c contains the data arrays that define the
built-in GUC variables, along with some already-exported constant
tables.
* GUC check/assign/show hook functions are moved to the variable's
home module, whenever that's clearly identifiable. A few hard-
to-classify hooks ended up in commands/variable.c, which was
already a home for miscellaneous GUC hook functions.
To avoid cluttering a lot more header files with #include "guc.h",
I also invented a new header file utils/guc_hooks.h and put all
the GUC hook functions' declarations there, regardless of their
originating module. That allowed removal of #include "guc.h"
from some existing headers. The fallout from that (hopefully
all caught here) demonstrates clearly why such inclusions are
best minimized: there are a lot of files that, for example,
were getting array.h at two or more levels of remove, despite
not having any connection at all to GUCs in themselves.
There is some very minor code beautification here, such as
renaming a couple of inconsistently-named hook functions
and improving some comments. But mostly this just moves
code from point A to point B and deals with the ensuing
needs for #include adjustments and exporting a few functions
that previously weren't exported.
Patch by me, per a suggestion from Andres Freund; thanks also
to Michael Paquier for the idea to invent guc_funcs.c.
Discussion: https://postgr.es/m/587607.1662836699@sss.pgh.pa.us
2022-09-13 17:05:07 +02:00
|
|
|
#include "utils/guc.h"
|
2016-03-18 10:26:27 +01:00
|
|
|
#include "utils/lsyscache.h"
|
2014-01-13 19:07:10 +01:00
|
|
|
#include "utils/memutils.h"
|
2015-04-14 16:03:42 +02:00
|
|
|
#include "utils/pg_crc.h"
|
2010-12-04 06:16:21 +01:00
|
|
|
|
2006-05-31 00:12:16 +02:00
|
|
|
PG_MODULE_MAGIC;
|
|
|
|
|
2016-03-16 15:44:58 +01:00
|
|
|
/* GUC variables */
|
2016-06-10 00:02:36 +02:00
|
|
|
double similarity_threshold = 0.3f;
|
|
|
|
double word_similarity_threshold = 0.6f;
|
2018-03-21 12:57:42 +01:00
|
|
|
double strict_word_similarity_threshold = 0.5f;
|
2016-03-16 15:44:58 +01:00
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
PG_FUNCTION_INFO_V1(set_limit);
|
2011-02-01 03:33:55 +01:00
|
|
|
PG_FUNCTION_INFO_V1(show_limit);
|
|
|
|
PG_FUNCTION_INFO_V1(show_trgm);
|
|
|
|
PG_FUNCTION_INFO_V1(similarity);
|
2016-03-16 16:59:21 +01:00
|
|
|
PG_FUNCTION_INFO_V1(word_similarity);
|
2018-03-21 12:57:42 +01:00
|
|
|
PG_FUNCTION_INFO_V1(strict_word_similarity);
|
2011-02-01 03:33:55 +01:00
|
|
|
PG_FUNCTION_INFO_V1(similarity_dist);
|
|
|
|
PG_FUNCTION_INFO_V1(similarity_op);
|
2016-03-16 16:59:21 +01:00
|
|
|
PG_FUNCTION_INFO_V1(word_similarity_op);
|
|
|
|
PG_FUNCTION_INFO_V1(word_similarity_commutator_op);
|
|
|
|
PG_FUNCTION_INFO_V1(word_similarity_dist_op);
|
|
|
|
PG_FUNCTION_INFO_V1(word_similarity_dist_commutator_op);
|
2018-03-21 12:57:42 +01:00
|
|
|
PG_FUNCTION_INFO_V1(strict_word_similarity_op);
|
|
|
|
PG_FUNCTION_INFO_V1(strict_word_similarity_commutator_op);
|
|
|
|
PG_FUNCTION_INFO_V1(strict_word_similarity_dist_op);
|
|
|
|
PG_FUNCTION_INFO_V1(strict_word_similarity_dist_commutator_op);
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
/* Trigram with position */
|
|
|
|
typedef struct
|
|
|
|
{
|
2016-06-10 00:02:36 +02:00
|
|
|
trgm trg;
|
|
|
|
int index;
|
2016-03-16 16:59:21 +01:00
|
|
|
} pos_trgm;
|
2011-02-01 03:33:55 +01:00
|
|
|
|
2018-03-21 12:57:42 +01:00
|
|
|
/* Trigram bound type */
|
|
|
|
typedef uint8 TrgmBound;
|
2018-04-26 20:47:16 +02:00
|
|
|
#define TRGM_BOUND_LEFT 0x01 /* trigram is left bound of word */
|
|
|
|
#define TRGM_BOUND_RIGHT 0x02 /* trigram is right bound of word */
|
2018-03-21 12:57:42 +01:00
|
|
|
|
|
|
|
/* Word similarity flags */
|
2018-04-26 20:47:16 +02:00
|
|
|
#define WORD_SIMILARITY_CHECK_ONLY 0x01 /* only check existence of similar
|
|
|
|
* search pattern in text */
|
|
|
|
#define WORD_SIMILARITY_STRICT 0x02 /* force bounds of extent to match
|
|
|
|
* word bounds */
|
2018-03-21 12:57:42 +01:00
|
|
|
|
2016-03-16 15:44:58 +01:00
|
|
|
/*
|
|
|
|
* Module load callback
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
_PG_init(void)
|
|
|
|
{
|
|
|
|
/* Define custom GUC variables. */
|
|
|
|
DefineCustomRealVariable("pg_trgm.similarity_threshold",
|
2019-06-10 19:14:19 +02:00
|
|
|
"Sets the threshold used by the % operator.",
|
2016-06-10 00:02:36 +02:00
|
|
|
"Valid range is 0.0 .. 1.0.",
|
|
|
|
&similarity_threshold,
|
2022-10-31 04:44:48 +01:00
|
|
|
0.3f,
|
2016-06-10 00:02:36 +02:00
|
|
|
0.0,
|
|
|
|
1.0,
|
|
|
|
PGC_USERSET,
|
|
|
|
0,
|
|
|
|
NULL,
|
|
|
|
NULL,
|
|
|
|
NULL);
|
2016-03-16 16:59:21 +01:00
|
|
|
DefineCustomRealVariable("pg_trgm.word_similarity_threshold",
|
2019-06-10 19:14:19 +02:00
|
|
|
"Sets the threshold used by the <% operator.",
|
2016-06-10 00:02:36 +02:00
|
|
|
"Valid range is 0.0 .. 1.0.",
|
|
|
|
&word_similarity_threshold,
|
2022-10-31 04:44:48 +01:00
|
|
|
0.6f,
|
2016-06-10 00:02:36 +02:00
|
|
|
0.0,
|
|
|
|
1.0,
|
|
|
|
PGC_USERSET,
|
|
|
|
0,
|
|
|
|
NULL,
|
|
|
|
NULL,
|
|
|
|
NULL);
|
2018-03-21 12:57:42 +01:00
|
|
|
DefineCustomRealVariable("pg_trgm.strict_word_similarity_threshold",
|
2019-06-10 19:14:19 +02:00
|
|
|
"Sets the threshold used by the <<% operator.",
|
2018-03-21 12:57:42 +01:00
|
|
|
"Valid range is 0.0 .. 1.0.",
|
|
|
|
&strict_word_similarity_threshold,
|
2022-10-31 04:44:48 +01:00
|
|
|
0.5f,
|
2018-03-21 12:57:42 +01:00
|
|
|
0.0,
|
|
|
|
1.0,
|
|
|
|
PGC_USERSET,
|
|
|
|
0,
|
|
|
|
NULL,
|
|
|
|
NULL,
|
|
|
|
NULL);
|
2021-12-21 18:12:24 +01:00
|
|
|
|
2022-02-21 20:10:15 +01:00
|
|
|
MarkGUCPrefixReserved("pg_trgm");
|
2016-03-16 15:44:58 +01:00
|
|
|
}
|
2011-02-01 03:33:55 +01:00
|
|
|
|
2016-03-16 15:44:58 +01:00
|
|
|
/*
|
|
|
|
* Deprecated function.
|
2016-03-16 16:59:21 +01:00
|
|
|
* Use "pg_trgm.similarity_threshold" GUC variable instead of this function.
|
2016-03-16 15:44:58 +01:00
|
|
|
*/
|
2004-05-31 19:18:12 +02:00
|
|
|
Datum
|
2004-08-29 07:07:03 +02:00
|
|
|
set_limit(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
float4 nlimit = PG_GETARG_FLOAT4(0);
|
2016-03-18 10:26:27 +01:00
|
|
|
char *nlimit_str;
|
|
|
|
Oid func_out_oid;
|
|
|
|
bool is_varlena;
|
|
|
|
|
|
|
|
getTypeOutputInfo(FLOAT4OID, &func_out_oid, &is_varlena);
|
|
|
|
|
|
|
|
nlimit_str = OidOutputFunctionCall(func_out_oid, Float4GetDatum(nlimit));
|
|
|
|
|
|
|
|
SetConfigOption("pg_trgm.similarity_threshold", nlimit_str,
|
|
|
|
PGC_USERSET, PGC_S_SESSION);
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2016-03-16 15:44:58 +01:00
|
|
|
PG_RETURN_FLOAT4(similarity_threshold);
|
2004-05-31 19:18:12 +02:00
|
|
|
}
|
|
|
|
|
2018-03-21 12:57:42 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get similarity threshold for given index scan strategy number.
|
|
|
|
*/
|
|
|
|
double
|
|
|
|
index_strategy_get_limit(StrategyNumber strategy)
|
|
|
|
{
|
|
|
|
switch (strategy)
|
|
|
|
{
|
|
|
|
case SimilarityStrategyNumber:
|
|
|
|
return similarity_threshold;
|
|
|
|
case WordSimilarityStrategyNumber:
|
|
|
|
return word_similarity_threshold;
|
|
|
|
case StrictWordSimilarityStrategyNumber:
|
|
|
|
return strict_word_similarity_threshold;
|
|
|
|
default:
|
|
|
|
elog(ERROR, "unrecognized strategy number: %d", strategy);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2018-04-26 20:47:16 +02:00
|
|
|
return 0.0; /* keep compiler quiet */
|
2018-03-21 12:57:42 +01:00
|
|
|
}
|
|
|
|
|
2016-03-16 15:44:58 +01:00
|
|
|
/*
|
|
|
|
* Deprecated function.
|
2016-03-16 16:59:21 +01:00
|
|
|
* Use "pg_trgm.similarity_threshold" GUC variable instead of this function.
|
2016-03-16 15:44:58 +01:00
|
|
|
*/
|
2004-05-31 19:18:12 +02:00
|
|
|
Datum
|
2004-08-29 07:07:03 +02:00
|
|
|
show_limit(PG_FUNCTION_ARGS)
|
|
|
|
{
|
2016-03-16 15:44:58 +01:00
|
|
|
PG_RETURN_FLOAT4(similarity_threshold);
|
2004-05-31 19:18:12 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2004-08-29 07:07:03 +02:00
|
|
|
comp_trgm(const void *a, const void *b)
|
|
|
|
{
|
|
|
|
return CMPTRGM(a, b);
|
2004-05-31 19:18:12 +02:00
|
|
|
}
|
|
|
|
|
2008-11-12 14:43:54 +01:00
|
|
|
/*
|
|
|
|
* Finds first word in string, returns pointer to the word,
|
|
|
|
* endword points to the character after word
|
|
|
|
*/
|
2009-06-11 16:49:15 +02:00
|
|
|
static char *
|
|
|
|
find_word(char *str, int lenstr, char **endword, int *charlen)
|
2008-11-12 14:43:54 +01:00
|
|
|
{
|
2009-06-11 16:49:15 +02:00
|
|
|
char *beginword = str;
|
2008-11-12 14:43:54 +01:00
|
|
|
|
2013-04-09 07:05:55 +02:00
|
|
|
while (beginword - str < lenstr && !ISWORDCHR(beginword))
|
2008-11-12 14:43:54 +01:00
|
|
|
beginword += pg_mblen(beginword);
|
|
|
|
|
|
|
|
if (beginword - str >= lenstr)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
*endword = beginword;
|
|
|
|
*charlen = 0;
|
2013-04-09 07:05:55 +02:00
|
|
|
while (*endword - str < lenstr && ISWORDCHR(*endword))
|
2008-11-12 14:43:54 +01:00
|
|
|
{
|
|
|
|
*endword += pg_mblen(*endword);
|
|
|
|
(*charlen)++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return beginword;
|
|
|
|
}
|
|
|
|
|
Get rid of USE_WIDE_UPPER_LOWER dependency in trigram construction.
contrib/pg_trgm's make_trigrams() was coded to ignore multibyte character
boundaries and just make trigrams from bytes if USE_WIDE_UPPER_LOWER wasn't
defined. This is a bit odd, since there's no obvious reason why trigram
compaction rules should depend on the presence of towlower() and friends.
What's more, there was an Assert() that would fail if that code path was
fed any multibyte characters.
We need to do something about this since the pending regex-indexing patch
has an assumption that you get just one "trgm" from any three characters.
The best solution seems to be to remove the USE_WIDE_UPPER_LOWER
dependency, which shouldn't really have been there in the first place.
The second loop in make_trigrams() is now just a fast path and not a
potentially incompatible algorithm.
If there is anybody still using Postgres on machines without wcstombs() or
towlower(), and they have non-ASCII data indexed by pg_trgm, they'll need
to REINDEX those indexes after pg_upgrade to 9.3, else searches may fail
incorrectly. It seems likely that there are no such installations, though.
In passing, rename cnt_trigram to compact_trigram, which seems to better
describe its functionality, and improve make_trigrams' test for whether it
has to use the slow path or not (per a suggestion from Alexander Korotkov).
2013-04-07 20:45:33 +02:00
|
|
|
/*
|
|
|
|
* Reduce a trigram (three possibly multi-byte characters) to a trgm,
|
|
|
|
* which is always exactly three bytes. If we have three single-byte
|
|
|
|
* characters, we just use them as-is; otherwise we form a hash value.
|
|
|
|
*/
|
2013-04-09 07:05:55 +02:00
|
|
|
void
|
Get rid of USE_WIDE_UPPER_LOWER dependency in trigram construction.
contrib/pg_trgm's make_trigrams() was coded to ignore multibyte character
boundaries and just make trigrams from bytes if USE_WIDE_UPPER_LOWER wasn't
defined. This is a bit odd, since there's no obvious reason why trigram
compaction rules should depend on the presence of towlower() and friends.
What's more, there was an Assert() that would fail if that code path was
fed any multibyte characters.
We need to do something about this since the pending regex-indexing patch
has an assumption that you get just one "trgm" from any three characters.
The best solution seems to be to remove the USE_WIDE_UPPER_LOWER
dependency, which shouldn't really have been there in the first place.
The second loop in make_trigrams() is now just a fast path and not a
potentially incompatible algorithm.
If there is anybody still using Postgres on machines without wcstombs() or
towlower(), and they have non-ASCII data indexed by pg_trgm, they'll need
to REINDEX those indexes after pg_upgrade to 9.3, else searches may fail
incorrectly. It seems likely that there are no such installations, though.
In passing, rename cnt_trigram to compact_trigram, which seems to better
describe its functionality, and improve make_trigrams' test for whether it
has to use the slow path or not (per a suggestion from Alexander Korotkov).
2013-04-07 20:45:33 +02:00
|
|
|
compact_trigram(trgm *tptr, char *str, int bytelen)
|
2008-11-12 14:43:54 +01:00
|
|
|
{
|
2009-06-11 16:49:15 +02:00
|
|
|
if (bytelen == 3)
|
2008-11-12 14:43:54 +01:00
|
|
|
{
|
2009-06-11 16:49:15 +02:00
|
|
|
CPTRGM(tptr, str);
|
2008-11-12 14:43:54 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
pg_crc32 crc;
|
|
|
|
|
Switch to CRC-32C in WAL and other places.
The old algorithm was found to not be the usual CRC-32 algorithm, used by
Ethernet et al. We were using a non-reflected lookup table with code meant
for a reflected lookup table. That's a strange combination that AFAICS does
not correspond to any bit-wise CRC calculation, which makes it difficult to
reason about its properties. Although it has worked well in practice, seems
safer to use a well-known algorithm.
Since we're changing the algorithm anyway, we might as well choose a
different polynomial. The Castagnoli polynomial has better error-correcting
properties than the traditional CRC-32 polynomial, even if we had
implemented it correctly. Another reason for picking that is that some new
CPUs have hardware support for calculating CRC-32C, but not CRC-32, let
alone our strange variant of it. This patch doesn't add any support for such
hardware, but a future patch could now do that.
The old algorithm is kept around for tsquery and pg_trgm, which use the
values in indexes that need to remain compatible so that pg_upgrade works.
While we're at it, share the old lookup table for CRC-32 calculation
between hstore, ltree and core. They all use the same table, so might as
well.
2014-11-04 10:35:15 +01:00
|
|
|
INIT_LEGACY_CRC32(crc);
|
|
|
|
COMP_LEGACY_CRC32(crc, str, bytelen);
|
|
|
|
FIN_LEGACY_CRC32(crc);
|
2008-11-12 14:43:54 +01:00
|
|
|
|
|
|
|
/*
|
2009-06-11 16:49:15 +02:00
|
|
|
* use only 3 upper bytes from crc, hope, it's good enough hashing
|
2008-11-12 14:43:54 +01:00
|
|
|
*/
|
|
|
|
CPTRGM(tptr, &crc);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2011-02-01 03:33:55 +01:00
|
|
|
* Adds trigrams from words (already padded).
|
2008-11-12 14:43:54 +01:00
|
|
|
*/
|
2009-06-11 16:49:15 +02:00
|
|
|
static trgm *
|
|
|
|
make_trigrams(trgm *tptr, char *str, int bytelen, int charlen)
|
2008-11-12 14:43:54 +01:00
|
|
|
{
|
2009-06-11 16:49:15 +02:00
|
|
|
char *ptr = str;
|
2008-11-12 14:43:54 +01:00
|
|
|
|
2009-06-11 16:49:15 +02:00
|
|
|
if (charlen < 3)
|
2008-11-12 14:43:54 +01:00
|
|
|
return tptr;
|
|
|
|
|
Get rid of USE_WIDE_UPPER_LOWER dependency in trigram construction.
contrib/pg_trgm's make_trigrams() was coded to ignore multibyte character
boundaries and just make trigrams from bytes if USE_WIDE_UPPER_LOWER wasn't
defined. This is a bit odd, since there's no obvious reason why trigram
compaction rules should depend on the presence of towlower() and friends.
What's more, there was an Assert() that would fail if that code path was
fed any multibyte characters.
We need to do something about this since the pending regex-indexing patch
has an assumption that you get just one "trgm" from any three characters.
The best solution seems to be to remove the USE_WIDE_UPPER_LOWER
dependency, which shouldn't really have been there in the first place.
The second loop in make_trigrams() is now just a fast path and not a
potentially incompatible algorithm.
If there is anybody still using Postgres on machines without wcstombs() or
towlower(), and they have non-ASCII data indexed by pg_trgm, they'll need
to REINDEX those indexes after pg_upgrade to 9.3, else searches may fail
incorrectly. It seems likely that there are no such installations, though.
In passing, rename cnt_trigram to compact_trigram, which seems to better
describe its functionality, and improve make_trigrams' test for whether it
has to use the slow path or not (per a suggestion from Alexander Korotkov).
2013-04-07 20:45:33 +02:00
|
|
|
if (bytelen > charlen)
|
2008-11-12 14:43:54 +01:00
|
|
|
{
|
Get rid of USE_WIDE_UPPER_LOWER dependency in trigram construction.
contrib/pg_trgm's make_trigrams() was coded to ignore multibyte character
boundaries and just make trigrams from bytes if USE_WIDE_UPPER_LOWER wasn't
defined. This is a bit odd, since there's no obvious reason why trigram
compaction rules should depend on the presence of towlower() and friends.
What's more, there was an Assert() that would fail if that code path was
fed any multibyte characters.
We need to do something about this since the pending regex-indexing patch
has an assumption that you get just one "trgm" from any three characters.
The best solution seems to be to remove the USE_WIDE_UPPER_LOWER
dependency, which shouldn't really have been there in the first place.
The second loop in make_trigrams() is now just a fast path and not a
potentially incompatible algorithm.
If there is anybody still using Postgres on machines without wcstombs() or
towlower(), and they have non-ASCII data indexed by pg_trgm, they'll need
to REINDEX those indexes after pg_upgrade to 9.3, else searches may fail
incorrectly. It seems likely that there are no such installations, though.
In passing, rename cnt_trigram to compact_trigram, which seems to better
describe its functionality, and improve make_trigrams' test for whether it
has to use the slow path or not (per a suggestion from Alexander Korotkov).
2013-04-07 20:45:33 +02:00
|
|
|
/* Find multibyte character boundaries and apply compact_trigram */
|
2009-06-11 16:49:15 +02:00
|
|
|
int lenfirst = pg_mblen(str),
|
|
|
|
lenmiddle = pg_mblen(str + lenfirst),
|
|
|
|
lenlast = pg_mblen(str + lenfirst + lenmiddle);
|
2008-11-12 14:43:54 +01:00
|
|
|
|
2009-06-11 16:49:15 +02:00
|
|
|
while ((ptr - str) + lenfirst + lenmiddle + lenlast <= bytelen)
|
2008-11-12 14:43:54 +01:00
|
|
|
{
|
Get rid of USE_WIDE_UPPER_LOWER dependency in trigram construction.
contrib/pg_trgm's make_trigrams() was coded to ignore multibyte character
boundaries and just make trigrams from bytes if USE_WIDE_UPPER_LOWER wasn't
defined. This is a bit odd, since there's no obvious reason why trigram
compaction rules should depend on the presence of towlower() and friends.
What's more, there was an Assert() that would fail if that code path was
fed any multibyte characters.
We need to do something about this since the pending regex-indexing patch
has an assumption that you get just one "trgm" from any three characters.
The best solution seems to be to remove the USE_WIDE_UPPER_LOWER
dependency, which shouldn't really have been there in the first place.
The second loop in make_trigrams() is now just a fast path and not a
potentially incompatible algorithm.
If there is anybody still using Postgres on machines without wcstombs() or
towlower(), and they have non-ASCII data indexed by pg_trgm, they'll need
to REINDEX those indexes after pg_upgrade to 9.3, else searches may fail
incorrectly. It seems likely that there are no such installations, though.
In passing, rename cnt_trigram to compact_trigram, which seems to better
describe its functionality, and improve make_trigrams' test for whether it
has to use the slow path or not (per a suggestion from Alexander Korotkov).
2013-04-07 20:45:33 +02:00
|
|
|
compact_trigram(tptr, ptr, lenfirst + lenmiddle + lenlast);
|
2008-11-12 14:43:54 +01:00
|
|
|
|
|
|
|
ptr += lenfirst;
|
|
|
|
tptr++;
|
|
|
|
|
2009-06-11 16:49:15 +02:00
|
|
|
lenfirst = lenmiddle;
|
|
|
|
lenmiddle = lenlast;
|
|
|
|
lenlast = pg_mblen(ptr + lenfirst + lenmiddle);
|
2008-11-12 14:43:54 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
Get rid of USE_WIDE_UPPER_LOWER dependency in trigram construction.
contrib/pg_trgm's make_trigrams() was coded to ignore multibyte character
boundaries and just make trigrams from bytes if USE_WIDE_UPPER_LOWER wasn't
defined. This is a bit odd, since there's no obvious reason why trigram
compaction rules should depend on the presence of towlower() and friends.
What's more, there was an Assert() that would fail if that code path was
fed any multibyte characters.
We need to do something about this since the pending regex-indexing patch
has an assumption that you get just one "trgm" from any three characters.
The best solution seems to be to remove the USE_WIDE_UPPER_LOWER
dependency, which shouldn't really have been there in the first place.
The second loop in make_trigrams() is now just a fast path and not a
potentially incompatible algorithm.
If there is anybody still using Postgres on machines without wcstombs() or
towlower(), and they have non-ASCII data indexed by pg_trgm, they'll need
to REINDEX those indexes after pg_upgrade to 9.3, else searches may fail
incorrectly. It seems likely that there are no such installations, though.
In passing, rename cnt_trigram to compact_trigram, which seems to better
describe its functionality, and improve make_trigrams' test for whether it
has to use the slow path or not (per a suggestion from Alexander Korotkov).
2013-04-07 20:45:33 +02:00
|
|
|
/* Fast path when there are no multibyte characters */
|
2009-06-11 16:49:15 +02:00
|
|
|
Assert(bytelen == charlen);
|
2008-11-12 14:43:54 +01:00
|
|
|
|
|
|
|
while (ptr - str < bytelen - 2 /* number of trigrams = strlen - 2 */ )
|
|
|
|
{
|
|
|
|
CPTRGM(tptr, ptr);
|
|
|
|
ptr++;
|
|
|
|
tptr++;
|
|
|
|
}
|
|
|
|
}
|
2009-06-11 16:49:15 +02:00
|
|
|
|
2008-11-12 14:43:54 +01:00
|
|
|
return tptr;
|
|
|
|
}
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
/*
|
|
|
|
* Make array of trigrams without sorting and removing duplicate items.
|
|
|
|
*
|
|
|
|
* trg: where to return the array of trigrams.
|
|
|
|
* str: source string, of length slen bytes.
|
2018-03-21 12:57:42 +01:00
|
|
|
* bounds: where to return bounds of trigrams (if needed).
|
2016-03-16 16:59:21 +01:00
|
|
|
*
|
|
|
|
* Returns length of the generated array.
|
|
|
|
*/
|
|
|
|
static int
|
2018-03-21 12:57:42 +01:00
|
|
|
generate_trgm_only(trgm *trg, char *str, int slen, TrgmBound *bounds)
|
2004-08-29 07:07:03 +02:00
|
|
|
{
|
|
|
|
trgm *tptr;
|
2016-03-16 16:59:21 +01:00
|
|
|
char *buf;
|
|
|
|
int charlen,
|
2008-11-12 14:43:54 +01:00
|
|
|
bytelen;
|
2009-06-11 16:49:15 +02:00
|
|
|
char *bword,
|
|
|
|
*eword;
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2004-08-29 07:07:03 +02:00
|
|
|
if (slen + LPADDING + RPADDING < 3 || slen == 0)
|
2016-03-16 16:59:21 +01:00
|
|
|
return 0;
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
tptr = trg;
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2014-01-13 19:07:10 +01:00
|
|
|
/* Allocate a buffer for case-folded, blank-padded words */
|
|
|
|
buf = (char *) palloc(slen * pg_database_encoding_max_length() + 4);
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2004-08-29 07:07:03 +02:00
|
|
|
if (LPADDING > 0)
|
|
|
|
{
|
2004-05-31 19:18:12 +02:00
|
|
|
*buf = ' ';
|
2004-08-29 07:07:03 +02:00
|
|
|
if (LPADDING > 1)
|
|
|
|
*(buf + 1) = ' ';
|
2004-05-31 19:18:12 +02:00
|
|
|
}
|
|
|
|
|
2008-11-12 14:43:54 +01:00
|
|
|
eword = str;
|
2009-06-11 16:49:15 +02:00
|
|
|
while ((bword = find_word(eword, slen - (eword - str), &eword, &charlen)) != NULL)
|
2004-08-29 07:07:03 +02:00
|
|
|
{
|
2008-11-12 14:43:54 +01:00
|
|
|
#ifdef IGNORECASE
|
|
|
|
bword = lowerstr_with_len(bword, eword - bword);
|
|
|
|
bytelen = strlen(bword);
|
2004-05-31 19:18:12 +02:00
|
|
|
#else
|
2008-11-12 14:43:54 +01:00
|
|
|
bytelen = eword - bword;
|
2004-05-31 19:18:12 +02:00
|
|
|
#endif
|
2008-11-12 14:43:54 +01:00
|
|
|
|
|
|
|
memcpy(buf + LPADDING, bword, bytelen);
|
2004-05-31 19:18:12 +02:00
|
|
|
|
|
|
|
#ifdef IGNORECASE
|
2008-11-12 14:43:54 +01:00
|
|
|
pfree(bword);
|
2004-05-31 19:18:12 +02:00
|
|
|
#endif
|
2014-01-13 19:07:10 +01:00
|
|
|
|
2009-06-11 16:49:15 +02:00
|
|
|
buf[LPADDING + bytelen] = ' ';
|
|
|
|
buf[LPADDING + bytelen + 1] = ' ';
|
2008-11-12 14:43:54 +01:00
|
|
|
|
2018-03-21 12:57:42 +01:00
|
|
|
/* Calculate trigrams marking their bounds if needed */
|
|
|
|
if (bounds)
|
|
|
|
bounds[tptr - trg] |= TRGM_BOUND_LEFT;
|
2009-06-11 16:49:15 +02:00
|
|
|
tptr = make_trigrams(tptr, buf, bytelen + LPADDING + RPADDING,
|
|
|
|
charlen + LPADDING + RPADDING);
|
2018-03-21 12:57:42 +01:00
|
|
|
if (bounds)
|
|
|
|
bounds[tptr - trg - 1] |= TRGM_BOUND_RIGHT;
|
2004-05-31 19:18:12 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
pfree(buf);
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
return tptr - trg;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Guard against possible overflow in the palloc requests below. (We
|
|
|
|
* don't worry about the additive constants, since palloc can detect
|
|
|
|
* requests that are a little above MaxAllocSize --- we just need to
|
|
|
|
* prevent integer overflow in the multiplications.)
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
protect_out_of_mem(int slen)
|
|
|
|
{
|
|
|
|
if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
|
|
|
|
(Size) slen >= (MaxAllocSize / pg_database_encoding_max_length()))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
|
|
|
errmsg("out of memory")));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make array of trigrams with sorting and removing duplicate items.
|
|
|
|
*
|
|
|
|
* str: source string, of length slen bytes.
|
|
|
|
*
|
|
|
|
* Returns the sorted array of unique trigrams.
|
|
|
|
*/
|
|
|
|
TRGM *
|
|
|
|
generate_trgm(char *str, int slen)
|
|
|
|
{
|
|
|
|
TRGM *trg;
|
|
|
|
int len;
|
|
|
|
|
|
|
|
protect_out_of_mem(slen);
|
|
|
|
|
2017-06-21 20:39:04 +02:00
|
|
|
trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
|
2016-03-16 16:59:21 +01:00
|
|
|
trg->flag = ARRKEY;
|
|
|
|
|
2018-03-21 12:57:42 +01:00
|
|
|
len = generate_trgm_only(GETARR(trg), str, slen, NULL);
|
2016-03-16 16:59:21 +01:00
|
|
|
SET_VARSIZE(trg, CALCGTSIZE(ARRKEY, len));
|
|
|
|
|
|
|
|
if (len == 0)
|
2004-05-31 19:18:12 +02:00
|
|
|
return trg;
|
|
|
|
|
2014-01-13 19:07:10 +01:00
|
|
|
/*
|
|
|
|
* Make trigrams unique.
|
|
|
|
*/
|
|
|
|
if (len > 1)
|
2004-08-29 07:07:03 +02:00
|
|
|
{
|
2023-02-07 06:53:05 +01:00
|
|
|
qsort(GETARR(trg), len, sizeof(trgm), comp_trgm);
|
2019-11-07 04:51:04 +01:00
|
|
|
len = qunique(GETARR(trg), len, sizeof(trgm), comp_trgm);
|
2004-08-29 07:07:03 +02:00
|
|
|
}
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2007-02-28 23:44:38 +01:00
|
|
|
SET_VARSIZE(trg, CALCGTSIZE(ARRKEY, len));
|
2004-05-31 19:18:12 +02:00
|
|
|
|
|
|
|
return trg;
|
|
|
|
}
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
/*
|
|
|
|
* Make array of positional trigrams from two trigram arrays trg1 and trg2.
|
|
|
|
*
|
|
|
|
* trg1: trigram array of search pattern, of length len1. trg1 is required
|
2016-06-10 00:02:36 +02:00
|
|
|
* word which positions don't matter and replaced with -1.
|
2016-03-16 16:59:21 +01:00
|
|
|
* trg2: trigram array of text, of length len2. trg2 is haystack where we
|
2016-06-10 00:02:36 +02:00
|
|
|
* search and have to store its positions.
|
2016-03-16 16:59:21 +01:00
|
|
|
*
|
|
|
|
* Returns concatenated trigram array.
|
|
|
|
*/
|
|
|
|
static pos_trgm *
|
|
|
|
make_positional_trgm(trgm *trg1, int len1, trgm *trg2, int len2)
|
|
|
|
{
|
|
|
|
pos_trgm *result;
|
2016-06-10 00:02:36 +02:00
|
|
|
int i,
|
|
|
|
len = len1 + len2;
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
result = (pos_trgm *) palloc(sizeof(pos_trgm) * len);
|
|
|
|
|
|
|
|
for (i = 0; i < len1; i++)
|
|
|
|
{
|
|
|
|
memcpy(&result[i].trg, &trg1[i], sizeof(trgm));
|
|
|
|
result[i].index = -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < len2; i++)
|
|
|
|
{
|
|
|
|
memcpy(&result[i + len1].trg, &trg2[i], sizeof(trgm));
|
|
|
|
result[i + len1].index = i;
|
|
|
|
}
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Compare position trigrams: compare trigrams first and position second.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
comp_ptrgm(const void *v1, const void *v2)
|
|
|
|
{
|
2016-06-10 00:02:36 +02:00
|
|
|
const pos_trgm *p1 = (const pos_trgm *) v1;
|
|
|
|
const pos_trgm *p2 = (const pos_trgm *) v2;
|
|
|
|
int cmp;
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
cmp = CMPTRGM(p1->trg, p2->trg);
|
|
|
|
if (cmp != 0)
|
|
|
|
return cmp;
|
|
|
|
|
2024-02-16 21:05:36 +01:00
|
|
|
return pg_cmp_s32(p1->index, p2->index);
|
2016-03-16 16:59:21 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Iterative search function which calculates maximum similarity with word in
|
2022-09-05 11:11:18 +02:00
|
|
|
* the string. Maximum similarity is only calculated only if the flag
|
|
|
|
* WORD_SIMILARITY_CHECK_ONLY isn't set.
|
2016-03-16 16:59:21 +01:00
|
|
|
*
|
|
|
|
* trg2indexes: array which stores indexes of the array "found".
|
|
|
|
* found: array which stores true of false values.
|
|
|
|
* ulen1: count of unique trigrams of array "trg1".
|
|
|
|
* len2: length of array "trg2" and array "trg2indexes".
|
|
|
|
* len: length of the array "found".
|
2022-09-05 11:11:18 +02:00
|
|
|
* flags: set of boolean flags parameterizing similarity calculation.
|
2018-03-21 12:57:42 +01:00
|
|
|
* bounds: whether each trigram is left/right bound of word.
|
2016-03-16 16:59:21 +01:00
|
|
|
*
|
|
|
|
* Returns word similarity.
|
|
|
|
*/
|
|
|
|
static float4
|
|
|
|
iterate_word_similarity(int *trg2indexes,
|
|
|
|
bool *found,
|
|
|
|
int ulen1,
|
|
|
|
int len2,
|
|
|
|
int len,
|
2018-03-21 12:57:42 +01:00
|
|
|
uint8 flags,
|
|
|
|
TrgmBound *bounds)
|
2016-03-16 16:59:21 +01:00
|
|
|
{
|
|
|
|
int *lastpos,
|
|
|
|
i,
|
|
|
|
ulen2 = 0,
|
|
|
|
count = 0,
|
|
|
|
upper = -1,
|
2018-03-21 12:57:42 +01:00
|
|
|
lower;
|
2016-03-16 16:59:21 +01:00
|
|
|
float4 smlr_cur,
|
|
|
|
smlr_max = 0.0f;
|
2018-03-21 12:57:42 +01:00
|
|
|
double threshold;
|
|
|
|
|
|
|
|
Assert(bounds || !(flags & WORD_SIMILARITY_STRICT));
|
|
|
|
|
|
|
|
/* Select appropriate threshold */
|
|
|
|
threshold = (flags & WORD_SIMILARITY_STRICT) ?
|
2018-04-26 20:47:16 +02:00
|
|
|
strict_word_similarity_threshold :
|
|
|
|
word_similarity_threshold;
|
2018-03-21 12:57:42 +01:00
|
|
|
|
|
|
|
/*
|
2019-06-17 09:13:16 +02:00
|
|
|
* Consider first trigram as initial lower bound for strict word
|
2018-04-26 20:47:16 +02:00
|
|
|
* similarity, or initialize it later with first trigram present for plain
|
|
|
|
* word similarity.
|
2018-03-21 12:57:42 +01:00
|
|
|
*/
|
|
|
|
lower = (flags & WORD_SIMILARITY_STRICT) ? 0 : -1;
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
/* Memorise last position of each trigram */
|
|
|
|
lastpos = (int *) palloc(sizeof(int) * len);
|
|
|
|
memset(lastpos, -1, sizeof(int) * len);
|
|
|
|
|
|
|
|
for (i = 0; i < len2; i++)
|
|
|
|
{
|
2022-09-05 11:10:57 +02:00
|
|
|
int trgindex;
|
|
|
|
|
|
|
|
CHECK_FOR_INTERRUPTS();
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
/* Get index of next trigram */
|
2022-09-05 11:10:57 +02:00
|
|
|
trgindex = trg2indexes[i];
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
/* Update last position of this trigram */
|
|
|
|
if (lower >= 0 || found[trgindex])
|
|
|
|
{
|
|
|
|
if (lastpos[trgindex] < 0)
|
|
|
|
{
|
|
|
|
ulen2++;
|
|
|
|
if (found[trgindex])
|
|
|
|
count++;
|
|
|
|
}
|
|
|
|
lastpos[trgindex] = i;
|
|
|
|
}
|
|
|
|
|
2018-03-21 12:57:42 +01:00
|
|
|
/*
|
|
|
|
* Adjust upper bound if trigram is upper bound of word for strict
|
|
|
|
* word similarity, or if trigram is present in required substring for
|
|
|
|
* plain word similarity
|
|
|
|
*/
|
|
|
|
if ((flags & WORD_SIMILARITY_STRICT) ? (bounds[i] & TRGM_BOUND_RIGHT)
|
2018-04-26 20:47:16 +02:00
|
|
|
: found[trgindex])
|
2016-03-16 16:59:21 +01:00
|
|
|
{
|
2016-06-10 00:02:36 +02:00
|
|
|
int prev_lower,
|
|
|
|
tmp_ulen2,
|
|
|
|
tmp_lower,
|
|
|
|
tmp_count;
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
upper = i;
|
|
|
|
if (lower == -1)
|
|
|
|
{
|
|
|
|
lower = i;
|
|
|
|
ulen2 = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
smlr_cur = CALCSML(count, ulen1, ulen2);
|
|
|
|
|
2018-03-21 12:35:56 +01:00
|
|
|
/* Also try to adjust lower bound for greater similarity */
|
2016-03-16 16:59:21 +01:00
|
|
|
tmp_count = count;
|
|
|
|
tmp_ulen2 = ulen2;
|
|
|
|
prev_lower = lower;
|
|
|
|
for (tmp_lower = lower; tmp_lower <= upper; tmp_lower++)
|
|
|
|
{
|
2018-03-21 12:57:42 +01:00
|
|
|
float smlr_tmp;
|
2016-06-10 00:02:36 +02:00
|
|
|
int tmp_trgindex;
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
/*
|
2018-03-21 12:57:42 +01:00
|
|
|
* Adjust lower bound only if trigram is lower bound of word
|
|
|
|
* for strict word similarity, or consider every trigram as
|
|
|
|
* lower bound for plain word similarity.
|
2016-03-16 16:59:21 +01:00
|
|
|
*/
|
2018-03-21 12:57:42 +01:00
|
|
|
if (!(flags & WORD_SIMILARITY_STRICT)
|
|
|
|
|| (bounds[tmp_lower] & TRGM_BOUND_LEFT))
|
|
|
|
{
|
|
|
|
smlr_tmp = CALCSML(tmp_count, ulen1, tmp_ulen2);
|
|
|
|
if (smlr_tmp > smlr_cur)
|
|
|
|
{
|
|
|
|
smlr_cur = smlr_tmp;
|
|
|
|
ulen2 = tmp_ulen2;
|
|
|
|
lower = tmp_lower;
|
|
|
|
count = tmp_count;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we only check that word similarity is greater than
|
|
|
|
* threshold we do not need to calculate a maximum
|
|
|
|
* similarity.
|
|
|
|
*/
|
|
|
|
if ((flags & WORD_SIMILARITY_CHECK_ONLY)
|
|
|
|
&& smlr_cur >= threshold)
|
|
|
|
break;
|
|
|
|
}
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
tmp_trgindex = trg2indexes[tmp_lower];
|
|
|
|
if (lastpos[tmp_trgindex] == tmp_lower)
|
|
|
|
{
|
|
|
|
tmp_ulen2--;
|
|
|
|
if (found[tmp_trgindex])
|
|
|
|
tmp_count--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
smlr_max = Max(smlr_max, smlr_cur);
|
2016-06-10 00:02:36 +02:00
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
/*
|
2018-04-26 20:47:16 +02:00
|
|
|
* if we only check that word similarity is greater than threshold
|
|
|
|
* we do not need to calculate a maximum similarity.
|
2016-03-16 16:59:21 +01:00
|
|
|
*/
|
2018-03-21 12:57:42 +01:00
|
|
|
if ((flags & WORD_SIMILARITY_CHECK_ONLY) && smlr_max >= threshold)
|
2016-03-16 16:59:21 +01:00
|
|
|
break;
|
|
|
|
|
|
|
|
for (tmp_lower = prev_lower; tmp_lower < lower; tmp_lower++)
|
|
|
|
{
|
2016-06-10 00:02:36 +02:00
|
|
|
int tmp_trgindex;
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
tmp_trgindex = trg2indexes[tmp_lower];
|
|
|
|
if (lastpos[tmp_trgindex] == tmp_lower)
|
|
|
|
lastpos[tmp_trgindex] = -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pfree(lastpos);
|
|
|
|
|
|
|
|
return smlr_max;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate word similarity.
|
|
|
|
* This function prepare two arrays: "trg2indexes" and "found". Then this arrays
|
|
|
|
* are used to calculate word similarity using iterate_word_similarity().
|
|
|
|
*
|
|
|
|
* "trg2indexes" is array which stores indexes of the array "found".
|
|
|
|
* In other words:
|
|
|
|
* trg2indexes[j] = i;
|
|
|
|
* found[i] = true (or false);
|
|
|
|
* If found[i] == true then there is trigram trg2[j] in array "trg1".
|
|
|
|
* If found[i] == false then there is not trigram trg2[j] in array "trg1".
|
|
|
|
*
|
|
|
|
* str1: search pattern string, of length slen1 bytes.
|
|
|
|
* str2: text in which we are looking for a word, of length slen2 bytes.
|
2019-10-30 02:03:00 +01:00
|
|
|
* flags: set of boolean flags parameterizing similarity calculation.
|
2016-03-16 16:59:21 +01:00
|
|
|
*
|
|
|
|
* Returns word similarity.
|
|
|
|
*/
|
|
|
|
static float4
|
|
|
|
calc_word_similarity(char *str1, int slen1, char *str2, int slen2,
|
2018-03-21 12:57:42 +01:00
|
|
|
uint8 flags)
|
2016-03-16 16:59:21 +01:00
|
|
|
{
|
|
|
|
bool *found;
|
|
|
|
pos_trgm *ptrg;
|
|
|
|
trgm *trg1;
|
|
|
|
trgm *trg2;
|
|
|
|
int len1,
|
|
|
|
len2,
|
|
|
|
len,
|
|
|
|
i,
|
|
|
|
j,
|
|
|
|
ulen1;
|
|
|
|
int *trg2indexes;
|
|
|
|
float4 result;
|
2018-04-26 20:47:16 +02:00
|
|
|
TrgmBound *bounds;
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
protect_out_of_mem(slen1 + slen2);
|
|
|
|
|
|
|
|
/* Make positional trigrams */
|
2017-06-21 20:39:04 +02:00
|
|
|
trg1 = (trgm *) palloc(sizeof(trgm) * (slen1 / 2 + 1) * 3);
|
|
|
|
trg2 = (trgm *) palloc(sizeof(trgm) * (slen2 / 2 + 1) * 3);
|
2018-03-21 12:57:42 +01:00
|
|
|
if (flags & WORD_SIMILARITY_STRICT)
|
|
|
|
bounds = (TrgmBound *) palloc0(sizeof(TrgmBound) * (slen2 / 2 + 1) * 3);
|
|
|
|
else
|
|
|
|
bounds = NULL;
|
2016-03-16 16:59:21 +01:00
|
|
|
|
2018-03-21 12:57:42 +01:00
|
|
|
len1 = generate_trgm_only(trg1, str1, slen1, NULL);
|
|
|
|
len2 = generate_trgm_only(trg2, str2, slen2, bounds);
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
ptrg = make_positional_trgm(trg1, len1, trg2, len2);
|
|
|
|
len = len1 + len2;
|
|
|
|
qsort(ptrg, len, sizeof(pos_trgm), comp_ptrgm);
|
|
|
|
|
|
|
|
pfree(trg1);
|
|
|
|
pfree(trg2);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Merge positional trigrams array: enumerate each trigram and find its
|
|
|
|
* presence in required word.
|
|
|
|
*/
|
|
|
|
trg2indexes = (int *) palloc(sizeof(int) * len2);
|
|
|
|
found = (bool *) palloc0(sizeof(bool) * len);
|
|
|
|
|
|
|
|
ulen1 = 0;
|
|
|
|
j = 0;
|
|
|
|
for (i = 0; i < len; i++)
|
|
|
|
{
|
|
|
|
if (i > 0)
|
|
|
|
{
|
2016-06-10 00:02:36 +02:00
|
|
|
int cmp = CMPTRGM(ptrg[i - 1].trg, ptrg[i].trg);
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
if (cmp != 0)
|
|
|
|
{
|
|
|
|
if (found[j])
|
|
|
|
ulen1++;
|
|
|
|
j++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ptrg[i].index >= 0)
|
|
|
|
{
|
|
|
|
trg2indexes[ptrg[i].index] = j;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
found[j] = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (found[j])
|
|
|
|
ulen1++;
|
|
|
|
|
|
|
|
/* Run iterative procedure to find maximum similarity with word */
|
|
|
|
result = iterate_word_similarity(trg2indexes, found, ulen1, len2, len,
|
2018-03-21 12:57:42 +01:00
|
|
|
flags, bounds);
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
pfree(trg2indexes);
|
|
|
|
pfree(found);
|
|
|
|
pfree(ptrg);
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-02-01 03:33:55 +01:00
|
|
|
/*
|
2016-03-18 14:38:59 +01:00
|
|
|
* Extract the next non-wildcard part of a search string, i.e. a word bounded
|
2011-02-01 03:33:55 +01:00
|
|
|
* by '_' or '%' meta-characters, non-word characters or string end.
|
|
|
|
*
|
|
|
|
* str: source string, of length lenstr bytes (need not be null-terminated)
|
|
|
|
* buf: where to return the substring (must be long enough)
|
|
|
|
* *bytelen: receives byte length of the found substring
|
|
|
|
* *charlen: receives character length of the found substring
|
|
|
|
*
|
|
|
|
* Returns pointer to end+1 of the found substring in the source string.
|
|
|
|
* Returns NULL if no word found (in which case buf, bytelen, charlen not set)
|
|
|
|
*
|
|
|
|
* If the found word is bounded by non-word characters or string boundaries
|
|
|
|
* then this function will include corresponding padding spaces into buf.
|
|
|
|
*/
|
|
|
|
static const char *
|
|
|
|
get_wildcard_part(const char *str, int lenstr,
|
|
|
|
char *buf, int *bytelen, int *charlen)
|
|
|
|
{
|
|
|
|
const char *beginword = str;
|
|
|
|
const char *endword;
|
|
|
|
char *s = buf;
|
2012-08-20 19:24:52 +02:00
|
|
|
bool in_leading_wildcard_meta = false;
|
|
|
|
bool in_trailing_wildcard_meta = false;
|
2011-04-10 17:42:00 +02:00
|
|
|
bool in_escape = false;
|
|
|
|
int clen;
|
2011-02-01 03:33:55 +01:00
|
|
|
|
|
|
|
/*
|
2012-08-20 19:24:52 +02:00
|
|
|
* Find the first word character, remembering whether preceding character
|
|
|
|
* was wildcard meta-character. Note that the in_escape state persists
|
|
|
|
* from this loop to the next one, since we may exit at a word character
|
|
|
|
* that is in_escape.
|
2011-02-01 03:33:55 +01:00
|
|
|
*/
|
|
|
|
while (beginword - str < lenstr)
|
|
|
|
{
|
|
|
|
if (in_escape)
|
|
|
|
{
|
2013-04-09 07:05:55 +02:00
|
|
|
if (ISWORDCHR(beginword))
|
2011-02-01 03:33:55 +01:00
|
|
|
break;
|
2012-08-20 19:24:52 +02:00
|
|
|
in_escape = false;
|
|
|
|
in_leading_wildcard_meta = false;
|
2011-02-01 03:33:55 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (ISESCAPECHAR(beginword))
|
|
|
|
in_escape = true;
|
|
|
|
else if (ISWILDCARDCHAR(beginword))
|
2012-08-20 19:24:52 +02:00
|
|
|
in_leading_wildcard_meta = true;
|
2013-04-09 07:05:55 +02:00
|
|
|
else if (ISWORDCHR(beginword))
|
2011-02-01 03:33:55 +01:00
|
|
|
break;
|
|
|
|
else
|
2012-08-20 19:24:52 +02:00
|
|
|
in_leading_wildcard_meta = false;
|
2011-02-01 03:33:55 +01:00
|
|
|
}
|
|
|
|
beginword += pg_mblen(beginword);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Handle string end.
|
|
|
|
*/
|
|
|
|
if (beginword - str >= lenstr)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/*
|
2012-08-20 19:24:52 +02:00
|
|
|
* Add left padding spaces if preceding character wasn't wildcard
|
2011-02-01 03:33:55 +01:00
|
|
|
* meta-character.
|
|
|
|
*/
|
|
|
|
*charlen = 0;
|
2012-08-20 19:24:52 +02:00
|
|
|
if (!in_leading_wildcard_meta)
|
2011-02-01 03:33:55 +01:00
|
|
|
{
|
|
|
|
if (LPADDING > 0)
|
|
|
|
{
|
|
|
|
*s++ = ' ';
|
|
|
|
(*charlen)++;
|
|
|
|
if (LPADDING > 1)
|
|
|
|
{
|
|
|
|
*s++ = ' ';
|
|
|
|
(*charlen)++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy data into buf until wildcard meta-character, non-word character or
|
|
|
|
* string boundary. Strip escapes during copy.
|
|
|
|
*/
|
|
|
|
endword = beginword;
|
|
|
|
while (endword - str < lenstr)
|
|
|
|
{
|
|
|
|
clen = pg_mblen(endword);
|
|
|
|
if (in_escape)
|
|
|
|
{
|
2013-04-09 07:05:55 +02:00
|
|
|
if (ISWORDCHR(endword))
|
2011-02-01 03:33:55 +01:00
|
|
|
{
|
|
|
|
memcpy(s, endword, clen);
|
|
|
|
(*charlen)++;
|
|
|
|
s += clen;
|
|
|
|
}
|
|
|
|
else
|
2012-08-20 19:24:52 +02:00
|
|
|
{
|
|
|
|
/*
|
2013-05-29 22:58:43 +02:00
|
|
|
* Back up endword to the escape character when stopping at an
|
|
|
|
* escaped char, so that subsequent get_wildcard_part will
|
2012-08-20 19:24:52 +02:00
|
|
|
* restart from the escape character. We assume here that
|
|
|
|
* escape chars are single-byte.
|
|
|
|
*/
|
|
|
|
endword--;
|
2011-02-01 03:33:55 +01:00
|
|
|
break;
|
2012-08-20 19:24:52 +02:00
|
|
|
}
|
|
|
|
in_escape = false;
|
2011-02-01 03:33:55 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (ISESCAPECHAR(endword))
|
|
|
|
in_escape = true;
|
|
|
|
else if (ISWILDCARDCHAR(endword))
|
|
|
|
{
|
2012-08-20 19:24:52 +02:00
|
|
|
in_trailing_wildcard_meta = true;
|
2011-02-01 03:33:55 +01:00
|
|
|
break;
|
|
|
|
}
|
2013-04-09 07:05:55 +02:00
|
|
|
else if (ISWORDCHR(endword))
|
2011-02-01 03:33:55 +01:00
|
|
|
{
|
|
|
|
memcpy(s, endword, clen);
|
|
|
|
(*charlen)++;
|
|
|
|
s += clen;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
endword += clen;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2012-08-20 19:24:52 +02:00
|
|
|
* Add right padding spaces if next character isn't wildcard
|
2011-02-01 03:33:55 +01:00
|
|
|
* meta-character.
|
|
|
|
*/
|
2012-08-20 19:24:52 +02:00
|
|
|
if (!in_trailing_wildcard_meta)
|
2011-02-01 03:33:55 +01:00
|
|
|
{
|
|
|
|
if (RPADDING > 0)
|
|
|
|
{
|
|
|
|
*s++ = ' ';
|
|
|
|
(*charlen)++;
|
|
|
|
if (RPADDING > 1)
|
|
|
|
{
|
|
|
|
*s++ = ' ';
|
|
|
|
(*charlen)++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
*bytelen = s - buf;
|
|
|
|
return endword;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Generates trigrams for wildcard search string.
|
|
|
|
*
|
|
|
|
* Returns array of trigrams that must occur in any string that matches the
|
|
|
|
* wildcard string. For example, given pattern "a%bcd%" the trigrams
|
|
|
|
* " a", "bcd" would be extracted.
|
|
|
|
*/
|
|
|
|
TRGM *
|
|
|
|
generate_wildcard_trgm(const char *str, int slen)
|
|
|
|
{
|
|
|
|
TRGM *trg;
|
|
|
|
char *buf,
|
2011-04-10 17:42:00 +02:00
|
|
|
*buf2;
|
2011-02-01 03:33:55 +01:00
|
|
|
trgm *tptr;
|
|
|
|
int len,
|
|
|
|
charlen,
|
|
|
|
bytelen;
|
|
|
|
const char *eword;
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
protect_out_of_mem(slen);
|
2014-01-13 19:07:10 +01:00
|
|
|
|
2017-06-21 20:39:04 +02:00
|
|
|
trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
|
2011-02-01 03:33:55 +01:00
|
|
|
trg->flag = ARRKEY;
|
|
|
|
SET_VARSIZE(trg, TRGMHDRSIZE);
|
|
|
|
|
|
|
|
if (slen + LPADDING + RPADDING < 3 || slen == 0)
|
|
|
|
return trg;
|
|
|
|
|
|
|
|
tptr = GETARR(trg);
|
|
|
|
|
2014-01-13 19:07:10 +01:00
|
|
|
/* Allocate a buffer for blank-padded, but not yet case-folded, words */
|
2011-02-01 03:33:55 +01:00
|
|
|
buf = palloc(sizeof(char) * (slen + 4));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract trigrams from each substring extracted by get_wildcard_part.
|
|
|
|
*/
|
|
|
|
eword = str;
|
|
|
|
while ((eword = get_wildcard_part(eword, slen - (eword - str),
|
|
|
|
buf, &bytelen, &charlen)) != NULL)
|
|
|
|
{
|
|
|
|
#ifdef IGNORECASE
|
|
|
|
buf2 = lowerstr_with_len(buf, bytelen);
|
|
|
|
bytelen = strlen(buf2);
|
|
|
|
#else
|
|
|
|
buf2 = buf;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* count trigrams
|
|
|
|
*/
|
|
|
|
tptr = make_trigrams(tptr, buf2, bytelen, charlen);
|
2014-01-13 19:07:10 +01:00
|
|
|
|
2011-02-01 03:33:55 +01:00
|
|
|
#ifdef IGNORECASE
|
|
|
|
pfree(buf2);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
pfree(buf);
|
|
|
|
|
|
|
|
if ((len = tptr - GETARR(trg)) == 0)
|
|
|
|
return trg;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make trigrams unique.
|
|
|
|
*/
|
2014-01-13 19:07:10 +01:00
|
|
|
if (len > 1)
|
2011-02-01 03:33:55 +01:00
|
|
|
{
|
2023-02-07 06:53:05 +01:00
|
|
|
qsort(GETARR(trg), len, sizeof(trgm), comp_trgm);
|
2019-11-07 04:51:04 +01:00
|
|
|
len = qunique(GETARR(trg), len, sizeof(trgm), comp_trgm);
|
2011-02-01 03:33:55 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
SET_VARSIZE(trg, CALCGTSIZE(ARRKEY, len));
|
|
|
|
|
|
|
|
return trg;
|
|
|
|
}
|
|
|
|
|
2008-11-12 14:43:54 +01:00
|
|
|
uint32
|
|
|
|
trgm2int(trgm *ptr)
|
|
|
|
{
|
2009-06-11 16:49:15 +02:00
|
|
|
uint32 val = 0;
|
2008-11-12 14:43:54 +01:00
|
|
|
|
2009-06-11 16:49:15 +02:00
|
|
|
val |= *(((unsigned char *) ptr));
|
2008-11-12 14:43:54 +01:00
|
|
|
val <<= 8;
|
2009-06-11 16:49:15 +02:00
|
|
|
val |= *(((unsigned char *) ptr) + 1);
|
2008-11-12 14:43:54 +01:00
|
|
|
val <<= 8;
|
2009-06-11 16:49:15 +02:00
|
|
|
val |= *(((unsigned char *) ptr) + 2);
|
2008-11-12 14:43:54 +01:00
|
|
|
|
|
|
|
return val;
|
|
|
|
}
|
2004-05-31 19:18:12 +02:00
|
|
|
|
|
|
|
Datum
|
2004-08-29 07:07:03 +02:00
|
|
|
show_trgm(PG_FUNCTION_ARGS)
|
|
|
|
{
|
2017-03-13 00:35:34 +01:00
|
|
|
text *in = PG_GETARG_TEXT_PP(0);
|
2004-08-29 07:07:03 +02:00
|
|
|
TRGM *trg;
|
|
|
|
Datum *d;
|
|
|
|
ArrayType *a;
|
|
|
|
trgm *ptr;
|
2007-07-13 01:10:57 +02:00
|
|
|
int i;
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2017-03-13 00:35:34 +01:00
|
|
|
trg = generate_trgm(VARDATA_ANY(in), VARSIZE_ANY_EXHDR(in));
|
2004-08-29 07:07:03 +02:00
|
|
|
d = (Datum *) palloc(sizeof(Datum) * (1 + ARRNELEM(trg)));
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2007-07-13 01:10:57 +02:00
|
|
|
for (i = 0, ptr = GETARR(trg); i < ARRNELEM(trg); i++, ptr++)
|
2004-08-29 07:07:03 +02:00
|
|
|
{
|
2009-06-11 16:49:15 +02:00
|
|
|
text *item = (text *) palloc(VARHDRSZ + Max(12, pg_database_encoding_max_length() * 3));
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2009-06-11 16:49:15 +02:00
|
|
|
if (pg_database_encoding_max_length() > 1 && !ISPRINTABLETRGM(ptr))
|
2008-11-12 14:43:54 +01:00
|
|
|
{
|
|
|
|
snprintf(VARDATA(item), 12, "0x%06x", trgm2int(ptr));
|
|
|
|
SET_VARSIZE(item, VARHDRSZ + strlen(VARDATA(item)));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
SET_VARSIZE(item, VARHDRSZ + 3);
|
|
|
|
CPTRGM(VARDATA(item), ptr);
|
|
|
|
}
|
2007-07-13 01:10:57 +02:00
|
|
|
d[i] = PointerGetDatum(item);
|
2004-05-31 19:18:12 +02:00
|
|
|
}
|
|
|
|
|
2022-07-01 10:51:45 +02:00
|
|
|
a = construct_array_builtin(d, ARRNELEM(trg), TEXTOID);
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2007-07-13 01:10:57 +02:00
|
|
|
for (i = 0; i < ARRNELEM(trg); i++)
|
|
|
|
pfree(DatumGetPointer(d[i]));
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
pfree(d);
|
|
|
|
pfree(trg);
|
2004-08-29 07:07:03 +02:00
|
|
|
PG_FREE_IF_COPY(in, 0);
|
2004-05-31 19:18:12 +02:00
|
|
|
|
|
|
|
PG_RETURN_POINTER(a);
|
|
|
|
}
|
|
|
|
|
|
|
|
float4
|
2016-03-16 16:59:21 +01:00
|
|
|
cnt_sml(TRGM *trg1, TRGM *trg2, bool inexact)
|
2004-08-29 07:07:03 +02:00
|
|
|
{
|
|
|
|
trgm *ptr1,
|
|
|
|
*ptr2;
|
|
|
|
int count = 0;
|
|
|
|
int len1,
|
|
|
|
len2;
|
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
ptr1 = GETARR(trg1);
|
|
|
|
ptr2 = GETARR(trg2);
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
len1 = ARRNELEM(trg1);
|
|
|
|
len2 = ARRNELEM(trg2);
|
|
|
|
|
2013-02-13 20:07:06 +01:00
|
|
|
/* explicit test is needed to avoid 0/0 division when both lengths are 0 */
|
|
|
|
if (len1 <= 0 || len2 <= 0)
|
|
|
|
return (float4) 0.0;
|
|
|
|
|
2004-08-29 07:07:03 +02:00
|
|
|
while (ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2)
|
|
|
|
{
|
|
|
|
int res = CMPTRGM(ptr1, ptr2);
|
|
|
|
|
|
|
|
if (res < 0)
|
2004-05-31 19:18:12 +02:00
|
|
|
ptr1++;
|
2004-08-29 07:07:03 +02:00
|
|
|
else if (res > 0)
|
2004-05-31 19:18:12 +02:00
|
|
|
ptr2++;
|
2004-08-29 07:07:03 +02:00
|
|
|
else
|
|
|
|
{
|
2004-05-31 19:18:12 +02:00
|
|
|
ptr1++;
|
|
|
|
ptr2++;
|
|
|
|
count++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
/*
|
|
|
|
* If inexact then len2 is equal to count, because we don't know actual
|
|
|
|
* length of second string in inexact search and we can assume that count
|
|
|
|
* is a lower bound of len2.
|
|
|
|
*/
|
|
|
|
return CALCSML(count, len1, inexact ? count : len2);
|
2004-05-31 19:18:12 +02:00
|
|
|
}
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
|
2011-02-01 03:33:55 +01:00
|
|
|
/*
|
|
|
|
* Returns whether trg2 contains all trigrams in trg1.
|
|
|
|
* This relies on the trigram arrays being sorted.
|
|
|
|
*/
|
|
|
|
bool
|
|
|
|
trgm_contained_by(TRGM *trg1, TRGM *trg2)
|
|
|
|
{
|
|
|
|
trgm *ptr1,
|
|
|
|
*ptr2;
|
|
|
|
int len1,
|
|
|
|
len2;
|
|
|
|
|
|
|
|
ptr1 = GETARR(trg1);
|
|
|
|
ptr2 = GETARR(trg2);
|
|
|
|
|
|
|
|
len1 = ARRNELEM(trg1);
|
|
|
|
len2 = ARRNELEM(trg2);
|
|
|
|
|
|
|
|
while (ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2)
|
|
|
|
{
|
|
|
|
int res = CMPTRGM(ptr1, ptr2);
|
|
|
|
|
|
|
|
if (res < 0)
|
|
|
|
return false;
|
|
|
|
else if (res > 0)
|
|
|
|
ptr2++;
|
|
|
|
else
|
|
|
|
{
|
|
|
|
ptr1++;
|
|
|
|
ptr2++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (ptr1 - GETARR(trg1) < len1)
|
|
|
|
return false;
|
|
|
|
else
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2013-04-10 19:30:14 +02:00
|
|
|
/*
|
|
|
|
* Return a palloc'd boolean array showing, for each trigram in "query",
|
|
|
|
* whether it is present in the trigram array "key".
|
|
|
|
* This relies on the "key" array being sorted, but "query" need not be.
|
|
|
|
*/
|
|
|
|
bool *
|
|
|
|
trgm_presence_map(TRGM *query, TRGM *key)
|
|
|
|
{
|
|
|
|
bool *result;
|
|
|
|
trgm *ptrq = GETARR(query),
|
|
|
|
*ptrk = GETARR(key);
|
|
|
|
int lenq = ARRNELEM(query),
|
|
|
|
lenk = ARRNELEM(key),
|
|
|
|
i;
|
|
|
|
|
|
|
|
result = (bool *) palloc0(lenq * sizeof(bool));
|
|
|
|
|
|
|
|
/* for each query trigram, do a binary search in the key array */
|
|
|
|
for (i = 0; i < lenq; i++)
|
|
|
|
{
|
|
|
|
int lo = 0;
|
|
|
|
int hi = lenk;
|
|
|
|
|
|
|
|
while (lo < hi)
|
|
|
|
{
|
|
|
|
int mid = (lo + hi) / 2;
|
|
|
|
int res = CMPTRGM(ptrq, ptrk + mid);
|
|
|
|
|
|
|
|
if (res < 0)
|
|
|
|
hi = mid;
|
|
|
|
else if (res > 0)
|
|
|
|
lo = mid + 1;
|
|
|
|
else
|
|
|
|
{
|
|
|
|
result[i] = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ptrq++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
Datum
|
2004-08-29 07:07:03 +02:00
|
|
|
similarity(PG_FUNCTION_ARGS)
|
|
|
|
{
|
2017-03-13 00:35:34 +01:00
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
2004-08-29 07:07:03 +02:00
|
|
|
TRGM *trg1,
|
|
|
|
*trg2;
|
|
|
|
float4 res;
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2017-03-13 00:35:34 +01:00
|
|
|
trg1 = generate_trgm(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1));
|
|
|
|
trg2 = generate_trgm(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2));
|
2004-05-31 19:18:12 +02:00
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
res = cnt_sml(trg1, trg2, false);
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
pfree(trg1);
|
|
|
|
pfree(trg2);
|
2004-08-29 07:07:03 +02:00
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
2016-03-16 16:59:21 +01:00
|
|
|
Datum
|
|
|
|
word_similarity(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
2016-06-10 00:02:36 +02:00
|
|
|
VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
2018-03-21 12:57:42 +01:00
|
|
|
0);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
strict_word_similarity(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
|
|
|
VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
|
|
|
WORD_SIMILARITY_STRICT);
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_FLOAT4(res);
|
|
|
|
}
|
|
|
|
|
2010-12-04 06:16:21 +01:00
|
|
|
Datum
|
|
|
|
similarity_dist(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
float4 res = DatumGetFloat4(DirectFunctionCall2(similarity,
|
|
|
|
PG_GETARG_DATUM(0),
|
|
|
|
PG_GETARG_DATUM(1)));
|
2011-04-10 17:42:00 +02:00
|
|
|
|
2010-12-04 06:16:21 +01:00
|
|
|
PG_RETURN_FLOAT4(1.0 - res);
|
|
|
|
}
|
|
|
|
|
2004-05-31 19:18:12 +02:00
|
|
|
Datum
|
2004-08-29 07:07:03 +02:00
|
|
|
similarity_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
2010-12-04 06:16:21 +01:00
|
|
|
float4 res = DatumGetFloat4(DirectFunctionCall2(similarity,
|
2005-10-15 04:49:52 +02:00
|
|
|
PG_GETARG_DATUM(0),
|
2010-12-04 06:16:21 +01:00
|
|
|
PG_GETARG_DATUM(1)));
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2016-03-16 15:44:58 +01:00
|
|
|
PG_RETURN_BOOL(res >= similarity_threshold);
|
2004-05-31 19:18:12 +02:00
|
|
|
}
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
Datum
|
|
|
|
word_similarity_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
2016-06-10 00:02:36 +02:00
|
|
|
VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
2018-03-21 12:57:42 +01:00
|
|
|
WORD_SIMILARITY_CHECK_ONLY);
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_BOOL(res >= word_similarity_threshold);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
word_similarity_commutator_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
2016-06-10 00:02:36 +02:00
|
|
|
VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
2018-03-21 12:57:42 +01:00
|
|
|
WORD_SIMILARITY_CHECK_ONLY);
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_BOOL(res >= word_similarity_threshold);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
word_similarity_dist_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
2016-06-10 00:02:36 +02:00
|
|
|
VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
2018-03-21 12:57:42 +01:00
|
|
|
0);
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_FLOAT4(1.0 - res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
word_similarity_dist_commutator_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
2016-06-10 00:02:36 +02:00
|
|
|
VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
2018-03-21 12:57:42 +01:00
|
|
|
0);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_FLOAT4(1.0 - res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
strict_word_similarity_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
|
|
|
VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
|
|
|
WORD_SIMILARITY_CHECK_ONLY | WORD_SIMILARITY_STRICT);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_BOOL(res >= strict_word_similarity_threshold);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
strict_word_similarity_commutator_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
|
|
|
VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
|
|
|
WORD_SIMILARITY_CHECK_ONLY | WORD_SIMILARITY_STRICT);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_BOOL(res >= strict_word_similarity_threshold);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
strict_word_similarity_dist_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
|
|
|
VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
|
|
|
WORD_SIMILARITY_STRICT);
|
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_FLOAT4(1.0 - res);
|
|
|
|
}
|
|
|
|
|
|
|
|
Datum
|
|
|
|
strict_word_similarity_dist_commutator_op(PG_FUNCTION_ARGS)
|
|
|
|
{
|
|
|
|
text *in1 = PG_GETARG_TEXT_PP(0);
|
|
|
|
text *in2 = PG_GETARG_TEXT_PP(1);
|
|
|
|
float4 res;
|
|
|
|
|
|
|
|
res = calc_word_similarity(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
|
|
|
|
VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
|
|
|
|
WORD_SIMILARITY_STRICT);
|
2016-03-16 16:59:21 +01:00
|
|
|
|
|
|
|
PG_FREE_IF_COPY(in1, 0);
|
|
|
|
PG_FREE_IF_COPY(in2, 1);
|
|
|
|
PG_RETURN_FLOAT4(1.0 - res);
|
|
|
|
}
|