rdelaage
/
postgresql
mirror of https://git.postgresql.org/git/postgresql.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Teach levenshtein() about multi-byte characters. 

Based on a patch by, and further ideas from, Alexander Korotkov.
This commit is contained in:
Robert Haas 2010-08-02 23:20:23 +00:00
parent ad17ff95cf
commit 57d9aefcaa
2 changed files with 120 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
contrib/fuzzystrmatch/fuzzystrmatch.c
View File

@ -5,7 +5,7 @@
*
* Joe Conway <mail@joeconway.com>
*
* $PostgreSQL: pgsql/contrib/fuzzystrmatch/fuzzystrmatch.c,v 1.33 2010/07/29 20:11:48 rhaas Exp $
* $PostgreSQL: pgsql/contrib/fuzzystrmatch/fuzzystrmatch.c,v 1.34 2010/08/02 23:20:23 rhaas Exp $
* Copyright (c) 2001-2010, PostgreSQL Global Development Group
* ALL RIGHTS RESERVED;
*
@ -50,6 +50,7 @@
#include <ctype.h>
#include "fmgr.h"
#include "mb/pg_wchar.h"
#include "utils/builtins.h"
PG_MODULE_MAGIC;
@ -183,6 +184,18 @@ getcode(char c)
/* These prevent GH from becoming F */
#define NOGHTOF(c) (getcode(c) & 16) /* BDH */
/* Faster than memcmp(), for this use case. */
static bool inline
rest_of_char_same(const char *s1, const char *s2, int len)
{
while (len > 0)
{
len--;
if (s1[len] != s2[len])
return false;
}
return true;
}
/*
* levenshtein_internal - Calculates Levenshtein distance metric
@ -195,16 +208,27 @@ levenshtein_internal(text *s, text *t,
int ins_c, int del_c, int sub_c)
{
int m,
n;
n,
s_bytes,
t_bytes;
int *prev;
int *curr;
int *s_char_len = NULL;
int i,
j;
const char *x;
const char *s_data;
const char *t_data;
const char *y;
m = VARSIZE_ANY_EXHDR(s);
n = VARSIZE_ANY_EXHDR(t);
/* Extract a pointer to the actual character data. */
s_data = VARDATA_ANY(s);
t_data = VARDATA_ANY(t);
/* Determine length of each string in bytes and characters. */
s_bytes = VARSIZE_ANY_EXHDR(s);
t_bytes = VARSIZE_ANY_EXHDR(t);
m = pg_mbstrlen_with_len(s_data, s_bytes);
n = pg_mbstrlen_with_len(t_data, t_bytes);
/*
* We can transform an empty s into t with n insertions, or a non-empty t
@ -226,6 +250,28 @@ levenshtein_internal(text *s, text *t,
errmsg("argument exceeds the maximum length of %d bytes",
MAX_LEVENSHTEIN_STRLEN)));
/*
* In order to avoid calling pg_mblen() repeatedly on each character in s,
* we cache all the lengths before starting the main loop -- but if all the
* characters in both strings are single byte, then we skip this and use
* a fast-path in the main loop. If only one string contains multi-byte
* characters, we still build the array, so that the fast-path needn't
* deal with the case where the array hasn't been initialized.
*/
if (m != s_bytes || n != t_bytes)
{
int i;
const char *cp = s_data;
s_char_len = (int *) palloc((m + 1) * sizeof(int));
for (i = 0; i < m; ++i)
{
s_char_len[i] = pg_mblen(cp);
cp += s_char_len[i];
}
s_char_len[i] = 0;
}
/* One more cell for initialization column and row. */
++m;
++n;
@ -244,9 +290,11 @@ levenshtein_internal(text *s, text *t,
prev[i] = i * del_c;
/* Loop through rows of the notional array */
for (y = VARDATA_ANY(t), j = 1; j < n; y++, j++)
for (y = t_data, j = 1; j < n; j++)
{
int *temp;
const char *x = s_data;
int y_char_len = n != t_bytes + 1 ? pg_mblen(y) : 1;
/*
* First cell must increment sequentially, as we're on the j'th row of
@ -254,26 +302,77 @@ levenshtein_internal(text *s, text *t,
*/
curr[0] = j * ins_c;
for (x = VARDATA_ANY(s), i = 1; i < m; x++, i++)
/*
* This inner loop is critical to performance, so we include a
* fast-path to handle the (fairly common) case where no multibyte
* characters are in the mix. The fast-path is entitled to assume
* that if s_char_len is not initialized then BOTH strings contain
* only single-byte characters.
*/
if (s_char_len != NULL)
{
int ins;
int del;
int sub;
for (i = 1; i < m; i++)
{
int ins;
int del;
int sub;
int x_char_len = s_char_len[i - 1];
/* Calculate costs for probable operations. */
ins = prev[i] + ins_c; /* Insertion */
del = curr[i - 1] + del_c; /* Deletion */
sub = prev[i - 1] + ((*x == *y) ? 0 : sub_c); /* Substitution */
/*
* Calculate costs for insertion, deletion, and substitution.
*
* When calculating cost for substitution, we compare the last
* character of each possibly-multibyte character first,
* because that's enough to rule out most mis-matches. If we
* get past that test, then we compare the lengths and the
* remaining bytes.
*/
ins = prev[i] + ins_c;
del = curr[i - 1] + del_c;
if (x[x_char_len-1] == y[y_char_len-1]
&& x_char_len == y_char_len &&
(x_char_len == 1 || rest_of_char_same(x, y, x_char_len)))
sub = prev[i - 1];
else
sub = prev[i - 1] + sub_c;
/* Take the one with minimum cost. */
curr[i] = Min(ins, del);
curr[i] = Min(curr[i], sub);
/* Take the one with minimum cost. */
curr[i] = Min(ins, del);
curr[i] = Min(curr[i], sub);
/* Point to next character. */
x += x_char_len;
}
}
else
{
for (i = 1; i < m; i++)
{
int ins;
int del;
int sub;
/* Calculate costs for insertion, deletion, and substitution. */
ins = prev[i] + ins_c;
del = curr[i - 1] + del_c;
sub = prev[i - 1] + ((*x == *y) ? 0 : sub_c);
/* Take the one with minimum cost. */
curr[i] = Min(ins, del);
curr[i] = Min(curr[i], sub);
/* Point to next character. */
x++;
}
}
/* Swap current row with previous row. */
temp = curr;
curr = prev;
prev = temp;
/* Point to next character. */
y += y_char_len;
}
/*

7
doc/src/sgml/fuzzystrmatch.sgml
View File

@ -1,4 +1,4 @@
<!-- $PostgreSQL: pgsql/doc/src/sgml/fuzzystrmatch.sgml,v 1.6 2010/07/29 19:34:40 petere Exp $ -->
<!-- $PostgreSQL: pgsql/doc/src/sgml/fuzzystrmatch.sgml,v 1.7 2010/08/02 23:20:23 rhaas Exp $ -->
<sect1 id="fuzzystrmatch">
<title>fuzzystrmatch</title>
@ -14,8 +14,9 @@
<caution>
<para>
At present, <filename>fuzzystrmatch</> does not work well with
multi-byte encodings (such as UTF-8).
At present, the <function>soundex</>, <function>metaphone</>,
<function>dmetaphone</>, and <function>dmetaphone_alt</> functions do
not work well with multi-byte encodings (such as UTF-8).
</para>
</caution>