diff --git a/contrib/fuzzystrmatch/.gitignore b/contrib/fuzzystrmatch/.gitignore index 5dcb3ff972..b474444663 100644 --- a/contrib/fuzzystrmatch/.gitignore +++ b/contrib/fuzzystrmatch/.gitignore @@ -1,3 +1,5 @@ +# Generated files +/daitch_mokotoff.h # Generated subdirectories /log/ /results/ diff --git a/contrib/fuzzystrmatch/Makefile b/contrib/fuzzystrmatch/Makefile index 0704894f88..e68bc0e33f 100644 --- a/contrib/fuzzystrmatch/Makefile +++ b/contrib/fuzzystrmatch/Makefile @@ -3,14 +3,17 @@ MODULE_big = fuzzystrmatch OBJS = \ $(WIN32RES) \ + daitch_mokotoff.o \ dmetaphone.o \ fuzzystrmatch.o EXTENSION = fuzzystrmatch -DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql +DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.1--1.2.sql \ + fuzzystrmatch--1.0--1.1.sql + PGFILEDESC = "fuzzystrmatch - similarities and distance between strings" -REGRESS = fuzzystrmatch +REGRESS = fuzzystrmatch fuzzystrmatch_utf8 ifdef USE_PGXS PG_CONFIG = pg_config @@ -22,3 +25,16 @@ top_builddir = ../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif + +# Force this dependency to be known even without dependency info built: +daitch_mokotoff.o: daitch_mokotoff.h + +daitch_mokotoff.h: daitch_mokotoff_header.pl + $(PERL) $< $@ + +# daitch_mokotoff.h is included in tarballs, so it has to be made by +# "distprep" and not cleaned except by "maintainer-clean". +distprep: daitch_mokotoff.h + +maintainer-clean: + rm -f daitch_mokotoff.h diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c new file mode 100644 index 0000000000..5acd817096 --- /dev/null +++ b/contrib/fuzzystrmatch/daitch_mokotoff.c @@ -0,0 +1,577 @@ +/* + * Daitch-Mokotoff Soundex + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * This module was originally sponsored by Finance Norway / + * Trafikkforsikringsforeningen, and implemented by Dag Lem + * + * The implementation of the Daitch-Mokotoff Soundex System aims at correctness + * and high performance, and can be summarized as follows: + * + * - The processing of each phoneme is initiated by an O(1) table lookup. + * - For phonemes containing more than one character, a coding tree is traversed + * to process the complete phoneme. + * - The (alternate) soundex codes are produced digit by digit in-place in + * another tree structure. + * + * References: + * + * https://www.avotaynu.com/soundex.htm + * https://www.jewishgen.org/InfoFiles/Soundex.html + * https://familypedia.fandom.com/wiki/Daitch-Mokotoff_Soundex + * https://stevemorse.org/census/soundex.html (dmlat.php, dmsoundex.php) + * https://github.com/apache/commons-codec/ (dmrules.txt, DaitchMokotoffSoundex.java) + * https://metacpan.org/pod/Text::Phonetic (DaitchMokotoff.pm) + * + * A few notes on other implementations: + * + * - All other known implementations have the same unofficial rules for "UE", + * these are also adapted by this implementation (0, 1, NC). + * - The only other known implementation which is capable of generating all + * correct soundex codes in all cases is the JOS Soundex Calculator at + * https://www.jewishgen.org/jos/jossound.htm + * - "J" is considered (only) a vowel in dmlat.php + * - The official rules for "RS" are commented out in dmlat.php + * - Identical code digits for adjacent letters are not collapsed correctly in + * dmsoundex.php when double digit codes are involved. E.g. "BESST" yields + * 744300 instead of 743000 as for "BEST". + * - "J" is considered (only) a consonant in DaitchMokotoffSoundex.java + * - "Y" is not considered a vowel in DaitchMokotoffSoundex.java +*/ + +#include "postgres.h" + +#include "catalog/pg_type.h" +#include "mb/pg_wchar.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + + +/* + * The soundex coding chart table is adapted from + * https://www.jewishgen.org/InfoFiles/Soundex.html + * See daitch_mokotoff_header.pl for details. +*/ + +/* Generated coding chart table */ +#include "daitch_mokotoff.h" + +#define DM_CODE_DIGITS 6 + +/* Node in soundex code tree */ +typedef struct dm_node +{ + int soundex_length; /* Length of generated soundex code */ + char soundex[DM_CODE_DIGITS]; /* Soundex code */ + int is_leaf; /* Candidate for complete soundex code */ + int last_update; /* Letter number for last update of node */ + char code_digit; /* Last code digit, 0 - 9 */ + + /* + * One or two alternate code digits leading to this node. If there are two + * digits, one of them is always an 'X'. Repeated code digits and 'X' lead + * back to the same node. + */ + char prev_code_digits[2]; + /* One or two alternate code digits moving forward. */ + char next_code_digits[2]; + /* ORed together code index(es) used to reach current node. */ + int prev_code_index; + int next_code_index; + /* Possible nodes branching out from this node - digits 0-9. */ + struct dm_node *children[10]; + /* Next node in linked list. Alternating index for each iteration. */ + struct dm_node *next[2]; +} dm_node; + +/* Template for new node in soundex code tree. */ +static const dm_node start_node = { + .soundex_length = 0, + .soundex = "000000", /* Six digits */ + .is_leaf = 0, + .last_update = 0, + .code_digit = '\0', + .prev_code_digits = {'\0', '\0'}, + .next_code_digits = {'\0', '\0'}, + .prev_code_index = 0, + .next_code_index = 0, + .children = {NULL}, + .next = {NULL} +}; + +/* Dummy soundex codes at end of input. */ +static const dm_codes end_codes[2] = +{ + { + "X", "X", "X" + } +}; + +/* Mapping from ISO8859-1 to upper-case ASCII, covering the range 0x60..0xFF. */ +static const char iso8859_1_to_ascii_upper[] = +/* +"`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬ ®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ" +*/ +"`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ ! ?AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDSAAAAAAECEEEEIIIIDNOOOOO/OUUUUYDY"; + +/* Internal C implementation */ +static bool daitch_mokotoff_coding(const char *word, ArrayBuildState *soundex); + + +PG_FUNCTION_INFO_V1(daitch_mokotoff); + +Datum +daitch_mokotoff(PG_FUNCTION_ARGS) +{ + text *arg = PG_GETARG_TEXT_PP(0); + Datum retval; + char *string; + ArrayBuildState *soundex; + MemoryContext old_ctx, + tmp_ctx; + + /* Work in a temporary context to simplify cleanup. */ + tmp_ctx = AllocSetContextCreate(CurrentMemoryContext, + "daitch_mokotoff temporary context", + ALLOCSET_DEFAULT_SIZES); + old_ctx = MemoryContextSwitchTo(tmp_ctx); + + /* We must convert the string to UTF-8 if it isn't already. */ + string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), + PG_UTF8); + + /* The result is built in this ArrayBuildState. */ + soundex = initArrayResult(TEXTOID, tmp_ctx, false); + + if (!daitch_mokotoff_coding(string, soundex)) + { + /* No encodable characters in input */ + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(tmp_ctx); + PG_RETURN_NULL(); + } + + retval = makeArrayResult(soundex, old_ctx); + + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(tmp_ctx); + + PG_RETURN_DATUM(retval); +} + + +/* Initialize soundex code tree node for next code digit. */ +static void +initialize_node(dm_node *node, int last_update) +{ + if (node->last_update < last_update) + { + node->prev_code_digits[0] = node->next_code_digits[0]; + node->prev_code_digits[1] = node->next_code_digits[1]; + node->next_code_digits[0] = '\0'; + node->next_code_digits[1] = '\0'; + node->prev_code_index = node->next_code_index; + node->next_code_index = 0; + node->is_leaf = 0; + node->last_update = last_update; + } +} + + +/* Update soundex code tree node with next code digit. */ +static void +add_next_code_digit(dm_node *node, int code_index, char code_digit) +{ + /* OR in index 1 or 2. */ + node->next_code_index |= code_index; + + if (!node->next_code_digits[0]) + node->next_code_digits[0] = code_digit; + else if (node->next_code_digits[0] != code_digit) + node->next_code_digits[1] = code_digit; +} + + +/* Mark soundex code tree node as leaf. */ +static void +set_leaf(dm_node *first_node[2], dm_node *last_node[2], + dm_node *node, int ix_node) +{ + if (!node->is_leaf) + { + node->is_leaf = 1; + + if (first_node[ix_node] == NULL) + first_node[ix_node] = node; + else + last_node[ix_node]->next[ix_node] = node; + + last_node[ix_node] = node; + node->next[ix_node] = NULL; + } +} + + +/* Find next node corresponding to code digit, or create a new node. */ +static dm_node * +find_or_create_child_node(dm_node *parent, char code_digit, + ArrayBuildState *soundex) +{ + int i = code_digit - '0'; + dm_node **nodes = parent->children; + dm_node *node = nodes[i]; + + if (node) + { + /* Found existing child node. Skip completed nodes. */ + return node->soundex_length < DM_CODE_DIGITS ? node : NULL; + } + + /* Create new child node. */ + node = palloc_object(dm_node); + nodes[i] = node; + + *node = start_node; + memcpy(node->soundex, parent->soundex, sizeof(parent->soundex)); + node->soundex_length = parent->soundex_length; + node->soundex[node->soundex_length++] = code_digit; + node->code_digit = code_digit; + node->next_code_index = node->prev_code_index; + + if (node->soundex_length < DM_CODE_DIGITS) + { + return node; + } + else + { + /* Append completed soundex code to output array. */ + text *out = cstring_to_text_with_len(node->soundex, + DM_CODE_DIGITS); + + accumArrayResult(soundex, + PointerGetDatum(out), + false, + TEXTOID, + CurrentMemoryContext); + return NULL; + } +} + + +/* Update node for next code digit(s). */ +static void +update_node(dm_node *first_node[2], dm_node *last_node[2], + dm_node *node, int ix_node, + int letter_no, int prev_code_index, int next_code_index, + const char *next_code_digits, int digit_no, + ArrayBuildState *soundex) +{ + int i; + char next_code_digit = next_code_digits[digit_no]; + int num_dirty_nodes = 0; + dm_node *dirty_nodes[2]; + + initialize_node(node, letter_no); + + if (node->prev_code_index && !(node->prev_code_index & prev_code_index)) + { + /* + * If the sound (vowel / consonant) of this letter encoding doesn't + * correspond to the coding index of the previous letter, we skip this + * letter encoding. Note that currently, only "J" can be either a + * vowel or a consonant. + */ + return; + } + + if (next_code_digit == 'X' || + (digit_no == 0 && + (node->prev_code_digits[0] == next_code_digit || + node->prev_code_digits[1] == next_code_digit))) + { + /* The code digit is the same as one of the previous (i.e. not added). */ + dirty_nodes[num_dirty_nodes++] = node; + } + + if (next_code_digit != 'X' && + (digit_no > 0 || + node->prev_code_digits[0] != next_code_digit || + node->prev_code_digits[1])) + { + /* The code digit is different from one of the previous (i.e. added). */ + node = find_or_create_child_node(node, next_code_digit, soundex); + if (node) + { + initialize_node(node, letter_no); + dirty_nodes[num_dirty_nodes++] = node; + } + } + + for (i = 0; i < num_dirty_nodes; i++) + { + /* Add code digit leading to the current node. */ + add_next_code_digit(dirty_nodes[i], next_code_index, next_code_digit); + + if (next_code_digits[++digit_no]) + { + update_node(first_node, last_node, dirty_nodes[i], ix_node, + letter_no, prev_code_index, next_code_index, + next_code_digits, digit_no, + soundex); + } + else + { + /* Add incomplete leaf node to linked list. */ + set_leaf(first_node, last_node, dirty_nodes[i], ix_node); + } + } +} + + +/* Update soundex tree leaf nodes. */ +static void +update_leaves(dm_node *first_node[2], int *ix_node, int letter_no, + const dm_codes *codes, const dm_codes *next_codes, + ArrayBuildState *soundex) +{ + int i, + j, + code_index; + dm_node *node, + *last_node[2]; + const dm_code *code, + *next_code; + int ix_node_next = (*ix_node + 1) & 1; /* Alternating index: 0, 1 */ + + /* Initialize for new linked list of leaves. */ + first_node[ix_node_next] = NULL; + last_node[ix_node_next] = NULL; + + /* Process all nodes. */ + for (node = first_node[*ix_node]; node; node = node->next[*ix_node]) + { + /* One or two alternate code sequences. */ + for (i = 0; i < 2 && (code = codes[i]) && code[0][0]; i++) + { + /* Coding for previous letter - before vowel: 1, all other: 2 */ + int prev_code_index = (code[0][0] > '1') + 1; + + /* One or two alternate next code sequences. */ + for (j = 0; j < 2 && (next_code = next_codes[j]) && next_code[0][0]; j++) + { + /* Determine which code to use. */ + if (letter_no == 0) + { + /* This is the first letter. */ + code_index = 0; + } + else if (next_code[0][0] <= '1') + { + /* The next letter is a vowel. */ + code_index = 1; + } + else + { + /* All other cases. */ + code_index = 2; + } + + /* One or two sequential code digits. */ + update_node(first_node, last_node, node, ix_node_next, + letter_no, prev_code_index, code_index, + code[code_index], 0, + soundex); + } + } + } + + *ix_node = ix_node_next; +} + + +/* + * Return next character, converted from UTF-8 to uppercase ASCII. + * *ix is the current string index and is incremented by the character length. + */ +static char +read_char(const unsigned char *str, int *ix) +{ + /* Substitute character for skipped code points. */ + const char na = '\x1a'; + pg_wchar c; + + /* Decode UTF-8 character to ISO 10646 code point. */ + str += *ix; + c = utf8_to_unicode(str); + + /* Advance *ix, but (for safety) not if we've reached end of string. */ + if (c) + *ix += pg_utf_mblen(str); + + /* Convert. */ + if (c >= (unsigned char) '[' && c <= (unsigned char) ']') + { + /* ASCII characters [, \, and ] are reserved for Ą, Ę, and Ţ/Ț. */ + return na; + } + else if (c < 0x60) + { + /* Other non-lowercase ASCII characters can be used as-is. */ + return (char) c; + } + else if (c < 0x100) + { + /* ISO-8859-1 code point; convert to upper-case ASCII via table. */ + return iso8859_1_to_ascii_upper[c - 0x60]; + } + else + { + /* Conversion of non-ASCII characters in the coding chart. */ + switch (c) + { + case 0x0104: + case 0x0105: + /* Ą/ą */ + return '['; + case 0x0118: + case 0x0119: + /* Ę/ę */ + return '\\'; + case 0x0162: + case 0x0163: + case 0x021A: + case 0x021B: + /* Ţ/ţ or Ț/ț */ + return ']'; + default: + return na; + } + } +} + + +/* Read next ASCII character, skipping any characters not in [A-\]]. */ +static char +read_valid_char(const char *str, int *ix) +{ + char c; + + while ((c = read_char((const unsigned char *) str, ix)) != '\0') + { + if (c >= 'A' && c <= ']') + break; + } + + return c; +} + + +/* Return sound coding for "letter" (letter sequence) */ +static const dm_codes * +read_letter(const char *str, int *ix) +{ + char c, + cmp; + int i, + j; + const dm_letter *letters; + const dm_codes *codes; + + /* First letter in sequence. */ + if ((c = read_valid_char(str, ix)) == '\0') + return NULL; + + letters = &letter_[c - 'A']; + codes = letters->codes; + i = *ix; + + /* Any subsequent letters in sequence. */ + while ((letters = letters->letters) && (c = read_valid_char(str, &i))) + { + for (j = 0; (cmp = letters[j].letter); j++) + { + if (cmp == c) + { + /* Letter found. */ + letters = &letters[j]; + if (letters->codes) + { + /* Coding for letter sequence found. */ + codes = letters->codes; + *ix = i; + } + break; + } + } + if (!cmp) + { + /* The sequence of letters has no coding. */ + break; + } + } + + return codes; +} + + +/* + * Generate all Daitch-Mokotoff soundex codes for word, + * adding them to the "soundex" ArrayBuildState. + * Returns false if string has no encodable characters, else true. + */ +static bool +daitch_mokotoff_coding(const char *word, ArrayBuildState *soundex) +{ + int i = 0; + int letter_no = 0; + int ix_node = 0; + const dm_codes *codes, + *next_codes; + dm_node *first_node[2], + *node; + + /* First letter. */ + if (!(codes = read_letter(word, &i))) + { + /* No encodable character in input. */ + return false; + } + + /* Starting point. */ + first_node[ix_node] = palloc_object(dm_node); + *first_node[ix_node] = start_node; + + /* + * Loop until either the word input is exhausted, or all generated soundex + * codes are completed to six digits. + */ + while (codes && first_node[ix_node]) + { + next_codes = read_letter(word, &i); + + /* Update leaf nodes. */ + update_leaves(first_node, &ix_node, letter_no, + codes, next_codes ? next_codes : end_codes, + soundex); + + codes = next_codes; + letter_no++; + } + + /* Append all remaining (incomplete) soundex codes to output array. */ + for (node = first_node[ix_node]; node; node = node->next[ix_node]) + { + text *out = cstring_to_text_with_len(node->soundex, + DM_CODE_DIGITS); + + accumArrayResult(soundex, + PointerGetDatum(out), + false, + TEXTOID, + CurrentMemoryContext); + } + + return true; +} diff --git a/contrib/fuzzystrmatch/daitch_mokotoff_header.pl b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl new file mode 100755 index 0000000000..5426ffd000 --- /dev/null +++ b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl @@ -0,0 +1,223 @@ +#!/usr/bin/perl +# +# Generation of types and lookup tables for Daitch-Mokotoff soundex. +# +# Copyright (c) 2023, PostgreSQL Global Development Group +# +# This module was originally sponsored by Finance Norway / +# Trafikkforsikringsforeningen, and implemented by Dag Lem +# + +use strict; +use warnings; + +use utf8; +use open IO => ':utf8', ':std'; +use Data::Dumper; + +die "Usage: $0 OUTPUT_FILE\n" if @ARGV != 1; +my $output_file = $ARGV[0]; + +# Open the output file +open my $OUTPUT, '>', $output_file + or die "Could not open output file $output_file: $!\n"; + +# Parse code table and generate tree for letter transitions. +my %codes; +my $table = [ {}, [ [ "", "", "" ] ] ]; +while () +{ + chomp; + my ($letters, $codes) = split(/\s+/); + my @codes = map { [ split(/,/) ] } split(/\|/, $codes); + + my $key = "codes_" . join("_or_", map { join("_", @$_) } @codes); + my $val = join( + ",\n", + map { + "\t{\n\t\t" + . join(", ", map { "\"$_\"" } @$_) . "\n\t}" + } @codes); + $codes{$key} = $val; + + for my $letter (split(/,/, $letters)) + { + my $ref = $table->[0]; + # Link each character to the next in the letter combination. + my @c = split(//, $letter); + my $last_c = pop(@c); + for my $c (@c) + { + $ref->{$c} //= [ {}, undef ]; + $ref->{$c}[0] //= {}; + $ref = $ref->{$c}[0]; + } + # The sound code for the letter combination is stored at the last character. + $ref->{$last_c}[1] = $key; + } +} +close(DATA); + +print $OUTPUT <[0]; + for my $key (sort keys %$h) + { + $ref = $h->{$key}; + my $children = "NULL"; + if (defined $ref->[0]) + { + $children = "letter_$letter$key"; + hash2code($ref, "$letter$key"); + } + my $codes = $ref->[1] // "NULL"; + push(@letters, "\t{\n\t\t'$key', $children, $codes\n\t}"); + } + + print $OUTPUT "static const dm_letter letter_$letter\[\] =\n{\n"; + for (@letters) + { + print $OUTPUT "$_,\n"; + } + print $OUTPUT "\t{\n\t\t'\\0'\n\t}\n"; + print $OUTPUT "};\n"; +} + +hash2code($table, ''); + +close $OUTPUT; + +# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html +# +# The conversion from the coding chart to the table should be self +# explanatory, but note the differences stated below. +# +# X = NC (not coded) +# +# The non-ASCII letters in the coding chart are coded with substitute +# lowercase ASCII letters, which sort after the uppercase ASCII letters: +# +# Ą => a (use '[' for table lookup) +# Ę => e (use '\\' for table lookup) +# Ţ => t (use ']' for table lookup) +# +# The rule for "UE" does not correspond to the coding chart, however +# it is used by all other known implementations, including the one at +# https://www.jewishgen.org/jos/jossound.htm (try e.g. "bouey"). +# +# Note that the implementation assumes that vowels are assigned code +# 0 or 1. "J" can be either a vowel or a consonant. +# + +__DATA__ +AI,AJ,AY 0,1,X +AU 0,7,X +a X,X,6|X,X,X +A 0,X,X +B 7,7,7 +CHS 5,54,54 +CH 5,5,5|4,4,4 +CK 5,5,5|45,45,45 +CZ,CS,CSZ,CZS 4,4,4 +C 5,5,5|4,4,4 +DRZ,DRS 4,4,4 +DS,DSH,DSZ 4,4,4 +DZ,DZH,DZS 4,4,4 +D,DT 3,3,3 +EI,EJ,EY 0,1,X +EU 1,1,X +e X,X,6|X,X,X +E 0,X,X +FB 7,7,7 +F 7,7,7 +G 5,5,5 +H 5,5,X +IA,IE,IO,IU 1,X,X +I 0,X,X +J 1,X,X|4,4,4 +KS 5,54,54 +KH 5,5,5 +K 5,5,5 +L 8,8,8 +MN 66,66,66 +M 6,6,6 +NM 66,66,66 +N 6,6,6 +OI,OJ,OY 0,1,X +O 0,X,X +P,PF,PH 7,7,7 +Q 5,5,5 +RZ,RS 94,94,94|4,4,4 +R 9,9,9 +SCHTSCH,SCHTSH,SCHTCH 2,4,4 +SCH 4,4,4 +SHTCH,SHCH,SHTSH 2,4,4 +SHT,SCHT,SCHD 2,43,43 +SH 4,4,4 +STCH,STSCH,SC 2,4,4 +STRZ,STRS,STSH 2,4,4 +ST 2,43,43 +SZCZ,SZCS 2,4,4 +SZT,SHD,SZD,SD 2,43,43 +SZ 4,4,4 +S 4,4,4 +TCH,TTCH,TTSCH 4,4,4 +TH 3,3,3 +TRZ,TRS 4,4,4 +TSCH,TSH 4,4,4 +TS,TTS,TTSZ,TC 4,4,4 +TZ,TTZ,TZS,TSZ 4,4,4 +t 3,3,3|4,4,4 +T 3,3,3 +UI,UJ,UY,UE 0,1,X +U 0,X,X +V 7,7,7 +W 7,7,7 +X 5,54,54 +Y 1,X,X +ZDZ,ZDZH,ZHDZH 2,4,4 +ZD,ZHD 2,43,43 +ZH,ZS,ZSCH,ZSH 4,4,4 +Z 4,4,4 diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out index 493c95cdfa..bcb837fd6b 100644 --- a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out +++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out @@ -65,3 +65,174 @@ SELECT dmetaphone_alt('gumbo'); KMP (1 row) +-- Wovels +SELECT daitch_mokotoff('Augsburg'); + daitch_mokotoff +----------------- + {054795} +(1 row) + +SELECT daitch_mokotoff('Breuer'); + daitch_mokotoff +----------------- + {791900} +(1 row) + +SELECT daitch_mokotoff('Freud'); + daitch_mokotoff +----------------- + {793000} +(1 row) + +-- The letter "H" +SELECT daitch_mokotoff('Halberstadt'); + daitch_mokotoff +----------------- + {587943,587433} +(1 row) + +SELECT daitch_mokotoff('Mannheim'); + daitch_mokotoff +----------------- + {665600} +(1 row) + +-- Adjacent sounds +SELECT daitch_mokotoff('Chernowitz'); + daitch_mokotoff +----------------- + {596740,496740} +(1 row) + +-- Adjacent letters with identical adjacent code digits +SELECT daitch_mokotoff('Cherkassy'); + daitch_mokotoff +----------------- + {595400,495400} +(1 row) + +SELECT daitch_mokotoff('Kleinman'); + daitch_mokotoff +----------------- + {586660} +(1 row) + +-- More than one word +SELECT daitch_mokotoff('Nowy Targ'); + daitch_mokotoff +----------------- + {673950} +(1 row) + +-- Padded with "0" +SELECT daitch_mokotoff('Berlin'); + daitch_mokotoff +----------------- + {798600} +(1 row) + +-- Other examples from https://www.avotaynu.com/soundex.htm +SELECT daitch_mokotoff('Ceniow'); + daitch_mokotoff +----------------- + {567000,467000} +(1 row) + +SELECT daitch_mokotoff('Tsenyuv'); + daitch_mokotoff +----------------- + {467000} +(1 row) + +SELECT daitch_mokotoff('Holubica'); + daitch_mokotoff +----------------- + {587500,587400} +(1 row) + +SELECT daitch_mokotoff('Golubitsa'); + daitch_mokotoff +----------------- + {587400} +(1 row) + +SELECT daitch_mokotoff('Przemysl'); + daitch_mokotoff +----------------- + {794648,746480} +(1 row) + +SELECT daitch_mokotoff('Pshemeshil'); + daitch_mokotoff +----------------- + {746480} +(1 row) + +SELECT daitch_mokotoff('Rosochowaciec'); + daitch_mokotoff +----------------------------------------------------------- + {945755,945754,945745,945744,944755,944754,944745,944744} +(1 row) + +SELECT daitch_mokotoff('Rosokhovatsets'); + daitch_mokotoff +----------------- + {945744} +(1 row) + +-- Ignored characters +SELECT daitch_mokotoff('''OBrien'); + daitch_mokotoff +----------------- + {079600} +(1 row) + +SELECT daitch_mokotoff('O''Brien'); + daitch_mokotoff +----------------- + {079600} +(1 row) + +-- "Difficult" cases, likely to cause trouble for other implementations. +SELECT daitch_mokotoff('CJC'); + daitch_mokotoff +--------------------------------------------- + {550000,540000,545000,450000,400000,440000} +(1 row) + +SELECT daitch_mokotoff('BESST'); + daitch_mokotoff +----------------- + {743000} +(1 row) + +SELECT daitch_mokotoff('BOUEY'); + daitch_mokotoff +----------------- + {710000} +(1 row) + +SELECT daitch_mokotoff('HANNMANN'); + daitch_mokotoff +----------------- + {566600} +(1 row) + +SELECT daitch_mokotoff('MCCOYJR'); + daitch_mokotoff +----------------------------------------------------------- + {651900,654900,654190,654490,645190,645490,641900,644900} +(1 row) + +SELECT daitch_mokotoff('ACCURSO'); + daitch_mokotoff +----------------------------------------------------------- + {059400,054000,054940,054400,045940,045400,049400,044000} +(1 row) + +SELECT daitch_mokotoff('BIERSCHBACH'); + daitch_mokotoff +----------------------------------------------------------- + {794575,794574,794750,794740,745750,745740,747500,747400} +(1 row) + diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out new file mode 100644 index 0000000000..b0dd4880ba --- /dev/null +++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out @@ -0,0 +1,61 @@ +/* + * This test must be run in a database with UTF-8 encoding, + * because other encodings don't support all the characters used. + */ +SELECT getdatabaseencoding() <> 'UTF8' + AS skip_test \gset +\if :skip_test +\quit +\endif +set client_encoding = utf8; +-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; +-- Accents +SELECT daitch_mokotoff('Müller'); + daitch_mokotoff +----------------- + {689000} +(1 row) + +SELECT daitch_mokotoff('Schäfer'); + daitch_mokotoff +----------------- + {479000} +(1 row) + +SELECT daitch_mokotoff('Straßburg'); + daitch_mokotoff +----------------- + {294795} +(1 row) + +SELECT daitch_mokotoff('Éregon'); + daitch_mokotoff +----------------- + {095600} +(1 row) + +-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html +SELECT daitch_mokotoff('gąszczu'); + daitch_mokotoff +----------------- + {564000,540000} +(1 row) + +SELECT daitch_mokotoff('brzęczy'); + daitch_mokotoff +------------------------------- + {794640,794400,746400,744000} +(1 row) + +SELECT daitch_mokotoff('ţamas'); + daitch_mokotoff +----------------- + {364000,464000} +(1 row) + +SELECT daitch_mokotoff('țamas'); + daitch_mokotoff +----------------- + {364000,464000} +(1 row) + diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out new file mode 100644 index 0000000000..37aead89c0 --- /dev/null +++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out @@ -0,0 +1,8 @@ +/* + * This test must be run in a database with UTF-8 encoding, + * because other encodings don't support all the characters used. + */ +SELECT getdatabaseencoding() <> 'UTF8' + AS skip_test \gset +\if :skip_test +\quit diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql new file mode 100644 index 0000000000..d8542a781c --- /dev/null +++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql @@ -0,0 +1,8 @@ +/* contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION fuzzystrmatch UPDATE TO '1.2'" to load this file. \quit + +CREATE FUNCTION daitch_mokotoff(text) RETURNS text[] +AS 'MODULE_PATHNAME', 'daitch_mokotoff' +LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control index 3cd6660bf9..8b6e9fd993 100644 --- a/contrib/fuzzystrmatch/fuzzystrmatch.control +++ b/contrib/fuzzystrmatch/fuzzystrmatch.control @@ -1,6 +1,6 @@ # fuzzystrmatch extension comment = 'determine similarities and distance between strings' -default_version = '1.1' +default_version = '1.2' module_pathname = '$libdir/fuzzystrmatch' relocatable = true trusted = true diff --git a/contrib/fuzzystrmatch/meson.build b/contrib/fuzzystrmatch/meson.build index 6f424be6d4..3ff84eb531 100644 --- a/contrib/fuzzystrmatch/meson.build +++ b/contrib/fuzzystrmatch/meson.build @@ -1,10 +1,19 @@ # Copyright (c) 2022-2023, PostgreSQL Global Development Group fuzzystrmatch_sources = files( - 'fuzzystrmatch.c', + 'daitch_mokotoff.c', 'dmetaphone.c', + 'fuzzystrmatch.c', ) +daitch_mokotoff_h = custom_target('daitch_mokotoff', + input: 'daitch_mokotoff_header.pl', + output: 'daitch_mokotoff.h', + command: [perl, '@INPUT@', '@OUTPUT@'], +) +generated_sources += daitch_mokotoff_h +fuzzystrmatch_sources += daitch_mokotoff_h + if host_system == 'windows' fuzzystrmatch_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ '--NAME', 'fuzzystrmatch', @@ -13,6 +22,7 @@ endif fuzzystrmatch = shared_module('fuzzystrmatch', fuzzystrmatch_sources, + include_directories: include_directories('.'), kwargs: contrib_mod_args, ) contrib_targets += fuzzystrmatch @@ -21,6 +31,7 @@ install_data( 'fuzzystrmatch.control', 'fuzzystrmatch--1.0--1.1.sql', 'fuzzystrmatch--1.1.sql', + 'fuzzystrmatch--1.1--1.2.sql', kwargs: contrib_data_args, ) @@ -31,6 +42,7 @@ tests += { 'regress': { 'sql': [ 'fuzzystrmatch', + 'fuzzystrmatch_utf8', ], }, } diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql index f05dc28ffb..db05c7d6b6 100644 --- a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql +++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql @@ -19,3 +19,48 @@ SELECT metaphone('GUMBO', 4); SELECT dmetaphone('gumbo'); SELECT dmetaphone_alt('gumbo'); + +-- Wovels +SELECT daitch_mokotoff('Augsburg'); +SELECT daitch_mokotoff('Breuer'); +SELECT daitch_mokotoff('Freud'); + +-- The letter "H" +SELECT daitch_mokotoff('Halberstadt'); +SELECT daitch_mokotoff('Mannheim'); + +-- Adjacent sounds +SELECT daitch_mokotoff('Chernowitz'); + +-- Adjacent letters with identical adjacent code digits +SELECT daitch_mokotoff('Cherkassy'); +SELECT daitch_mokotoff('Kleinman'); + +-- More than one word +SELECT daitch_mokotoff('Nowy Targ'); + +-- Padded with "0" +SELECT daitch_mokotoff('Berlin'); + +-- Other examples from https://www.avotaynu.com/soundex.htm +SELECT daitch_mokotoff('Ceniow'); +SELECT daitch_mokotoff('Tsenyuv'); +SELECT daitch_mokotoff('Holubica'); +SELECT daitch_mokotoff('Golubitsa'); +SELECT daitch_mokotoff('Przemysl'); +SELECT daitch_mokotoff('Pshemeshil'); +SELECT daitch_mokotoff('Rosochowaciec'); +SELECT daitch_mokotoff('Rosokhovatsets'); + +-- Ignored characters +SELECT daitch_mokotoff('''OBrien'); +SELECT daitch_mokotoff('O''Brien'); + +-- "Difficult" cases, likely to cause trouble for other implementations. +SELECT daitch_mokotoff('CJC'); +SELECT daitch_mokotoff('BESST'); +SELECT daitch_mokotoff('BOUEY'); +SELECT daitch_mokotoff('HANNMANN'); +SELECT daitch_mokotoff('MCCOYJR'); +SELECT daitch_mokotoff('ACCURSO'); +SELECT daitch_mokotoff('BIERSCHBACH'); diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql new file mode 100644 index 0000000000..f42c01a1bb --- /dev/null +++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql @@ -0,0 +1,26 @@ +/* + * This test must be run in a database with UTF-8 encoding, + * because other encodings don't support all the characters used. + */ + +SELECT getdatabaseencoding() <> 'UTF8' + AS skip_test \gset +\if :skip_test +\quit +\endif + +set client_encoding = utf8; + +-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; + +-- Accents +SELECT daitch_mokotoff('Müller'); +SELECT daitch_mokotoff('Schäfer'); +SELECT daitch_mokotoff('Straßburg'); +SELECT daitch_mokotoff('Éregon'); + +-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html +SELECT daitch_mokotoff('gąszczu'); +SELECT daitch_mokotoff('brzęczy'); +SELECT daitch_mokotoff('ţamas'); +SELECT daitch_mokotoff('țamas'); diff --git a/doc/src/sgml/fuzzystrmatch.sgml b/doc/src/sgml/fuzzystrmatch.sgml index 1a5ebbb22f..bcadc440e3 100644 --- a/doc/src/sgml/fuzzystrmatch.sgml +++ b/doc/src/sgml/fuzzystrmatch.sgml @@ -17,6 +17,8 @@ At present, the soundex, metaphone, dmetaphone, and dmetaphone_alt functions do not work well with multibyte encodings (such as UTF-8). + Use daitch_mokotoff + or levenshtein with such data. @@ -88,6 +90,159 @@ SELECT * FROM s WHERE difference(s.nm, 'john') > 2; + + Daitch-Mokotoff Soundex + + + Like the original Soundex system, Daitch-Mokotoff Soundex matches + similar-sounding names by converting them to the same code. + However, Daitch-Mokotoff Soundex is significantly more useful for + non-English names than the original system. + Major improvements over the original system include: + + + + + The code is based on the first six meaningful letters rather than four. + + + + + A letter or combination of letters maps into ten possible codes rather + than seven. + + + + + Where two consecutive letters have a single sound, they are coded as a + single number. + + + + + When a letter or combination of letters may have different sounds, + multiple codes are emitted to cover all possibilities. + + + + + + + daitch_mokotoff + + + + This function generates the Daitch-Mokotoff soundex codes for its input: + + + +daitch_mokotoff(source text) returns text[] + + + + The result may contain one or more codes depending on how many plausible + pronunciations there are, so it is represented as an array. + + + + Since a Daitch-Mokotoff soundex code consists of only 6 digits, + source should be preferably a single word or name. + + + + Here are some examples: + + + +SELECT daitch_mokotoff('George'); + daitch_mokotoff +----------------- + {595000} + +SELECT daitch_mokotoff('John'); + daitch_mokotoff +----------------- + {160000,460000} + +SELECT daitch_mokotoff('Bierschbach'); + daitch_mokotoff +----------------------------------------------------------- + {794575,794574,794750,794740,745750,745740,747500,747400} + +SELECT daitch_mokotoff('Schwartzenegger'); + daitch_mokotoff +----------------- + {479465} + + + + For matching of single names, returned text arrays can be matched + directly using the && operator: any overlap + can be considered a match. A GIN index may + be used for efficiency, see and this example: + + + +CREATE TABLE s (nm text); +CREATE INDEX ix_s_dm ON s USING gin (daitch_mokotoff(nm)) WITH (fastupdate = off); + +INSERT INTO s (nm) VALUES + ('Schwartzenegger'), + ('John'), + ('James'), + ('Steinman'), + ('Steinmetz'); + +SELECT * FROM s WHERE daitch_mokotoff(nm) && daitch_mokotoff('Swartzenegger'); +SELECT * FROM s WHERE daitch_mokotoff(nm) && daitch_mokotoff('Jane'); +SELECT * FROM s WHERE daitch_mokotoff(nm) && daitch_mokotoff('Jens'); + + + + For indexing and matching of any number of names in any order, Full Text + Search features can be used. See and this + example: + + + +CREATE FUNCTION soundex_tsvector(v_name text) RETURNS tsvector +BEGIN ATOMIC + SELECT to_tsvector('simple', + string_agg(array_to_string(daitch_mokotoff(n), ' '), ' ')) + FROM regexp_split_to_table(v_name, '\s+') AS n; +END; + +CREATE FUNCTION soundex_tsquery(v_name text) RETURNS tsquery +BEGIN ATOMIC + SELECT string_agg('(' || array_to_string(daitch_mokotoff(n), '|') || ')', '&')::tsquery + FROM regexp_split_to_table(v_name, '\s+') AS n; +END; + +CREATE TABLE s (nm text); +CREATE INDEX ix_s_txt ON s USING gin (soundex_tsvector(nm)) WITH (fastupdate = off); + +INSERT INTO s (nm) VALUES + ('John Doe'), + ('Jane Roe'), + ('Public John Q.'), + ('George Best'), + ('John Yamson'); + +SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john'); +SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('jane doe'); +SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john public'); +SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('besst, giorgio'); +SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('Jameson John'); + + + + If it is desired to avoid recalculation of soundex codes during index + rechecks, an index on a separate column can be used instead of an index on + an expression. A stored generated column can be used for this; see + . + + + Levenshtein @@ -104,10 +259,10 @@ SELECT * FROM s WHERE difference(s.nm, 'john') > 2; -levenshtein(text source, text target, int ins_cost, int del_cost, int sub_cost) returns int -levenshtein(text source, text target) returns int -levenshtein_less_equal(text source, text target, int ins_cost, int del_cost, int sub_cost, int max_d) returns int -levenshtein_less_equal(text source, text target, int max_d) returns int +levenshtein(source text, target text, ins_cost int, del_cost int, sub_cost int) returns int +levenshtein(source text, target text) returns int +levenshtein_less_equal(source text, target text, ins_cost int, del_cost int, sub_cost int, max_d int) returns int +levenshtein_less_equal(source text, target text, max_d int) returns int @@ -177,7 +332,7 @@ test=# SELECT levenshtein_less_equal('extensive', 'exhaustive', 4); -metaphone(text source, int max_output_length) returns text +metaphone(source text, max_output_length int) returns text @@ -220,8 +375,8 @@ test=# SELECT metaphone('GUMBO', 4); -dmetaphone(text source) returns text -dmetaphone_alt(text source) returns text +dmetaphone(source text) returns text +dmetaphone_alt(source text) returns text