--- /dev/null
+/*-------------------------------------------------------------------------
+ *
+ * unaccent.c
+ * Text search unaccent dictionary
+ *
+ * Copyright (c) 2009, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * $PostgreSQL: pgsql/contrib/unaccent/unaccent.c,v 1.1 2009/08/18 10:34:39 teodor Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "catalog/namespace.h"
+#include "commands/defrem.h"
+#include "mb/pg_wchar.h"
+#include "tsearch/ts_cache.h"
+#include "tsearch/ts_locale.h"
+#include "tsearch/ts_public.h"
+#include "utils/builtins.h"
+
+PG_MODULE_MAGIC;
+
+/*
+ * Unaccent dictionary uses uncompressed suffix tree to find a
+ * character to replace. Each node of tree is an array of
+ * SuffixChar struct with length = 256 (n-th element of array
+ * corresponds to byte)
+ */
+typedef struct SuffixChar {
+ struct SuffixChar *nextChar;
+ char *replaceTo;
+ int replacelen;
+} SuffixChar;
+
+/*
+ * placeChar - put str into tree's structure, byte by byte.
+ */
+static SuffixChar*
+placeChar(SuffixChar *node, unsigned char *str, int lenstr, char *replaceTo, int replacelen)
+{
+ SuffixChar *curnode;
+
+ if ( !node )
+ {
+ node = palloc(sizeof(SuffixChar) * 256);
+ memset(node, 0, sizeof(SuffixChar) * 256);
+ }
+
+ curnode = node + *str;
+
+ if ( lenstr == 1 )
+ {
+ if ( curnode->replaceTo )
+ elog(WARNING, "duplicate TO argument, use first one");
+ else
+ {
+ curnode->replacelen = replacelen;
+ curnode->replaceTo = palloc( replacelen );
+ memcpy(curnode->replaceTo, replaceTo, replacelen);
+ }
+ }
+ else
+ {
+ curnode->nextChar = placeChar( curnode->nextChar, str+1, lenstr-1, replaceTo, replacelen);
+ }
+
+ return node;
+}
+
+/*
+ * initSuffixTree - create suffix tree from file. Function converts
+ * UTF8-encoded file into current encoding.
+ */
+static SuffixChar*
+initSuffixTree(char *filename)
+{
+ SuffixChar *rootSuffixTree = NULL;
+ MemoryContext ccxt = CurrentMemoryContext;
+ tsearch_readline_state trst;
+ bool skip;
+
+ filename = get_tsearch_config_filename(filename, "rules");
+ if (!tsearch_readline_begin(&trst, filename))
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("could not open unaccent file \"%s\": %m",
+ filename)));
+
+ do
+ {
+ char src[4096];
+ char trg[4096];
+ int srclen;
+ int trglen;
+ char *line = NULL;
+
+ skip = true;
+
+ PG_TRY();
+ {
+ /*
+ * pg_do_encoding_conversion() (called by tsearch_readline())
+ * will emit exception if it finds untranslatable characters in current locale.
+ * We just skip such characters.
+ */
+ while ((line = tsearch_readline(&trst)) != NULL)
+ {
+ if ( sscanf(line, "%s\t%s\n", src, trg)!=2 )
+ continue;
+
+ srclen = strlen(src);
+ trglen = strlen(trg);
+
+ rootSuffixTree = placeChar(rootSuffixTree,
+ (unsigned char*)src, srclen,
+ trg, trglen);
+ skip = false;
+ pfree(line);
+ }
+ }
+ PG_CATCH();
+ {
+ ErrorData *errdata;
+ MemoryContext ecxt;
+
+ ecxt = MemoryContextSwitchTo(ccxt);
+ errdata = CopyErrorData();
+ if (errdata->sqlerrcode == ERRCODE_UNTRANSLATABLE_CHARACTER)
+ {
+ FlushErrorState();
+ }
+ else
+ {
+ MemoryContextSwitchTo(ecxt);
+ PG_RE_THROW();
+ }
+ }
+ PG_END_TRY();
+ }
+ while(skip);
+
+ tsearch_readline_end(&trst);
+
+ return rootSuffixTree;
+}
+
+/*
+ * findReplaceTo - find multibyte character in tree
+ */
+static SuffixChar *
+findReplaceTo( SuffixChar *node, unsigned char *src, int srclen )
+{
+ while( node )
+ {
+ node = node + *src;
+ if ( srclen == 1 )
+ return node;
+
+ src++;
+ srclen--;
+ node = node->nextChar;
+ }
+
+ return NULL;
+}
+
+PG_FUNCTION_INFO_V1(unaccent_init);
+Datum unaccent_init(PG_FUNCTION_ARGS);
+Datum
+unaccent_init(PG_FUNCTION_ARGS)
+{
+ List *dictoptions = (List *) PG_GETARG_POINTER(0);
+ SuffixChar *rootSuffixTree;
+ bool fileloaded = false;
+ ListCell *l;
+
+ foreach(l, dictoptions)
+ {
+ DefElem *defel = (DefElem *) lfirst(l);
+
+ if (pg_strcasecmp("Rules", defel->defname) == 0)
+ {
+ if (fileloaded)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("multiple Rules parameters")));
+ rootSuffixTree = initSuffixTree(defGetString(defel));
+ fileloaded = true;
+ }
+ else
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("unrecognized Unaccent parameter: \"%s\"",
+ defel->defname)));
+ }
+ }
+
+ if (!fileloaded)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("missing Rules parameter")));
+ }
+
+ PG_RETURN_POINTER(rootSuffixTree);
+}
+
+PG_FUNCTION_INFO_V1(unaccent_lexize);
+Datum unaccent_lexize(PG_FUNCTION_ARGS);
+Datum
+unaccent_lexize(PG_FUNCTION_ARGS)
+{
+ SuffixChar *rootSuffixTree = (SuffixChar*)PG_GETARG_POINTER(0);
+ char *srcchar = (char *) PG_GETARG_POINTER(1);
+ int32 len = PG_GETARG_INT32(2);
+ char *srcstart, *trgchar;
+ int charlen;
+ TSLexeme *res = NULL;
+ SuffixChar *node;
+
+ srcstart = srcchar;
+ while( srcchar - srcstart < len )
+ {
+ charlen = pg_mblen(srcchar);
+
+ node = findReplaceTo( rootSuffixTree, (unsigned char *) srcchar, charlen );
+ if ( node && node->replaceTo )
+ {
+ if ( !res )
+ {
+ /* allocate res only it it's needed */
+ res = palloc0(sizeof(TSLexeme) * 2);
+ res->lexeme = trgchar = palloc( len * pg_database_encoding_max_length() + 1 /* \0 */ );
+ res->flags = TSL_FILTER;
+ if ( srcchar != srcstart )
+ {
+ memcpy(trgchar, srcstart, srcchar - srcstart);
+ trgchar += (srcchar - srcstart);
+ }
+ }
+ memcpy( trgchar, node->replaceTo, node->replacelen );
+ trgchar += node->replacelen;
+ }
+ else if ( res )
+ {
+ memcpy( trgchar, srcchar, charlen );
+ trgchar += charlen;
+ }
+
+ srcchar += charlen;
+ }
+
+ if ( res )
+ *trgchar = '\0';
+
+ PG_RETURN_POINTER(res);
+}
+
+/*
+ * Function-like wrapper for dictionary
+ */
+PG_FUNCTION_INFO_V1(unaccent_dict);
+Datum unaccent_dict(PG_FUNCTION_ARGS);
+Datum
+unaccent_dict(PG_FUNCTION_ARGS)
+{
+ text *str;
+ int strArg;
+ Oid dictOid;
+ TSDictionaryCacheEntry *dict;
+ TSLexeme *res;
+
+ if (PG_NARGS() == 1)
+ {
+ dictOid = TSDictionaryGetDictid(stringToQualifiedNameList("unaccent"), false);
+ strArg = 0;
+ }
+ else
+ {
+ dictOid = PG_GETARG_OID(0);
+ strArg = 1;
+ }
+ str = PG_GETARG_TEXT_P(strArg);
+
+ dict = lookup_ts_dictionary_cache(dictOid);
+
+ res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize),
+ PointerGetDatum(dict->dictData),
+ PointerGetDatum(VARDATA(str)),
+ Int32GetDatum(VARSIZE(str) - VARHDRSZ),
+ PointerGetDatum(NULL)));
+
+ PG_FREE_IF_COPY(str, strArg);
+
+ if ( res == NULL )
+ {
+ PG_RETURN_TEXT_P(PG_GETARG_TEXT_P_COPY(strArg));
+ }
+ else if ( res->lexeme == NULL )
+ {
+ pfree(res);
+ PG_RETURN_TEXT_P(PG_GETARG_TEXT_P_COPY(strArg));
+ }
+ else
+ {
+ text *txt = cstring_to_text(res->lexeme);
+
+ pfree(res->lexeme);
+ pfree(res);
+
+ PG_RETURN_TEXT_P(txt);
+ }
+}
--- /dev/null
+
+
unaccent
+
+
+
+
+ unaccent removes accents (diacritic signs) from a lexeme.
+ It's a filtering dictionary, that means its output is
+ always passed to the next dictionary (if any), contrary to the standard
+ behaviour. Currently, it supports most important accents from european
+ languages.
+
+
+ Limitation: Current implementation of unaccent
+ dictionary cannot be used as a normalizing dictionary for
+ thesaurus dictionary.
+
+
+
+
Configuration
+
+ A unaccent dictionary accepts the following options:
+
+
+
+ RULES is the base name of the file containing the list of
+ translation rules. This file must be stored in
+ $SHAREDIR/tsearch_data/ (where $SHAREDIR means
+ the
PostgreSQL installation's shared-data directory).+ Its name must end in .rules (which is not to be included in
+ the RULES parameter).
+
+
+
+ The rules file has the following format:
+
+
+
+ Each line represents pair: character_with_accent character_without_accent
+À A
+Á A
+Â A
+Ã A
+Ä A
+Å A
+Æ A
+
+
+
+
+
+ Look at unaccent.rules, which is installed in
+ $SHAREDIR/tsearch_data/, for an example.
+
+
+
+
+
Usage
+
+ Running the installation script creates a text search template
+ unaccent and a dictionary unaccent
+ based on it, with default parameters. You can alter the
+ parameters, for example
+
+=# ALTER TEXT SEARCH DICTIONARY unaccent (RULES='my_rules');
+
+
+ or create new dictionaries based on the template.
+
+
+ To test the dictionary, you can try
+
+=# select ts_lexize('unaccent','Hôtel');
+ ts_lexize
+-----------
+ {Hotel}
+(1 row)
+
+
+
+ Filtering dictionary are useful for correct work of
+ ts_headline function.
+=# CREATE TEXT SEARCH CONFIGURATION fr ( COPY = french );
+=# ALTER TEXT SEARCH CONFIGURATION fr
+ ALTER MAPPING FOR hword, hword_part, word
+ WITH unaccent, french_stem;
+=# select to_tsvector('fr','Hôtels de la Mer');
+ to_tsvector
+-------------------
+ 'hotel':1 'mer':4
+(1 row)
+
+=# select to_tsvector('fr','Hôtel de la Mer') @@ to_tsquery('fr','Hotels');
+ ?column?
+----------
+ t
+(1 row)
+=# select ts_headline('fr','Hôtel de la Mer',to_tsquery('fr','Hotels'));
+ ts_headline
+------------------------
+ <b>Hôtel</b>de la Mer
+(1 row)
+
+
+
+
+
+
+
Function
+
+ unaccent function removes accents (diacritic signs) from
+ argument string. Basically, it's a wrapper around
+ unaccent dictionary.
+
+
+
+
+
+
+ unaccent(dictionary,
+ string)
+ returns text
+
+
+SELECT unaccent('unaccent','Hôtel');
+SELECT unaccent('Hôtel');
+
+
+
+
+
projects / postgresql.git / commitdiff