improve support of agglutinative languages (query with compound words).

regression=# select to_tsquery( '\'fotballklubber\'');
                   to_tsquery
------------------------------------------------
 'fotball' & 'klubb' | 'fot' & 'ball' & 'klubb'
(1 row)

So, changed interface to dictionaries, lexize method of dictionary shoud return
pointer to aray of TSLexeme structs instead of char**. Last element should
have TSLexeme->lexeme == NULL.

typedef struct {
        /* number of variant of split word , for example
                Word 'fotballklubber' (norwegian) has two varian to split:
                ( fotball, klubb ) and ( fot, ball, klubb ). So, dictionary
                should return:
                nvariant        lexeme
                1               fotball
                1               klubb
                2               fot
                2               ball
                2               klubb

        */
        uint16  nvariant;

        /* currently unused */
        uint16  flags;

        /* C-string */
        char    *lexeme;
} TSLexeme;

diff --git a/contrib/tsearch2/dict.c b/contrib/tsearch2/dict.c
index 357097681e55d0a839c95aeb1e50eab2fb22b6f8..7a3626b3bc9934c5590ea2188b9931a1d04086b8 100644 (file)
--- a/contrib/tsearch2/dict.c
+++ b/contrib/tsearch2/dict.c
@@ -183,15 +183,15 @@ lexize(PG_FUNCTION_ARGS)
 {
    text       *in = PG_GETARG_TEXT_P(1);
    DictInfo   *dict;
-   char      **res,
-             **ptr;
+   TSLexeme      *res,
+             *ptr;
    Datum      *da;
    ArrayType  *a;
 
    SET_FUNCOID();
    dict = finddict(PG_GETARG_OID(0));
 
-   ptr = res = (char **) DatumGetPointer(
+   ptr = res = (TSLexeme *) DatumGetPointer(
                                      FunctionCall3(&(dict->lexize_info),
                                       PointerGetDatum(dict->dictionary),
                                            PointerGetDatum(VARDATA(in)),
@@ -207,13 +207,13 @@ lexize(PG_FUNCTION_ARGS)
            PG_RETURN_NULL();
    }
 
-   while (*ptr)
+   while (ptr->lexeme)
        ptr++;
    da = (Datum *) palloc(sizeof(Datum) * (ptr - res + 1));
    ptr = res;
-   while (*ptr)
+   while (ptr->lexeme)
    {
-       da[ptr - res] = PointerGetDatum(char2text(*ptr));
+       da[ptr - res] = PointerGetDatum(char2text(ptr->lexeme));
        ptr++;
    }
 
@@ -227,10 +227,10 @@ lexize(PG_FUNCTION_ARGS)
        );
 
    ptr = res;
-   while (*ptr)
+   while (ptr->lexeme)
    {
        pfree(DatumGetPointer(da[ptr - res]));
-       pfree(*ptr);
+       pfree(ptr->lexeme);
        ptr++;
    }
    pfree(res);

diff --git a/contrib/tsearch2/dict.h b/contrib/tsearch2/dict.h
index 86ea42263e5069a201de18fa041f4b13952ecd66..a21086a49dd4b0732b64d317051221eca221f22f 100644 (file)
--- a/contrib/tsearch2/dict.h
+++ b/contrib/tsearch2/dict.h
@@ -38,4 +38,27 @@ typedef struct
 
 void       parse_cfgdict(text *in, Map ** m);
 
+/* return struct for any lexize function */
+typedef struct {
+   /* number of variant of split word , for example
+       Word 'fotballklubber' (norwegian) has two varian to split:
+       ( fotball, klubb ) and ( fot, ball, klubb ). So, dictionary
+       should return:
+       nvariant    lexeme
+       1       fotball
+       1       klubb
+       2       fot
+       2       ball
+       2       klubb
+
+   */
+   uint16  nvariant;
+
+   /* currently unused */
+   uint16  flags;
+
+   /* C-string */
+   char    *lexeme;
+} TSLexeme;
+
 #endif

diff --git a/contrib/tsearch2/dict_ex.c b/contrib/tsearch2/dict_ex.c
index a8fb20453ba2125bed97bbadc0cfff28cef5112f..241161a5c21567a4338751107f8280b89f56e8ae 100644 (file)
--- a/contrib/tsearch2/dict_ex.c
+++ b/contrib/tsearch2/dict_ex.c
@@ -54,16 +54,16 @@ dex_lexize(PG_FUNCTION_ARGS)
    DictExample *d = (DictExample *) PG_GETARG_POINTER(0);
    char       *in = (char *) PG_GETARG_POINTER(1);
    char       *txt = pnstrdup(in, PG_GETARG_INT32(2));
-   char      **res = palloc(sizeof(char *) * 2);
+   TSLexeme   *res = palloc(sizeof(TSLexeme) * 2);
+
+   memset(res,0,sizeof(TSLexeme) * 2);
 
    if (*txt == '\0' || searchstoplist(&(d->stoplist), txt))
    {
        pfree(txt);
-       res[0] = NULL;
    }
    else
-       res[0] = txt;
-   res[1] = NULL;
+       res[0].lexeme = txt;
 
    PG_RETURN_POINTER(res);
 }

diff --git a/contrib/tsearch2/dict_ispell.c b/contrib/tsearch2/dict_ispell.c
index 5725c8fb3629020d8efaa3d3c475879e80c301e2..9af11edf8e21434afb59a16805bd2431476e58f2 100644 (file)
--- a/contrib/tsearch2/dict_ispell.c
+++ b/contrib/tsearch2/dict_ispell.c
@@ -159,14 +159,13 @@ spell_lexize(PG_FUNCTION_ARGS)
    DictISpell *d = (DictISpell *) PG_GETARG_POINTER(0);
    char       *in = (char *) PG_GETARG_POINTER(1);
    char       *txt;
-   char      **res;
-   char      **ptr,
-             **cptr;
+   TSLexeme      *res;
+   TSLexeme      *ptr,
+             *cptr;
 
    if (!PG_GETARG_INT32(2))
        PG_RETURN_POINTER(NULL);
 
-   res = palloc(sizeof(char *) * 2);
    txt = pnstrdup(in, PG_GETARG_INT32(2));
    res = NINormalizeWord(&(d->obj), txt);
    pfree(txt);
@@ -175,22 +174,22 @@ spell_lexize(PG_FUNCTION_ARGS)
        PG_RETURN_POINTER(NULL);
 
    ptr = cptr = res;
-   while (*ptr)
+   while (ptr->lexeme)
    {
-       if (searchstoplist(&(d->stoplist), *ptr))
+       if (searchstoplist(&(d->stoplist), ptr->lexeme))
        {
-           pfree(*ptr);
-           *ptr = NULL;
+           pfree(ptr->lexeme);
+           ptr->lexeme = NULL;
            ptr++;
        }
        else
        {
-           *cptr = *ptr;
+           memcpy(cptr, ptr, sizeof(TSLexeme));
            cptr++;
            ptr++;
        }
    }
-   *cptr = NULL;
+   cptr->lexeme = NULL;
 
    PG_RETURN_POINTER(res);
 }

diff --git a/contrib/tsearch2/dict_snowball.c b/contrib/tsearch2/dict_snowball.c
index 51dba0444994e36eb8017aaa8c3b86287054151b..03850b33ea44843d52e59e7ed83b30018062acbb 100644 (file)
--- a/contrib/tsearch2/dict_snowball.c
+++ b/contrib/tsearch2/dict_snowball.c
@@ -105,12 +105,12 @@ snb_lexize(PG_FUNCTION_ARGS)
    DictSnowball *d = (DictSnowball *) PG_GETARG_POINTER(0);
    char       *in = (char *) PG_GETARG_POINTER(1);
    char       *txt = pnstrdup(in, PG_GETARG_INT32(2));
-   char      **res = palloc(sizeof(char *) * 2);
+   TSLexeme      *res = palloc(sizeof(TSLexeme) * 2);
 
+   memset(res, 0, sizeof(TSLexeme) * 2);
    if (*txt == '\0' || searchstoplist(&(d->stoplist), txt))
    {
        pfree(txt);
-       res[0] = NULL;
    }
    else
    {
@@ -122,10 +122,8 @@ snb_lexize(PG_FUNCTION_ARGS)
            memcpy(txt, d->z->p, d->z->l);
            txt[d->z->l] = '\0';
        }
-       res[0] = txt;
+       res->lexeme = txt;
    }
-   res[1] = NULL;
-
 
    PG_RETURN_POINTER(res);
 }

diff --git a/contrib/tsearch2/dict_syn.c b/contrib/tsearch2/dict_syn.c
index 046a59490394195501c35adf1178f6edaba7448a..6e3ed8643450892d7352f4d03aed90a2773ae503 100644 (file)
--- a/contrib/tsearch2/dict_syn.c
+++ b/contrib/tsearch2/dict_syn.c
@@ -162,7 +162,7 @@ syn_lexize(PG_FUNCTION_ARGS)
    char       *in = (char *) PG_GETARG_POINTER(1);
    Syn         key,
               *found;
-   char      **res = NULL;
+   TSLexeme      *res = NULL;
 
    if (!PG_GETARG_INT32(2))
        PG_RETURN_POINTER(NULL);
@@ -176,10 +176,9 @@ syn_lexize(PG_FUNCTION_ARGS)
    if (!found)
        PG_RETURN_POINTER(NULL);
 
-   res = palloc(sizeof(char *) * 2);
-
-   res[0] = pstrdup(found->out);
-   res[1] = NULL;
+   res = palloc(sizeof(TSLexeme) * 2);
+   memset(res,0,sizeof(TSLexeme) * 2);
+   res[0].lexeme = pstrdup(found->out);
 
    PG_RETURN_POINTER(res);
 }

diff --git a/contrib/tsearch2/gendict/dict_tmpl.c.IN b/contrib/tsearch2/gendict/dict_tmpl.c.IN
index deafdcead8a374c757387d7d88d06dd879c17eef..e534ed30a78ab5560d25cea199d687446ee16f8f 100644 (file)
--- a/contrib/tsearch2/gendict/dict_tmpl.c.IN
+++ b/contrib/tsearch2/gendict/dict_tmpl.c.IN
@@ -52,15 +52,15 @@ dlexize_CFG_MODNAME(PG_FUNCTION_ARGS) {
 HASINIT    DictExample *d = (DictExample*)PG_GETARG_POINTER(0);
    char       *in = (char*)PG_GETARG_POINTER(1);
    char *txt = pnstrdup(in, PG_GETARG_INT32(2));
-   char    **res=palloc(sizeof(char*)*2);
+   TSLexeme    *res=palloc(sizeof(TSLexeme*)*2);
 
-   /* Your INIT dictionary code */
+   /* Your LEXIZE dictionary code */
 HASINIT    if ( *txt=='\0' || searchstoplist(&(d->stoplist),txt) ) {
 HASINIT        pfree(txt);
-HASINIT        res[0]=NULL;
+HASINIT        res[0].lexeme=NULL;
 HASINIT    } else 
-       res[0]=txt;
-   res[1]=NULL;
+       res[0].lexeme=txt;
+   res[1].lexeme=NULL;
 
    PG_RETURN_POINTER(res);
 }

diff --git a/contrib/tsearch2/ispell/spell.c b/contrib/tsearch2/ispell/spell.c
index 54b01e8ed73b6bf9e727ada5b7d27607da0fdfa3..f9053c63011a087e95e22915afc1ba60bac9de0a 100644 (file)
--- a/contrib/tsearch2/ispell/spell.c
+++ b/contrib/tsearch2/ispell/spell.c
@@ -1119,17 +1119,32 @@ SplitToVariants(IspellDict * Conf, SPNode * snode, SplitVar * orig, char *word,
    return var;
 }
 
-char     **
+TSLexeme *
 NINormalizeWord(IspellDict * Conf, char *word)
 {
    char      **res = NormalizeSubWord(Conf, word, 0);
+   TSLexeme *lcur=NULL, *lres=NULL;
+   u_int16_t NVariant=1;
+
+   if (res) {
+       char **ptr = res;
+       lcur = lres = (TSLexeme*)palloc( MAX_NORM * sizeof(TSLexeme) );
+       while(*ptr) {
+           lcur->lexeme=*ptr;
+           lcur->flags=0;
+           lcur->nvariant = NVariant++;
+           lcur++;
+           ptr++;
+       }
+       lcur->lexeme=NULL;
+       pfree(res);
+   }
 
    if (Conf->compoundcontrol != '\t')
    {
        int         wordlen = strlen(word);
        SplitVar   *ptr,
                   *var = SplitToVariants(Conf, NULL, NULL, word, wordlen, 0, -1);
-       char      **cur = res;
        int         i;
 
        while (var)
@@ -1140,30 +1155,31 @@ NINormalizeWord(IspellDict * Conf, char *word)
 
                if (subres)
                {
-                   char      **ptr = subres;
+                   char      **subptr = subres;
+
+                   if ( !lcur )
+                       lcur = lres = (TSLexeme*)palloc( MAX_NORM * sizeof(TSLexeme) );
+       
+                   while(*subptr) {
+                       for(i=0;instem-1;i++) {
+                           lcur->lexeme=(subptr==subres) ? var->stem[ i ] : pstrdup(var->stem[ i ]);
+                           lcur->flags=0;
+                           lcur->nvariant = NVariant;
+                           lcur++;
+                       }
 
-                   if (cur)
-                   {
-                       while (*cur)
-                           cur++;
-                   }
-                   else
-                       res = cur = (char **) palloc(MAX_NORM * sizeof(char *));
+                       lcur->lexeme=*subptr;
+                       lcur->flags=0;
+                       lcur->nvariant = NVariant;
+                       lcur++;
+                       subptr++;
+                       NVariant++;
+                   }   
 
-                   for (i = 0; i < var->nstem - 1; i++)
-                   {
-                       *cur = var->stem[i];
-                       cur++;
-                   }
-                   while (*ptr)
-                   {
-                       *cur = *ptr;
-                       cur++;
-                       ptr++;
-                   }
-                   *cur = NULL;
+                   lcur->lexeme=NULL;
                    pfree(subres);
                    var->stem[0] = NULL;
+                   pfree( var->stem[ var->nstem-1 ] ); 
                }
            }
 
@@ -1175,7 +1191,7 @@ NINormalizeWord(IspellDict * Conf, char *word)
            var = ptr;
        }
    }
-   return res;
+   return lres;
 }
 
 

diff --git a/contrib/tsearch2/ispell/spell.h b/contrib/tsearch2/ispell/spell.h
index cc7935fd743f6f2d122ad829a0e97da0dec0d494..a3695113a18391c883aae5aec33cf5b06e528d4f 100644 (file)
--- a/contrib/tsearch2/ispell/spell.h
+++ b/contrib/tsearch2/ispell/spell.h
@@ -3,10 +3,11 @@
 
 #include 
 #include "regex/regex.h"
-#include "regis.h"
 #include "c.h"
 
-
+#include "regis.h"
+#include "dict.h"
+ 
 struct SPNode;
 
 
@@ -116,7 +117,7 @@ typedef struct
 
 }  IspellDict;
 
-char     **NINormalizeWord(IspellDict * Conf, char *word);
+TSLexeme     *NINormalizeWord(IspellDict * Conf, char *word);
 int            NIImportAffixes(IspellDict * Conf, const char *filename);
 int            NIImportDictionary(IspellDict * Conf, const char *filename);
 

diff --git a/contrib/tsearch2/query.c b/contrib/tsearch2/query.c
index 6787b63ae8638263e41eb4ee33c31066dfceeef2..ee4f779d58dd41dd2b76ed9b88f15fcb59c604bd 100644 (file)
--- a/contrib/tsearch2/query.c
+++ b/contrib/tsearch2/query.c
@@ -265,6 +265,7 @@ pushval_morph(QPRS_STATE * state, int typeval, char *strval, int lenval, int2 we
 {
    int4        count = 0;
    PRSTEXT     prs;
+   uint32      variant, pos, cntvar=0, cntpos=0, cnt=0;
 
    prs.lenwords = 32;
    prs.curwords = 0;
@@ -273,17 +274,39 @@ pushval_morph(QPRS_STATE * state, int typeval, char *strval, int lenval, int2 we
 
    parsetext_v2(findcfg(state->cfg_id), &prs, strval, lenval);
 
-   for (count = 0; count < prs.curwords; count++)
-   {
-       pushval_asis(state, VAL, prs.words[count].word, prs.words[count].len, weight);
-       pfree(prs.words[count].word);
-       if (count)
-           pushquery(state, OPR, (int4) '&', 0, 0, 0);
-   }
-   pfree(prs.words);
+   if ( prs.curwords>0 ) {
+
+       while (count < prs.curwords) {
+           pos = prs.words[count].pos.pos;
+           cntvar=0;
+           while(count < prs.curwords && pos==prs.words[count].pos.pos) {
+               variant = prs.words[count].nvariant;
+
+               cnt=0;
+               while(count < prs.curwords && pos==prs.words[count].pos.pos && variant==prs.words[count].nvariant)  {
+                   
+                   pushval_asis(state, VAL, prs.words[count].word, prs.words[count].len, weight);
+                   pfree(prs.words[count].word);
+                   if ( cnt ) 
+                       pushquery(state, OPR, (int4) '&', 0, 0, 0);
+                   cnt++;
+                   count++;
+               }
+
+               if ( cntvar ) 
+                   pushquery(state, OPR, (int4) '|', 0, 0, 0);
+               cntvar++;
+           }
+
+           if (cntpos) 
+               pushquery(state, OPR, (int4) '&', 0, 0, 0);
+       
+           cntpos++;
+       }
+
+       pfree(prs.words);
 
-   /* XXX */
-   if (prs.curwords == 0)
+   } else
        pushval_asis(state, VALSTOP, NULL, 0, 0);
 }
 

diff --git a/contrib/tsearch2/ts_cfg.c b/contrib/tsearch2/ts_cfg.c
index afebb11319963885595e3b7a364a9720169ee787..79f25c43d95c54d2e7d7ec84768047619f1ed787 100644 (file)
--- a/contrib/tsearch2/ts_cfg.c
+++ b/contrib/tsearch2/ts_cfg.c
@@ -321,10 +321,10 @@ parsetext_v2(TSCfgInfo * cfg, PRSTEXT * prs, char *buf, int4 buflen)
        for (i = 0; i < cfg->map[type].len; i++)
        {
            DictInfo   *dict = finddict(DatumGetObjectId(cfg->map[type].dict_id[i]));
-           char      **norms,
-                     **ptr;
+           TSLexeme      *norms,
+                     *ptr;
 
-           norms = ptr = (char **) DatumGetPointer(
+           norms = ptr = (TSLexeme *) DatumGetPointer(
                                                    FunctionCall3(
                                                    &(dict->lexize_info),
                                       PointerGetDatum(dict->dictionary),
@@ -337,7 +337,7 @@ parsetext_v2(TSCfgInfo * cfg, PRSTEXT * prs, char *buf, int4 buflen)
 
            prs->pos++;         /* set pos */
 
-           while (*ptr)
+           while (ptr->lexeme)
            {
                if (prs->curwords == prs->lenwords)
                {
@@ -345,8 +345,9 @@ parsetext_v2(TSCfgInfo * cfg, PRSTEXT * prs, char *buf, int4 buflen)
                    prs->words = (TSWORD *) repalloc((void *) prs->words, prs->lenwords * sizeof(TSWORD));
                }
 
-               prs->words[prs->curwords].len = strlen(*ptr);
-               prs->words[prs->curwords].word = *ptr;
+               prs->words[prs->curwords].len = strlen(ptr->lexeme);
+               prs->words[prs->curwords].word = ptr->lexeme;
+               prs->words[prs->curwords].nvariant = ptr->nvariant;
                prs->words[prs->curwords].alen = 0;
                prs->words[prs->curwords].pos.pos = LIMITPOS(prs->pos);
                ptr++;
@@ -458,10 +459,10 @@ hlparsetext(TSCfgInfo * cfg, HLPRSTEXT * prs, QUERYTYPE * query, char *buf, int4
        for (i = 0; i < cfg->map[type].len; i++)
        {
            DictInfo   *dict = finddict(DatumGetObjectId(cfg->map[type].dict_id[i]));
-           char      **norms,
-                     **ptr;
+           TSLexeme      *norms,
+                     *ptr;
 
-           norms = ptr = (char **) DatumGetPointer(
+           norms = ptr = (TSLexeme *) DatumGetPointer(
                                                    FunctionCall3(
                                                    &(dict->lexize_info),
                                       PointerGetDatum(dict->dictionary),
@@ -472,10 +473,10 @@ hlparsetext(TSCfgInfo * cfg, HLPRSTEXT * prs, QUERYTYPE * query, char *buf, int4
            if (!norms)         /* dictionary doesn't know this lexem */
                continue;
 
-           while (*ptr)
+           while (ptr->lexeme)
            {
-               hlfinditem(prs, query, *ptr, strlen(*ptr));
-               pfree(*ptr);
+               hlfinditem(prs, query, ptr->lexeme, strlen(ptr->lexeme));
+               pfree(ptr->lexeme);
                ptr++;
            }
            pfree(norms);

diff --git a/contrib/tsearch2/ts_cfg.h b/contrib/tsearch2/ts_cfg.h
index e000233178d1e556029ba7f8a46e336644765124..7bffdbcdd611c4168ffc3a62d5b93f403f2fc618 100644 (file)
--- a/contrib/tsearch2/ts_cfg.h
+++ b/contrib/tsearch2/ts_cfg.h
@@ -27,6 +27,7 @@ void      reset_cfg(void);
 typedef struct
 {
    uint16      len;
+   uint16      nvariant;
    union
    {
        uint16      pos;