+ StopLow = node->data;
+ StopHigh = node->data+node->length;
+ while (StopLow < StopHigh) {
+ StopMiddle = StopLow + (StopHigh - StopLow) / 2;
+ symbol = GETWCHAR(word,wrdlen,*level,type);
+ if ( StopMiddle->val == symbol ) {
+ if ( StopMiddle->naff )
+ return StopMiddle;
+ node=StopMiddle->node;
+ (*level)++;
+ break;
+ } else if ( StopMiddle->val < symbol ) {
+ StopLow = StopMiddle + 1;
+ } else {
+ StopHigh = StopMiddle;
}
- Conf->SuffixTree.Right[Let] = i;
}
+ if ( StopLow >= StopHigh )
+ break;
}
+ return NULL;
}
static char *
-CheckSuffix(const char *word, size_t len, AFFIX * Affix, int *res, IspellDict * Conf)
-{
+CheckAffix(const char *word, size_t len, AFFIX * Affix, char flagflags, char *newword) {
regmatch_t subs[2]; /* workaround for apache&linux */
- char newword[2 * MAXNORMLEN] = "";
int err;
- *res = strbncmp(word, Affix->repl, Affix->replen);
- if (*res < 0)
- return NULL;
- if (*res > 0)
- return NULL;
- strcpy(newword, word);
- strcpy(newword + len - Affix->replen, Affix->find);
+ if ( flagflags & FF_COMPOUNDONLYAFX ) {
+ if ( (Affix->flagflags & FF_COMPOUNDONLYAFX) == 0 )
+ return NULL;
+ } else {
+ if ( Affix->flagflags & FF_COMPOUNDONLYAFX )
+ return NULL;
+ }
+
+ if ( Affix->type=='s' ) {
+ strcpy(newword, word);
+ strcpy(newword + len - Affix->replen, Affix->find);
+ } else {
+ strcpy(newword, Affix->find);
+ strcat(newword, word + Affix->replen);
+ }
if (Affix->compile)
{
}
Affix->compile = 0;
}
- if (!(err = regexec(&(Affix->reg), newword, 1, subs, 0)))
- {
- if (FindWord(Conf, newword, Affix->flag))
- return pstrdup(newword);
- }
+ if (!(err = regexec(&(Affix->reg), newword, 1, subs, 0)))
+ return newword;
return NULL;
}
-#define NS 1
-#define MAX_NORM 512
-static int
-CheckPrefix(const char *word, size_t len, AFFIX * Affix, IspellDict * Conf, int pi,
- char **forms, char ***cur)
-{
- regmatch_t subs[NS * 2];
+
+static char **
+NormalizeSubWord(IspellDict * Conf, char *word, char flag) {
+ AffixNodeData *suffix=NULL, *prefix=NULL;
+ int slevel=0, plevel=0;
+ int wrdlen = strlen(word), swrdlen;
+ char **forms;
+ char **cur;
char newword[2 * MAXNORMLEN] = "";
- int err,
- ls,
- res,
- lres;
- size_t newlen;
- AFFIX *CAffix = Conf->Affix;
-
- res = strncmp(word, Affix->repl, Affix->replen);
- if (res != 0)
- return res;
- strcpy(newword, Affix->find);
- strcat(newword, word + Affix->replen);
+ char pnewword[2 * MAXNORMLEN] = "";
+ AffixNode *snode = Conf->Suffix, *pnode;
+ int i,j;
- if (Affix->compile)
- {
- err = regcomp(&(Affix->reg), Affix->mask, REG_EXTENDED | REG_ICASE | REG_NOSUB);
- if (err)
- {
- /* regerror(err, &(Affix->reg), regerrstr, ERRSTRSIZE); */
- regfree(&(Affix->reg));
- return (0);
- }
- Affix->compile = 0;
+ if (wrdlen > MAXNORMLEN) return NULL;
+ strlower(word);
+ cur = forms = (char **) palloc(MAX_NORM * sizeof(char *));
+ *cur = NULL;
+
+
+ /* Check that the word itself is normal form */
+ if (FindWord(Conf, word, 0, flag & FF_COMPOUNDWORD)) {
+ *cur = pstrdup(word);
+ cur++;
+ *cur = NULL;
}
- if (!(err = regexec(&(Affix->reg), newword, 1, subs, 0)))
- {
- SPELL *curspell;
- if ((curspell = FindWord(Conf, newword, Affix->flag)))
- {
- if ((*cur - forms) < (MAX_NORM - 1))
- {
- **cur = pstrdup(newword);
- (*cur)++;
- **cur = NULL;
+ /* Find all other NORMAL forms of the 'word' (check only prefix)*/
+ pnode=Conf->Prefix;
+ plevel=0;
+ while(pnode) {
+ prefix=FinfAffixes(pnode, word, wrdlen, &plevel,'p');
+ if (!prefix) break;
+ if ( CheckAffix(word,wrdlen,prefix->aff[j], flag, newword) ) {
+ /* prefix success */
+ if ( FindWord(Conf, newword, prefix->aff[j]->flag, flag&FF_COMPOUNDWORD) && (cur - forms) < (MAX_NORM-1) ) {
+ /* word search success */
+ *cur = pstrdup(newword);
+ cur++;
+ *cur=NULL;
+ }
}
}
- newlen = strlen(newword);
- ls = Conf->SuffixTree.Left[pi];
- if (ls >= 0 && ((*cur - forms) < (MAX_NORM - 1)))
- {
- **cur = CheckSuffix(newword, newlen, &CAffix[ls], &lres, Conf);
- if (**cur)
- {
- (*cur)++;
- **cur = NULL;
+ pnode = prefix->node;
+ plevel++;
+ }
+
+ /* Find all other NORMAL forms of the 'word' (check suffix and then prefix)*/
+ while( snode ) {
+ /* find possible suffix */
+ suffix = FinfAffixes(snode, word, wrdlen, &slevel, 's');
+ if (!suffix) break;
+ /* foreach suffix check affix */
+ for(i=0;inaff;i++) {
+ if ( CheckAffix(word, wrdlen, suffix->aff[i], flag, newword) ) {
+ /* suffix success */
+ if ( FindWord(Conf, newword, suffix->aff[i]->flag, flag&FF_COMPOUNDWORD) && (cur - forms) < (MAX_NORM-1) ) {
+ /* word search success */
+ *cur = pstrdup(newword);
+ cur++;
+ *cur=NULL;
+ }
+ /* now we will look changed word with prefixes */
+ pnode=Conf->Prefix;
+ plevel=0;
+ swrdlen=strlen(newword);
+ while(pnode) {
+ prefix=FinfAffixes(pnode, newword, swrdlen, &plevel,'p');
+ if (!prefix) break;
+ if ( CheckAffix(newword,swrdlen,prefix->aff[j], flag, pnewword) ) {
+ /* prefix success */
+ int ff=( prefix->aff[j]->flagflags & suffix->aff[i]->flagflags & FF_CROSSPRODUCT ) ?
+ 0 : prefix->aff[j]->flag;
+ if ( FindWord(Conf, pnewword, ff, flag&FF_COMPOUNDWORD) && (cur - forms) < (MAX_NORM-1) ) {
+ /* word search success */
+ *cur = pstrdup(pnewword);
+ cur++;
+ *cur=NULL;
+ }
+ }
+ }
+ pnode = prefix->node;
+ plevel++;
+ }
}
}
- }
- return 0;
-}
+ snode=suffix->node;
+ slevel++;
+ }
-char **
-NormalizeWord(IspellDict * Conf, char *word)
-{
-/*regmatch_t subs[NS];*/
- size_t len;
- char **forms;
- char **cur;
- AFFIX *Affix;
- int ri,
- pi,
- ipi,
- lp,
- rp,
- cp,
- ls,
- rs;
- int lres,
- rres,
- cres = 0;
- SPELL *spell;
-
- len = strlen(word);
- if (len > MAXNORMLEN)
+ if (cur == forms) {
+ free(forms);
return (NULL);
+ }
+ return (forms);
+}
- strlower(word);
-
- forms = (char **) palloc(MAX_NORM * sizeof(char **));
- cur = forms;
- *cur = NULL;
-
- ri = (int) (*word) & 255;
- pi = (int) (word[strlen(word) - 1]) & 255;
- Affix = (AFFIX *) Conf->Affix;
-
- /* Check that the word itself is normal form */
- if ((spell = FindWord(Conf, word, 0)))
- {
- *cur = pstrdup(word);
- cur++;
- *cur = NULL;
+typedef struct SplitVar {
+ int nstem;
+ char **stem;
+ struct SplitVar *next;
+} SplitVar;
+
+static int
+CheckCompoundAffixes(CMPDAffix **ptr, char *word, int len) {
+ while( (*ptr)->affix ) {
+ if ( len > (*ptr)->len && strncmp((*ptr)->affix, word, (*ptr)->len)==0 ) {
+ len = (*ptr)->len;
+ (*ptr)++;
+ return len;
+ }
+ (*ptr)++;
}
+ return 0;
+}
- /* Find all other NORMAL forms of the 'word' */
+static SplitVar*
+CopyVar(SplitVar *s, int makedup) {
+ SplitVar *v = (SplitVar*)palloc(sizeof(SplitVar));
+
+ v->stem=(char**)palloc( sizeof(char*) * (MAX_NORM) );
+ v->next=NULL;
+ if ( s ) {
+ int i;
+ v->nstem = s->nstem;
+ for(i=0;instem;i++)
+ v->stem[i] = (makedup) ? pstrdup( s->stem[i] ) : s->stem[i];
+ } else {
+ v->nstem=0;
+ }
+ return v;
+}
- for (ipi = 0; ipi <= pi; ipi += pi)
- {
- /* check prefix */
- lp = Conf->PrefixTree.Left[ri];
- rp = Conf->PrefixTree.Right[ri];
- while (lp >= 0 && lp <= rp)
- {
- cp = (lp + rp) >> 1;
- cres = 0;
- if ((cur - forms) < (MAX_NORM - 1))
- cres = CheckPrefix(word, len, &Affix[cp], Conf, ipi, forms, &cur);
- if ((lp < cp) && ((cur - forms) < (MAX_NORM - 1)))
- lres = CheckPrefix(word, len, &Affix[lp], Conf, ipi, forms, &cur);
- if ((rp > cp) && ((cur - forms) < (MAX_NORM - 1)))
- rres = CheckPrefix(word, len, &Affix[rp], Conf, ipi, forms, &cur);
- if (cres < 0)
- {
- rp = cp - 1;
- lp++;
- }
- else if (cres > 0)
- {
- lp = cp + 1;
- rp--;
+static SplitVar*
+SplitToVariants( IspellDict * Conf, SPNode *snode, SplitVar * orig, char *word, int wordlen, int startpos, int minpos ) {
+ SplitVar *var=NULL;
+ SPNodeData *StopLow, *StopHigh, *StopMiddle;
+ SPNode *node = (snode) ? snode : Conf->Dictionary;
+ int level=(snode) ? minpos : startpos; /* recursive minpos==level*/
+ int lenaff;
+ CMPDAffix *caff;
+ char notprobed[wordlen];
+
+ memset(notprobed,1,wordlen);
+ var = CopyVar(orig,1);
+
+ while( node && level
+ StopLow = node->data;
+ StopHigh = node->data+node->length;
+ while (StopLow < StopHigh) {
+ StopMiddle = StopLow + (StopHigh - StopLow) / 2;
+ if ( StopMiddle->val == ((u_int8_t*)(word))[level] ) {
+ break;
+ } else if ( StopMiddle->val < ((u_int8_t*)(word))[level] ) {
+ StopLow = StopMiddle + 1;
+ } else {
+ StopHigh = StopMiddle;
}
- else
- {
- lp++;
- rp--;
+ }
+ if ( StopLow >= StopHigh )
+ break;
+
+ /* find word with epenthetic */
+ caff = Conf->CompoundAffix;
+ while ( level>startpos && (lenaff=CheckCompoundAffixes( &caff, word + level, wordlen - level ))>0 ) {
+ /* there is one of compound suffixes, so check word for existings */
+ char buf[MAXNORMLEN];
+ char **subres;
+
+ lenaff=level-startpos+lenaff;
+
+ if ( !notprobed[startpos+lenaff-1] )
+ continue;
+
+ if ( level+lenaff-1 <= minpos )
+ continue;
+
+ memcpy(buf, word+startpos, lenaff);
+ buf[lenaff]='\0';
+
+ subres = NormalizeSubWord(Conf, buf, FF_COMPOUNDWORD | FF_COMPOUNDONLYAFX);
+ if ( subres ) {
+ /* Yes, it was a word from dictionary */
+ SplitVar *new=CopyVar(var,0);
+ SplitVar *ptr=var;
+ char **sptr=subres;
+
+ notprobed[startpos+lenaff-1]=0;
+
+ while(*sptr) {
+ new->stem[ new->nstem ] = *sptr;
+ new->nstem++;
+ sptr++;
+ }
+ free(subres);
+
+ while( ptr->next )
+ ptr = ptr->next;
+ ptr->next = SplitToVariants(Conf, NULL, new, word, wordlen, startpos+lenaff, startpos+lenaff);
+
+ free(new->stem);
+ free(new);
}
}
- /* check suffix */
- ls = Conf->SuffixTree.Left[ipi];
- rs = Conf->SuffixTree.Right[ipi];
- while (ls >= 0 && ls <= rs)
- {
- if (((cur - forms) < (MAX_NORM - 1)))
- {
- *cur = CheckSuffix(word, len, &Affix[ls], &lres, Conf);
- if (*cur)
- {
- cur++;
- *cur = NULL;
+ /* find infinitive */
+ if ( StopMiddle->isword && StopMiddle->compoundallow && notprobed[level] ) {
+ /* ok, we found full compoundallowed word*/
+ if ( level>minpos ) {
+ /* and its length more than minimal */
+ if ( wordlen==level+1 ) {
+ /* well, it was last word */
+ var->stem[ var->nstem ] = strndup(word + startpos, wordlen - startpos);
+ var->nstem++;
+ return var;
+ } else {
+ /* then we will search more big word at the same point */
+ SplitVar *ptr=var;
+ while( ptr->next )
+ ptr = ptr->next;
+ ptr->next=SplitToVariants(Conf, node, var, word, wordlen, startpos, level);
+ /* we can find next word */
+ level++;
+ var->stem[ var->nstem ] = strndup(word + startpos, level - startpos);
+ var->nstem++;
+ node = Conf->Dictionary;
+ startpos=level;
+ continue;
}
}
- if ((rs > ls) && ((cur - forms) < (MAX_NORM - 1)))
- {
- *cur = CheckSuffix(word, len, &Affix[rs], &rres, Conf);
- if (*cur)
- {
- cur++;
- *cur = NULL;
+ }
+ level++;
+ node=StopMiddle->node;
+ }
+
+ var->stem[ var->nstem ] = strndup(word + startpos, wordlen - startpos);
+ var->nstem++;
+ return var;
+}
+
+char **
+NINormalizeWord(IspellDict * Conf, char *word) {
+ char **res= NormalizeSubWord(Conf, word, 0);
+
+ if ( Conf->compoundcontrol != '\t' ) {
+ int wordlen=strlen(word);
+ SplitVar *ptr, *var = SplitToVariants(Conf,NULL,NULL, word, wordlen, 0, -1);
+ char **cur=res;
+ int i;
+
+ while(var) {
+ if ( var->nstem > 1 ) {
+ char **subres = NormalizeSubWord(Conf, var->stem[ var->nstem-1 ], FF_COMPOUNDWORD);
+ if ( subres ) {
+ char **ptr=subres;
+
+ if ( cur ) {
+ while(*cur)
+ cur++;
+ } else {
+ res=cur=(char **) palloc(MAX_NORM * sizeof(char *));
+ }
+
+ for(i=0;instem-1;i++) {
+ *cur=var->stem[ i ];
+ cur++;
+ }
+ while(*ptr) {
+ *cur=*ptr;
+ cur++; ptr++;
+ }
+ *cur=NULL;
+ free(subres);
+ var->stem[ 0 ] = NULL;
}
}
- ls++;
- rs--;
- } /* end while */
+
+ for(i=0;instem && var->stem[ i ];i++)
+ free( var->stem[i] );
+ ptr = var->next;
+ free(var->stem);
+ free(var);
+ var=ptr;
+ }
+ }
+ return res;
+}
- } /* for ipi */
- if (cur == forms)
- {
- pfree(forms);
- return (NULL);
+static void freeSPNode(SPNode *node) {
+ SPNodeData *data;
+
+ if (!node) return;
+ data=node->data;
+ while( node->length ) {
+ freeSPNode(data->node);
+ data++;
+ node->length--;
}
- return (forms);
+ free(node);
}
+
+static void freeANode(AffixNode *node) {
+ AffixNodeData *data;
+
+ if (!node) return;
+ data=node->data;
+ while( node->length ) {
+ freeANode(data->node);
+ if (data->naff)
+ free(data->aff);
+ data++;
+ node->length--;
+ }
+ free(node);
+}
+
void
-FreeIspell(IspellDict * Conf)
+NIFree(IspellDict * Conf)
{
int i;
AFFIX *Affix = (AFFIX *) Conf->Affix;
+ char** aff = Conf->AffixData;
+
+ if ( aff ) {
+ while(*aff) {
+ free(*aff);
+ aff++;
+ }
+ free(Conf->AffixData);
+ }
+
for (i = 0; i < Conf->naffixes; i++)
{
if (Affix[i].compile == 0)
regfree(&(Affix[i].reg));
}
- for (i = 0; i < Conf->naffixes; i++)
- free(Conf->Spell[i].word);
- free(Conf->Affix);
- free(Conf->Spell);
+ if (Conf->Spell) {
+ for (i = 0; i < Conf->nspell; i++)
+ free(Conf->Spell[i].word);
+ free(Conf->Spell);
+ }
+
+ if (Conf->Affix) free(Conf->Affix);
+ if ( Conf->CompoundAffix ) free(Conf->CompoundAffix);
+ freeSPNode(Conf->Dictionary);
+ freeANode(Conf->Suffix);
+ freeANode(Conf->Prefix);
memset((void *) Conf, 0, sizeof(IspellDict));
return;
}
#include
#include
+
+struct SPNode;
+
+
+typedef struct {
+ u_int32_t
+ val:8,
+ isword:1,
+ compoundallow:1,
+ affix:22;
+ struct SPNode *node;
+} SPNodeData;
+
+typedef struct SPNode {
+ u_int32_t length;
+ SPNodeData data[1];
+} SPNode;
+
+#define SPNHRDSZ (sizeof(u_int32_t))
+
+
typedef struct spell_struct
{
char *word;
- char flag[10];
+ union {
+ char flag[16];
+ struct {
+ int affix;
+ int len;
+ } d;
+ } p;
} SPELL;
typedef struct aff_struct
{
char flag;
+ char flagflags;
char type;
char mask[33];
char find[16];
char compile;
} AFFIX;
+#define FF_CROSSPRODUCT 0x01
+#define FF_COMPOUNDWORD 0x02
+#define FF_COMPOUNDONLYAFX 0x04
+
+struct AffixNode;
+
+typedef struct {
+ u_int32_t
+ val:8,
+ naff:24;
+ AFFIX **aff;
+ struct AffixNode *node;
+} AffixNodeData;
+
+typedef struct AffixNode {
+ u_int32_t length;
+ AffixNodeData data[1];
+} AffixNode;
+
+#define ANHRDSZ (sizeof(u_int32_t))
+
typedef struct Tree_struct
{
int Left[256],
Right[256];
} Tree_struct;
+typedef struct {
+ char *affix;
+ int len;
+} CMPDAffix;
+
typedef struct
{
int maffixes;
int naffixes;
AFFIX *Affix;
+ char compoundcontrol;
int nspell;
int mspell;
SPELL *Spell;
- Tree_struct SpellTree;
- Tree_struct PrefixTree;
- Tree_struct SuffixTree;
+
+ AffixNode *Suffix;
+ AffixNode *Prefix;
+
+ SPNode *Dictionary;
+ char **AffixData;
+ CMPDAffix *CompoundAffix;
} IspellDict;
-char **NormalizeWord(IspellDict * Conf, char *word);
-int ImportAffixes(IspellDict * Conf, const char *filename);
-int ImportDictionary(IspellDict * Conf, const char *filename);
+char **NINormalizeWord(IspellDict * Conf, char *word);
+int NIImportAffixes(IspellDict * Conf, const char *filename);
+int NIImportDictionary(IspellDict * Conf, const char *filename);
-int AddSpell(IspellDict * Conf, const char *word, const char *flag);
-int AddAffix(IspellDict * Conf, int flag, const char *mask, const char *find, const char *repl, int type);
-void SortDictionary(IspellDict * Conf);
-void SortAffixes(IspellDict * Conf);
-void FreeIspell(IspellDict * Conf);
+int NIAddSpell(IspellDict * Conf, const char *word, const char *flag);
+int NIAddAffix(IspellDict * Conf, int flag, char flagflags, const char *mask, const char *find, const char *repl, int type);
+void NISortDictionary(IspellDict * Conf);
+void NISortAffixes(IspellDict * Conf);
+void NIFree(IspellDict * Conf);
#endif
projects / postgresql.git / commitdiff