1
/* ***** BEGIN LICENSE BLOCK *****
2
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
4
* The contents of this file are subject to the Mozilla Public License Version
5
* 1.1 (the "License"); you may not use this file except in compliance with
6
* the License. You may obtain a copy of the License at
7
* http://www.mozilla.org/MPL/
9
* Software distributed under the License is distributed on an "AS IS" basis,
10
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
11
* for the specific language governing rights and limitations under the
14
* The Original Code is Hunspell, based on MySpell.
16
* The Initial Developers of the Original Code are
17
* Kevin Hendricks (MySpell) and Németh László (Hunspell).
18
* Portions created by the Initial Developers are Copyright (C) 2002-2005
19
* the Initial Developers. All Rights Reserved.
21
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
22
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
23
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
24
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
25
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
27
* Alternatively, the contents of this file may be used under the terms of
28
* either the GNU General Public License Version 2 or later (the "GPL"), or
29
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
30
* in which case the provisions of the GPL or the LGPL are applicable instead
31
* of those above. If you wish to allow use of your version of this file only
32
* under the terms of either the GPL or the LGPL, and not to allow others to
33
* use your version of this file under the terms of the MPL, indicate your
34
* decision by deleting the provisions above and replace them with the notice
35
* and other provisions required by the GPL or the LGPL. If you do not delete
36
* the provisions above, a recipient may use your version of this file under
37
* the terms of any one of the MPL, the GPL or the LGPL.
39
* ***** END LICENSE BLOCK ***** */
41
* Copyright 2002 Kevin B. Hendricks, Stratford, Ontario, Canada
42
* And Contributors. All rights reserved.
44
* Redistribution and use in source and binary forms, with or without
45
* modification, are permitted provided that the following conditions
48
* 1. Redistributions of source code must retain the above copyright
49
* notice, this list of conditions and the following disclaimer.
51
* 2. Redistributions in binary form must reproduce the above copyright
52
* notice, this list of conditions and the following disclaimer in the
53
* documentation and/or other materials provided with the distribution.
55
* 3. All modifications to the source code must be clearly marked as
56
* such. Binary redistributions based on modified source code
57
* must be clearly marked as modified versions in the documentation
58
* and/or other materials provided with the distribution.
60
* THIS SOFTWARE IS PROVIDED BY KEVIN B. HENDRICKS AND CONTRIBUTORS
61
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
62
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
63
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
64
* KEVIN B. HENDRICKS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
65
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
66
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
67
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
68
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
69
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
70
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
84
#include "affixmgr.hxx"
85
#include "affentry.hxx"
86
#include "langnum.hxx"
90
AffixMgr::AffixMgr(const char * affpath, HashMgr** ptr, int * md, const char * key)
92
// register hash manager and load affix data from aff file
110
checkcpdtable = NULL;
111
// allow simplified compound forms (see 3rd field of CHECKCOMPOUNDPATTERN)
117
compoundflag = FLAG_NULL; // permits word in compound forms
118
compoundbegin = FLAG_NULL; // may be first word in compound forms
119
compoundmiddle = FLAG_NULL; // may be middle word in compound forms
120
compoundend = FLAG_NULL; // may be last word in compound forms
121
compoundroot = FLAG_NULL; // compound word signing flag
122
compoundpermitflag = FLAG_NULL; // compound permitting flag for suffixed word
123
compoundforbidflag = FLAG_NULL; // compound fordidden flag for suffixed word
124
compoundmoresuffixes = 0; // allow more suffixes within compound words
125
checkcompounddup = 0; // forbid double words in compounds
126
checkcompoundrep = 0; // forbid bad compounds (may be non compound word with a REP substitution)
127
checkcompoundcase = 0; // forbid upper and lowercase combinations at word bounds
128
checkcompoundtriple = 0; // forbid compounds with triple letters
129
simplifiedtriple = 0; // allow simplified triple letters in compounds (Schiff+fahrt -> Schiffahrt)
130
forbiddenword = FORBIDDENWORD; // forbidden word signing flag
131
nosuggest = FLAG_NULL; // don't suggest words signed with NOSUGGEST flag
132
nongramsuggest = FLAG_NULL;
133
lang = NULL; // language
134
langnum = 0; // language code (see http://l10n.openoffice.org/languages.html)
135
needaffix = FLAG_NULL; // forbidden root, allowed only with suffixes
136
cpdwordmax = -1; // default: unlimited wordcount in compound words
137
cpdmin = -1; // undefined
138
cpdmaxsyllable = 0; // default: unlimited syllablecount in compound words
139
cpdvowels=NULL; // vowels (for calculating of Hungarian compounding limit, O(n) search! XXX)
140
cpdvowels_utf16=NULL; // vowels for UTF-8 encoding (bsearch instead of O(n) search)
141
cpdvowels_utf16_len=0; // vowels
142
pfxappnd=NULL; // previous prefix for counting the syllables of prefix BUG
143
sfxappnd=NULL; // previous suffix for counting a special syllables BUG
144
cpdsyllablenum=NULL; // syllable count incrementing flag
145
checknum=0; // checking numbers, and word with numbers
146
wordchars=NULL; // letters + spec. word characters
147
wordchars_utf16=NULL; // letters + spec. word characters
148
wordchars_utf16_len=0; // letters + spec. word characters
149
ignorechars=NULL; // letters + spec. word characters
150
ignorechars_utf16=NULL; // letters + spec. word characters
151
ignorechars_utf16_len=0; // letters + spec. word characters
152
version=NULL; // affix and dictionary file version string
153
havecontclass=0; // flags of possible continuing classes (double affix)
154
// LEMMA_PRESENT: not put root into the morphological output. Lemma presents
155
// in morhological description in dictionary file. It's often combined with PSEUDOROOT.
156
lemma_present = FLAG_NULL;
157
circumfix = FLAG_NULL;
158
onlyincompound = FLAG_NULL;
159
maxngramsugs = -1; // undefined
160
maxdiff = -1; // undefined
162
maxcpdsugs = -1; // undefined
170
substandard = FLAG_NULL;
176
for (int i=0; i < SETSIZE; i++) {
183
for (int j=0; j < CONTSIZE; j++) {
187
if (parse_file(affpath, key)) {
188
HUNSPELL_WARNING(stderr, "Failure loading aff file %s\n",affpath);
191
if (cpdmin == -1) cpdmin = MINCPDLEN;
196
AffixMgr::~AffixMgr()
198
// pass through linked prefix entries and clean up
199
for (int i=0; i < SETSIZE ;i++) {
201
PfxEntry * ptr = pStart[i];
202
PfxEntry * nptr = NULL;
204
nptr = ptr->getNext();
211
// pass through linked suffix entries and clean up
212
for (int j=0; j < SETSIZE ; j++) {
214
SfxEntry * ptr = sStart[j];
215
SfxEntry * nptr = NULL;
217
nptr = ptr->getNext();
225
if (keystring) free(keystring);
227
if (trystring) free(trystring);
229
if (encoding) free(encoding);
232
for (int j=0; j < nummap; j++) {
233
for (int k=0; k < maptable[j].len; k++) {
234
if (maptable[j].set[k]) free(maptable[j].set[k]);
236
free(maptable[j].set);
237
maptable[j].set = NULL;
245
for (int j=0; j < numbreak; j++) {
246
if (breaktable[j]) free(breaktable[j]);
247
breaktable[j] = NULL;
254
for (int j=0; j < numrep; j++) {
255
free(reptable[j].pattern);
256
free(reptable[j].pattern2);
261
if (iconvtable) delete iconvtable;
262
if (oconvtable) delete oconvtable;
263
if (phone && phone->rules) {
264
for (int j=0; j < phone->num + 1; j++) {
265
free(phone->rules[j * 2]);
266
free(phone->rules[j * 2 + 1]);
274
for (int j=0; j < numdefcpd; j++) {
275
free(defcpdtable[j].def);
276
defcpdtable[j].def = NULL;
283
for (int j=0; j < numcheckcpd; j++) {
284
free(checkcpdtable[j].pattern);
285
free(checkcpdtable[j].pattern2);
286
free(checkcpdtable[j].pattern3);
287
checkcpdtable[j].pattern = NULL;
288
checkcpdtable[j].pattern2 = NULL;
289
checkcpdtable[j].pattern3 = NULL;
292
checkcpdtable = NULL;
295
FREE_FLAG(compoundflag);
296
FREE_FLAG(compoundbegin);
297
FREE_FLAG(compoundmiddle);
298
FREE_FLAG(compoundend);
299
FREE_FLAG(compoundpermitflag);
300
FREE_FLAG(compoundforbidflag);
301
FREE_FLAG(compoundroot);
302
FREE_FLAG(forbiddenword);
303
FREE_FLAG(nosuggest);
304
FREE_FLAG(nongramsuggest);
305
FREE_FLAG(needaffix);
306
FREE_FLAG(lemma_present);
307
FREE_FLAG(circumfix);
308
FREE_FLAG(onlyincompound);
314
if (cpdvowels) free(cpdvowels);
315
if (cpdvowels_utf16) free(cpdvowels_utf16);
316
if (cpdsyllablenum) free(cpdsyllablenum);
318
if (lang) free(lang);
319
if (wordchars) free(wordchars);
320
if (wordchars_utf16) free(wordchars_utf16);
321
if (ignorechars) free(ignorechars);
322
if (ignorechars_utf16) free(ignorechars_utf16);
323
if (version) free(version);
325
#ifdef MOZILLA_CLIENT
330
void AffixMgr::finishFileMgr(FileMgr *afflst)
334
// convert affix trees to sorted list
335
process_pfx_tree_to_list();
336
process_sfx_tree_to_list();
339
// read in aff file and build up prefix and suffix entry objects
340
int AffixMgr::parse_file(const char * affpath, const char * key)
342
char * line; // io buffers
343
char ft; // affix type
345
// checking flag duplication
346
char dupflags[CONTSIZE];
347
char dupflags_ini = 1;
349
// first line indicator for removing byte order mark
352
// open the affix file
353
FileMgr * afflst = new FileMgr(affpath, key);
355
HUNSPELL_WARNING(stderr, "error: could not open affix description file %s\n",affpath);
359
// step one is to parse the affix file building up the internal
360
// affix data structures
362
// read in each line ignoring any that do not
363
// start with a known line type indicator
364
while ((line = afflst->getline()) != NULL) {
367
/* remove byte order mark */
370
// Affix file begins with byte order mark: possible incompatibility with old Hunspell versions
371
if (strncmp(line,"\xEF\xBB\xBF",3) == 0) {
372
memmove(line, line+3, strlen(line+3)+1);
376
/* parse in the keyboard string */
377
if (strncmp(line,"KEY",3) == 0) {
378
if (parse_string(line, &keystring, afflst->getlinenum())) {
379
finishFileMgr(afflst);
384
/* parse in the try string */
385
if (strncmp(line,"TRY",3) == 0) {
386
if (parse_string(line, &trystring, afflst->getlinenum())) {
387
finishFileMgr(afflst);
392
/* parse in the name of the character set used by the .dict and .aff */
393
if (strncmp(line,"SET",3) == 0) {
394
if (parse_string(line, &encoding, afflst->getlinenum())) {
395
finishFileMgr(afflst);
398
if (strcmp(encoding, "UTF-8") == 0) {
400
#ifndef OPENOFFICEORG
401
#ifndef MOZILLA_CLIENT
402
if (initialize_utf_tbl()) {
403
finishFileMgr(afflst);
411
/* parse COMPLEXPREFIXES for agglutinative languages with right-to-left writing system */
412
if (strncmp(line,"COMPLEXPREFIXES",15) == 0)
415
/* parse in the flag used by the controlled compound words */
416
if (strncmp(line,"COMPOUNDFLAG",12) == 0) {
417
if (parse_flag(line, &compoundflag, afflst)) {
418
finishFileMgr(afflst);
423
/* parse in the flag used by compound words */
424
if (strncmp(line,"COMPOUNDBEGIN",13) == 0) {
425
if (complexprefixes) {
426
if (parse_flag(line, &compoundend, afflst)) {
427
finishFileMgr(afflst);
431
if (parse_flag(line, &compoundbegin, afflst)) {
432
finishFileMgr(afflst);
438
/* parse in the flag used by compound words */
439
if (strncmp(line,"COMPOUNDMIDDLE",14) == 0) {
440
if (parse_flag(line, &compoundmiddle, afflst)) {
441
finishFileMgr(afflst);
445
/* parse in the flag used by compound words */
446
if (strncmp(line,"COMPOUNDEND",11) == 0) {
447
if (complexprefixes) {
448
if (parse_flag(line, &compoundbegin, afflst)) {
449
finishFileMgr(afflst);
453
if (parse_flag(line, &compoundend, afflst)) {
454
finishFileMgr(afflst);
460
/* parse in the data used by compound_check() method */
461
if (strncmp(line,"COMPOUNDWORDMAX",15) == 0) {
462
if (parse_num(line, &cpdwordmax, afflst)) {
463
finishFileMgr(afflst);
468
/* parse in the flag sign compounds in dictionary */
469
if (strncmp(line,"COMPOUNDROOT",12) == 0) {
470
if (parse_flag(line, &compoundroot, afflst)) {
471
finishFileMgr(afflst);
476
/* parse in the flag used by compound_check() method */
477
if (strncmp(line,"COMPOUNDPERMITFLAG",18) == 0) {
478
if (parse_flag(line, &compoundpermitflag, afflst)) {
479
finishFileMgr(afflst);
484
/* parse in the flag used by compound_check() method */
485
if (strncmp(line,"COMPOUNDFORBIDFLAG",18) == 0) {
486
if (parse_flag(line, &compoundforbidflag, afflst)) {
487
finishFileMgr(afflst);
492
if (strncmp(line,"COMPOUNDMORESUFFIXES",20) == 0) {
493
compoundmoresuffixes = 1;
496
if (strncmp(line,"CHECKCOMPOUNDDUP",16) == 0) {
497
checkcompounddup = 1;
500
if (strncmp(line,"CHECKCOMPOUNDREP",16) == 0) {
501
checkcompoundrep = 1;
504
if (strncmp(line,"CHECKCOMPOUNDTRIPLE",19) == 0) {
505
checkcompoundtriple = 1;
508
if (strncmp(line,"SIMPLIFIEDTRIPLE",16) == 0) {
509
simplifiedtriple = 1;
512
if (strncmp(line,"CHECKCOMPOUNDCASE",17) == 0) {
513
checkcompoundcase = 1;
516
if (strncmp(line,"NOSUGGEST",9) == 0) {
517
if (parse_flag(line, &nosuggest, afflst)) {
518
finishFileMgr(afflst);
523
if (strncmp(line,"NONGRAMSUGGEST",14) == 0) {
524
if (parse_flag(line, &nongramsuggest, afflst)) {
525
finishFileMgr(afflst);
530
/* parse in the flag used by forbidden words */
531
if (strncmp(line,"FORBIDDENWORD",13) == 0) {
532
if (parse_flag(line, &forbiddenword, afflst)) {
533
finishFileMgr(afflst);
538
/* parse in the flag used by forbidden words */
539
if (strncmp(line,"LEMMA_PRESENT",13) == 0) {
540
if (parse_flag(line, &lemma_present, afflst)) {
541
finishFileMgr(afflst);
546
/* parse in the flag used by circumfixes */
547
if (strncmp(line,"CIRCUMFIX",9) == 0) {
548
if (parse_flag(line, &circumfix, afflst)) {
549
finishFileMgr(afflst);
554
/* parse in the flag used by fogemorphemes */
555
if (strncmp(line,"ONLYINCOMPOUND",14) == 0) {
556
if (parse_flag(line, &onlyincompound, afflst)) {
557
finishFileMgr(afflst);
562
/* parse in the flag used by `needaffixs' */
563
if (strncmp(line,"PSEUDOROOT",10) == 0) {
564
if (parse_flag(line, &needaffix, afflst)) {
565
finishFileMgr(afflst);
570
/* parse in the flag used by `needaffixs' */
571
if (strncmp(line,"NEEDAFFIX",9) == 0) {
572
if (parse_flag(line, &needaffix, afflst)) {
573
finishFileMgr(afflst);
578
/* parse in the minimal length for words in compounds */
579
if (strncmp(line,"COMPOUNDMIN",11) == 0) {
580
if (parse_num(line, &cpdmin, afflst)) {
581
finishFileMgr(afflst);
584
if (cpdmin < 1) cpdmin = 1;
587
/* parse in the max. words and syllables in compounds */
588
if (strncmp(line,"COMPOUNDSYLLABLE",16) == 0) {
589
if (parse_cpdsyllable(line, afflst)) {
590
finishFileMgr(afflst);
595
/* parse in the flag used by compound_check() method */
596
if (strncmp(line,"SYLLABLENUM",11) == 0) {
597
if (parse_string(line, &cpdsyllablenum, afflst->getlinenum())) {
598
finishFileMgr(afflst);
603
/* parse in the flag used by the controlled compound words */
604
if (strncmp(line,"CHECKNUM",8) == 0) {
608
/* parse in the extra word characters */
609
if (strncmp(line,"WORDCHARS",9) == 0) {
610
if (parse_array(line, &wordchars, &wordchars_utf16, &wordchars_utf16_len, utf8, afflst->getlinenum())) {
611
finishFileMgr(afflst);
616
/* parse in the ignored characters (for example, Arabic optional diacretics charachters */
617
if (strncmp(line,"IGNORE",6) == 0) {
618
if (parse_array(line, &ignorechars, &ignorechars_utf16, &ignorechars_utf16_len, utf8, afflst->getlinenum())) {
619
finishFileMgr(afflst);
624
/* parse in the typical fault correcting table */
625
if (strncmp(line,"REP",3) == 0) {
626
if (parse_reptable(line, afflst)) {
627
finishFileMgr(afflst);
632
/* parse in the input conversion table */
633
if (strncmp(line,"ICONV",5) == 0) {
634
if (parse_convtable(line, afflst, &iconvtable, "ICONV")) {
635
finishFileMgr(afflst);
640
/* parse in the input conversion table */
641
if (strncmp(line,"OCONV",5) == 0) {
642
if (parse_convtable(line, afflst, &oconvtable, "OCONV")) {
643
finishFileMgr(afflst);
648
/* parse in the phonetic translation table */
649
if (strncmp(line,"PHONE",5) == 0) {
650
if (parse_phonetable(line, afflst)) {
651
finishFileMgr(afflst);
656
/* parse in the checkcompoundpattern table */
657
if (strncmp(line,"CHECKCOMPOUNDPATTERN",20) == 0) {
658
if (parse_checkcpdtable(line, afflst)) {
659
finishFileMgr(afflst);
664
/* parse in the defcompound table */
665
if (strncmp(line,"COMPOUNDRULE",12) == 0) {
666
if (parse_defcpdtable(line, afflst)) {
667
finishFileMgr(afflst);
672
/* parse in the related character map table */
673
if (strncmp(line,"MAP",3) == 0) {
674
if (parse_maptable(line, afflst)) {
675
finishFileMgr(afflst);
680
/* parse in the word breakpoints table */
681
if (strncmp(line,"BREAK",5) == 0) {
682
if (parse_breaktable(line, afflst)) {
683
finishFileMgr(afflst);
688
/* parse in the language for language specific codes */
689
if (strncmp(line,"LANG",4) == 0) {
690
if (parse_string(line, &lang, afflst->getlinenum())) {
691
finishFileMgr(afflst);
694
langnum = get_lang_num(lang);
697
if (strncmp(line,"VERSION",7) == 0) {
698
for(line = line + 7; *line == ' ' || *line == '\t'; line++);
699
version = mystrdup(line);
702
if (strncmp(line,"MAXNGRAMSUGS",12) == 0) {
703
if (parse_num(line, &maxngramsugs, afflst)) {
704
finishFileMgr(afflst);
709
if (strncmp(line,"ONLYMAXDIFF", 11) == 0)
712
if (strncmp(line,"MAXDIFF",7) == 0) {
713
if (parse_num(line, &maxdiff, afflst)) {
714
finishFileMgr(afflst);
719
if (strncmp(line,"MAXCPDSUGS",10) == 0) {
720
if (parse_num(line, &maxcpdsugs, afflst)) {
721
finishFileMgr(afflst);
726
if (strncmp(line,"NOSPLITSUGS",11) == 0) {
730
if (strncmp(line,"FULLSTRIP",9) == 0) {
734
if (strncmp(line,"SUGSWITHDOTS",12) == 0) {
738
/* parse in the flag used by forbidden words */
739
if (strncmp(line,"KEEPCASE",8) == 0) {
740
if (parse_flag(line, &keepcase, afflst)) {
741
finishFileMgr(afflst);
746
/* parse in the flag used by `forceucase' */
747
if (strncmp(line,"FORCEUCASE",10) == 0) {
748
if (parse_flag(line, &forceucase, afflst)) {
749
finishFileMgr(afflst);
754
/* parse in the flag used by `warn' */
755
if (strncmp(line,"WARN",4) == 0) {
756
if (parse_flag(line, &warn, afflst)) {
757
finishFileMgr(afflst);
762
if (strncmp(line,"FORBIDWARN",10) == 0) {
766
/* parse in the flag used by the affix generator */
767
if (strncmp(line,"SUBSTANDARD",11) == 0) {
768
if (parse_flag(line, &substandard, afflst)) {
769
finishFileMgr(afflst);
774
if (strncmp(line,"CHECKSHARPS",11) == 0) {
778
/* parse this affix: P - prefix, S - suffix */
780
if (strncmp(line,"PFX",3) == 0) ft = complexprefixes ? 'S' : 'P';
781
if (strncmp(line,"SFX",3) == 0) ft = complexprefixes ? 'P' : 'S';
784
memset(dupflags, 0, sizeof(dupflags));
787
if (parse_affix(line, ft, afflst, dupflags)) {
788
finishFileMgr(afflst);
794
finishFileMgr(afflst);
795
// affix trees are sorted now
797
// now we can speed up performance greatly taking advantage of the
798
// relationship between the affixes and the idea of "subsets".
800
// View each prefix as a potential leading subset of another and view
801
// each suffix (reversed) as a potential trailing subset of another.
803
// To illustrate this relationship if we know the prefix "ab" is found in the
804
// word to examine, only prefixes that "ab" is a leading subset of need be examined.
805
// Furthermore is "ab" is not present then none of the prefixes that "ab" is
806
// is a subset need be examined.
807
// The same argument goes for suffix string that are reversed.
809
// Then to top this off why not examine the first char of the word to quickly
810
// limit the set of prefixes to examine (i.e. the prefixes to examine must
811
// be leading supersets of the first character of the word (if they exist)
813
// To take advantage of this "subset" relationship, we need to add two links
814
// from entry. One to take next if the current prefix is found (call it nexteq)
815
// and one to take next if the current prefix is not found (call it nextne).
817
// Since we have built ordered lists, all that remains is to properly initialize
818
// the nextne and nexteq pointers that relate them
823
/* get encoding for CHECKCOMPOUNDCASE */
825
char * enc = get_encoding();
826
csconv = get_current_cs(enc);
832
expw.assign(wordchars);
836
for (int i = 0; i <= 255; i++) {
837
if ( (csconv[i].cupper != csconv[i].clower) &&
838
(expw.find((char)i) == std::string::npos)) {
839
expw.push_back((char) i);
843
wordchars = mystrdup(expw.c_str());
846
// default BREAK definition
847
if (numbreak == -1) {
848
breaktable = (char **) malloc(sizeof(char *) * 3);
849
if (!breaktable) return 1;
850
breaktable[0] = mystrdup("-");
851
breaktable[1] = mystrdup("^-");
852
breaktable[2] = mystrdup("-$");
853
if (breaktable[0] && breaktable[1] && breaktable[2]) numbreak = 3;
859
// we want to be able to quickly access prefix information
860
// both by prefix flag, and sorted by prefix string itself
861
// so we need to set up two indexes
863
int AffixMgr::build_pfxtree(PfxEntry* pfxptr)
867
PfxEntry * ep = pfxptr;
869
// get the right starting points
870
const char * key = ep->getKey();
871
const unsigned char flg = (unsigned char) (ep->getFlag() & 0x00FF);
873
// first index by flag which must exist
879
// handle the special case of null affix string
880
if (strlen(key) == 0) {
881
// always inset them at head of list at element 0
888
// now handle the normal case
892
unsigned char sp = *((const unsigned char *)key);
895
// handle the first insert
902
// otherwise use binary tree insertion so that a sorted
903
// list can easily be generated later
907
if (strcmp(ep->getKey(), ptr->getKey() ) <= 0) {
908
ptr = ptr->getNextEQ();
914
ptr = ptr->getNextNE();
924
// we want to be able to quickly access suffix information
925
// both by suffix flag, and sorted by the reverse of the
926
// suffix string itself; so we need to set up two indexes
927
int AffixMgr::build_sfxtree(SfxEntry* sfxptr)
931
SfxEntry * ep = sfxptr;
933
/* get the right starting point */
934
const char * key = ep->getKey();
935
const unsigned char flg = (unsigned char) (ep->getFlag() & 0x00FF);
937
// first index by flag which must exist
942
// next index by affix string
944
// handle the special case of null affix string
945
if (strlen(key) == 0) {
946
// always inset them at head of list at element 0
953
// now handle the normal case
957
unsigned char sp = *((const unsigned char *)key);
960
// handle the first insert
966
// otherwise use binary tree insertion so that a sorted
967
// list can easily be generated later
971
if (strcmp(ep->getKey(), ptr->getKey() ) <= 0) {
972
ptr = ptr->getNextEQ();
978
ptr = ptr->getNextNE();
988
// convert from binary tree to sorted list
989
int AffixMgr::process_pfx_tree_to_list()
991
for (int i=1; i< SETSIZE; i++) {
992
pStart[i] = process_pfx_in_order(pStart[i],NULL);
998
PfxEntry* AffixMgr::process_pfx_in_order(PfxEntry* ptr, PfxEntry* nptr)
1001
nptr = process_pfx_in_order(ptr->getNextNE(), nptr);
1003
nptr = process_pfx_in_order(ptr->getNextEQ(), ptr);
1009
// convert from binary tree to sorted list
1010
int AffixMgr:: process_sfx_tree_to_list()
1012
for (int i=1; i< SETSIZE; i++) {
1013
sStart[i] = process_sfx_in_order(sStart[i],NULL);
1018
SfxEntry* AffixMgr::process_sfx_in_order(SfxEntry* ptr, SfxEntry* nptr)
1021
nptr = process_sfx_in_order(ptr->getNextNE(), nptr);
1023
nptr = process_sfx_in_order(ptr->getNextEQ(), ptr);
1029
// reinitialize the PfxEntry links NextEQ and NextNE to speed searching
1030
// using the idea of leading subsets this time
1031
int AffixMgr::process_pfx_order()
1035
// loop through each prefix list starting point
1036
for (int i=1; i < SETSIZE; i++) {
1040
// look through the remainder of the list
1041
// and find next entry with affix that
1042
// the current one is not a subset of
1043
// mark that as destination for NextNE
1044
// use next in list that you are a subset
1047
for (; ptr != NULL; ptr = ptr->getNext()) {
1049
PfxEntry * nptr = ptr->getNext();
1050
for (; nptr != NULL; nptr = nptr->getNext()) {
1051
if (! isSubset( ptr->getKey() , nptr->getKey() )) break;
1053
ptr->setNextNE(nptr);
1054
ptr->setNextEQ(NULL);
1055
if ((ptr->getNext()) && isSubset(ptr->getKey() , (ptr->getNext())->getKey()))
1056
ptr->setNextEQ(ptr->getNext());
1059
// now clean up by adding smart search termination strings:
1060
// if you are already a superset of the previous prefix
1061
// but not a subset of the next, search can end here
1062
// so set NextNE properly
1065
for (; ptr != NULL; ptr = ptr->getNext()) {
1066
PfxEntry * nptr = ptr->getNext();
1067
PfxEntry * mptr = NULL;
1068
for (; nptr != NULL; nptr = nptr->getNext()) {
1069
if (! isSubset(ptr->getKey(),nptr->getKey())) break;
1072
if (mptr) mptr->setNextNE(NULL);
1078
// initialize the SfxEntry links NextEQ and NextNE to speed searching
1079
// using the idea of leading subsets this time
1080
int AffixMgr::process_sfx_order()
1084
// loop through each prefix list starting point
1085
for (int i=1; i < SETSIZE; i++) {
1089
// look through the remainder of the list
1090
// and find next entry with affix that
1091
// the current one is not a subset of
1092
// mark that as destination for NextNE
1093
// use next in list that you are a subset
1096
for (; ptr != NULL; ptr = ptr->getNext()) {
1097
SfxEntry * nptr = ptr->getNext();
1098
for (; nptr != NULL; nptr = nptr->getNext()) {
1099
if (! isSubset(ptr->getKey(),nptr->getKey())) break;
1101
ptr->setNextNE(nptr);
1102
ptr->setNextEQ(NULL);
1103
if ((ptr->getNext()) && isSubset(ptr->getKey(),(ptr->getNext())->getKey()))
1104
ptr->setNextEQ(ptr->getNext());
1108
// now clean up by adding smart search termination strings:
1109
// if you are already a superset of the previous suffix
1110
// but not a subset of the next, search can end here
1111
// so set NextNE properly
1114
for (; ptr != NULL; ptr = ptr->getNext()) {
1115
SfxEntry * nptr = ptr->getNext();
1116
SfxEntry * mptr = NULL;
1117
for (; nptr != NULL; nptr = nptr->getNext()) {
1118
if (! isSubset(ptr->getKey(),nptr->getKey())) break;
1121
if (mptr) mptr->setNextNE(NULL);
1127
// add flags to the result for dictionary debugging
1128
void AffixMgr::debugflag(char * result, unsigned short flag) {
1129
char * st = encode_flag(flag);
1130
mystrcat(result, " ", MAXLNLEN);
1131
mystrcat(result, MORPH_FLAG, MAXLNLEN);
1133
mystrcat(result, st, MAXLNLEN);
1138
// add flags to the result for dictionary debugging
1139
std::string& AffixMgr::debugflag(std::string& result, unsigned short flag) {
1140
char * st = encode_flag(flag);
1142
result.append(MORPH_FLAG);
1150
// calculate the character length of the condition
1151
int AffixMgr::condlen(char * st)
1159
} else if (*st == ']') group = false;
1160
else if (!group && (!utf8 ||
1161
(!(*st & 0x80) || ((*st & 0xc0) == 0x80)))) l++;
1166
int AffixMgr::encodeit(affentry &entry, char * cs)
1168
if (strcmp(cs,".") != 0) {
1169
entry.numconds = (char) condlen(cs);
1170
// coverity[buffer_size_warning] - deliberate use of lack of end of conds padded by strncpy as long condition flag
1171
strncpy(entry.c.conds, cs, MAXCONDLEN);
1172
if (entry.c.conds[MAXCONDLEN - 1] && cs[MAXCONDLEN]) {
1173
entry.opts += aeLONGCOND;
1174
entry.c.l.conds2 = mystrdup(cs + MAXCONDLEN_1);
1175
if (!entry.c.l.conds2) return 1;
1179
entry.c.conds[0] = '\0';
1184
// return 1 if s1 is a leading subset of s2 (dots are for infixes)
1185
inline int AffixMgr::isSubset(const char * s1, const char * s2)
1187
while (((*s1 == *s2) || (*s1 == '.')) && (*s1 != '\0')) {
1191
return (*s1 == '\0');
1195
// check word for prefixes
1196
struct hentry * AffixMgr::prefix_check(const char * word, int len, char in_compound,
1197
const FLAG needflag)
1199
struct hentry * rv= NULL;
1205
// first handle the special case of 0 length prefixes
1206
PfxEntry * pe = pStart[0];
1210
((in_compound != IN_CPD_NOT) || !(pe->getCont() &&
1211
(TESTAFF(pe->getCont(), onlyincompound, pe->getContLen())))) &&
1212
// permit prefixes in compounds
1213
((in_compound != IN_CPD_END) || (pe->getCont() &&
1214
(TESTAFF(pe->getCont(), compoundpermitflag, pe->getContLen()))))
1217
rv = pe->checkword(word, len, in_compound, needflag);
1219
pfx=pe; // BUG: pfx not stateless
1226
// now handle the general case
1227
unsigned char sp = *((const unsigned char *)word);
1228
PfxEntry * pptr = pStart[sp];
1231
if (isSubset(pptr->getKey(),word)) {
1234
((in_compound != IN_CPD_NOT) || !(pptr->getCont() &&
1235
(TESTAFF(pptr->getCont(), onlyincompound, pptr->getContLen())))) &&
1236
// permit prefixes in compounds
1237
((in_compound != IN_CPD_END) || (pptr->getCont() &&
1238
(TESTAFF(pptr->getCont(), compoundpermitflag, pptr->getContLen()))))
1241
rv = pptr->checkword(word, len, in_compound, needflag);
1243
pfx=pptr; // BUG: pfx not stateless
1247
pptr = pptr->getNextEQ();
1249
pptr = pptr->getNextNE();
1256
// check word for prefixes
1257
struct hentry * AffixMgr::prefix_check_twosfx(const char * word, int len,
1258
char in_compound, const FLAG needflag)
1260
struct hentry * rv= NULL;
1265
// first handle the special case of 0 length prefixes
1266
PfxEntry * pe = pStart[0];
1269
rv = pe->check_twosfx(word, len, in_compound, needflag);
1274
// now handle the general case
1275
unsigned char sp = *((const unsigned char *)word);
1276
PfxEntry * pptr = pStart[sp];
1279
if (isSubset(pptr->getKey(),word)) {
1280
rv = pptr->check_twosfx(word, len, in_compound, needflag);
1285
pptr = pptr->getNextEQ();
1287
pptr = pptr->getNextNE();
1294
// check word for prefixes
1295
char * AffixMgr::prefix_check_morph(const char * word, int len, char in_compound,
1296
const FLAG needflag)
1300
char result[MAXLNLEN];
1306
// first handle the special case of 0 length prefixes
1307
PfxEntry * pe = pStart[0];
1309
st = pe->check_morph(word,len,in_compound, needflag);
1311
mystrcat(result, st, MAXLNLEN);
1314
// if (rv) return rv;
1318
// now handle the general case
1319
unsigned char sp = *((const unsigned char *)word);
1320
PfxEntry * pptr = pStart[sp];
1323
if (isSubset(pptr->getKey(),word)) {
1324
st = pptr->check_morph(word,len,in_compound, needflag);
1327
if ((in_compound != IN_CPD_NOT) || !((pptr->getCont() &&
1328
(TESTAFF(pptr->getCont(), onlyincompound, pptr->getContLen()))))) {
1329
mystrcat(result, st, MAXLNLEN);
1334
pptr = pptr->getNextEQ();
1336
pptr = pptr->getNextNE();
1340
if (*result) return mystrdup(result);
1345
// check word for prefixes
1346
char * AffixMgr::prefix_check_twosfx_morph(const char * word, int len,
1347
char in_compound, const FLAG needflag)
1351
char result[MAXLNLEN];
1357
// first handle the special case of 0 length prefixes
1358
PfxEntry * pe = pStart[0];
1360
st = pe->check_twosfx_morph(word,len,in_compound, needflag);
1362
mystrcat(result, st, MAXLNLEN);
1368
// now handle the general case
1369
unsigned char sp = *((const unsigned char *)word);
1370
PfxEntry * pptr = pStart[sp];
1373
if (isSubset(pptr->getKey(),word)) {
1374
st = pptr->check_twosfx_morph(word, len, in_compound, needflag);
1376
mystrcat(result, st, MAXLNLEN);
1380
pptr = pptr->getNextEQ();
1382
pptr = pptr->getNextNE();
1386
if (*result) return mystrdup(result);
1390
// Is word a non compound with a REP substitution (see checkcompoundrep)?
1391
int AffixMgr::cpdrep_check(const char * word, int wl)
1395
if ((wl < 2) || !numrep) return 0;
1397
for (int i=0; i < numrep; i++ ) {
1399
int lenp = strlen(reptable[i].pattern);
1400
// search every occurence of the pattern in the word
1401
while ((r=strstr(r, reptable[i].pattern)) != NULL) {
1402
std::string candidate(word);
1403
candidate.replace(r-word, lenp, reptable[i].pattern2);
1404
if (candidate_check(candidate.c_str(), candidate.size())) return 1;
1405
r++; // search for the next letter
1411
// forbid compoundings when there are special patterns at word bound
1412
int AffixMgr::cpdpat_check(const char * word, int pos, hentry * r1, hentry * r2, const char /*affixed*/)
1415
for (int i = 0; i < numcheckcpd; i++) {
1416
if (isSubset(checkcpdtable[i].pattern2, word + pos) &&
1417
(!r1 || !checkcpdtable[i].cond ||
1418
(r1->astr && TESTAFF(r1->astr, checkcpdtable[i].cond, r1->alen))) &&
1419
(!r2 || !checkcpdtable[i].cond2 ||
1420
(r2->astr && TESTAFF(r2->astr, checkcpdtable[i].cond2, r2->alen))) &&
1421
// zero length pattern => only TESTAFF
1422
// zero pattern (0/flag) => unmodified stem (zero affixes allowed)
1423
(!*(checkcpdtable[i].pattern) || (
1424
(*(checkcpdtable[i].pattern)=='0' && r1->blen <= pos && strncmp(word + pos - r1->blen, r1->word, r1->blen) == 0) ||
1425
(*(checkcpdtable[i].pattern)!='0' && ((len = strlen(checkcpdtable[i].pattern)) != 0) &&
1426
strncmp(word + pos - len, checkcpdtable[i].pattern, len) == 0)))) {
1433
// forbid compounding with neighbouring upper and lower case characters at word bounds
1434
int AffixMgr::cpdcase_check(const char * word, int pos)
1439
u8_u16(&u, 1, word + pos);
1440
for (p = word + pos - 1; (*p & 0xc0) == 0x80; p--);
1442
unsigned short a = (u.h << 8) + u.l;
1443
unsigned short b = (w.h << 8) + w.l;
1444
if (((unicodetoupper(a, langnum) == a) || (unicodetoupper(b, langnum) == b)) &&
1445
(a != '-') && (b != '-')) return 1;
1447
unsigned char a = *(word + pos - 1);
1448
unsigned char b = *(word + pos);
1449
if ((csconv[a].ccase || csconv[b].ccase) && (a != '-') && (b != '-')) return 1;
1454
// check compound patterns
1455
int AffixMgr::defcpd_check(hentry *** words, short wnum, hentry * rv, hentry ** def, char all)
1457
signed short btpp[MAXWORDLEN]; // metacharacter (*, ?) positions for backtracking
1458
signed short btwp[MAXWORDLEN]; // word positions for metacharacters
1459
int btnum[MAXWORDLEN]; // number of matched characters in metacharacter positions
1474
(*words)[wnum] = rv;
1476
// has the last word COMPOUNDRULE flag?
1477
if (rv->alen == 0) {
1478
(*words)[wnum] = NULL;
1479
if (w) *words = NULL;
1483
for (i = 0; i < numdefcpd; i++) {
1484
for (j = 0; j < defcpdtable[i].len; j++) {
1485
if (defcpdtable[i].def[j] != '*' && defcpdtable[i].def[j] != '?' &&
1486
TESTAFF(rv->astr, defcpdtable[i].def[j], rv->alen)) {
1493
(*words)[wnum] = NULL;
1494
if (w) *words = NULL;
1498
for (i = 0; i < numdefcpd; i++) {
1499
signed short pp = 0; // pattern position
1500
signed short wp = 0; // "words" position
1505
while ((pp < defcpdtable[i].len) && (wp <= wnum)) {
1506
if (((pp+1) < defcpdtable[i].len) &&
1507
((defcpdtable[i].def[pp+1] == '*') || (defcpdtable[i].def[pp+1] == '?'))) {
1508
int wend = (defcpdtable[i].def[pp+1] == '?') ? wp : wnum;
1513
while (wp <= wend) {
1514
if (!(*words)[wp]->alen ||
1515
!TESTAFF((*words)[wp]->astr, defcpdtable[i].def[pp-2], (*words)[wp]->alen)) {
1521
if (wp <= wnum) ok2 = 0;
1522
btnum[bt] = wp - btwp[bt];
1523
if (btnum[bt] > 0) bt++;
1527
if (!(*words)[wp] || !(*words)[wp]->alen ||
1528
!TESTAFF((*words)[wp]->astr, defcpdtable[i].def[pp], (*words)[wp]->alen)) {
1534
if ((defcpdtable[i].len == pp) && !(wp > wnum)) ok = 0;
1539
while ((defcpdtable[i].len > r) && ((r+1) < defcpdtable[i].len) &&
1540
((defcpdtable[i].def[r+1] == '*') || (defcpdtable[i].def[r+1] == '?'))) r+=2;
1541
if (defcpdtable[i].len <= r) return 1;
1548
wp = btwp[bt - 1] + (signed short) btnum[bt - 1];
1549
} while ((btnum[bt - 1] < 0) && --bt);
1552
if (ok && ok2 && (!all || (defcpdtable[i].len <= pp))) return 1;
1554
// check zero ending
1555
while (ok && ok2 && (defcpdtable[i].len > pp) && ((pp+1) < defcpdtable[i].len) &&
1556
((defcpdtable[i].def[pp+1] == '*') || (defcpdtable[i].def[pp+1] == '?'))) pp+=2;
1557
if (ok && ok2 && (defcpdtable[i].len <= pp)) return 1;
1559
(*words)[wnum] = NULL;
1560
if (w) *words = NULL;
1564
inline int AffixMgr::candidate_check(const char * word, int len)
1566
struct hentry * rv=NULL;
1571
// rv = prefix_check(word,len,1);
1572
// if (rv) return 1;
1574
rv = affix_check(word,len);
1579
// calculate number of syllable for compound-checking
1580
short AffixMgr::get_syllable(const char * word, int wlen)
1582
if (cpdmaxsyllable==0) return 0;
1587
for (int i=0; i<wlen; i++) {
1588
if (strchr(cpdvowels, word[i])) num++;
1590
} else if (cpdvowels_utf16) {
1591
w_char w[MAXWORDUTF8LEN];
1592
int i = u8_u16(w, MAXWORDUTF8LEN, word);
1593
for (; i > 0; i--) {
1594
if (flag_bsearch((unsigned short *) cpdvowels_utf16,
1595
((unsigned short *) w)[i - 1], cpdvowels_utf16_len)) num++;
1601
void AffixMgr::setcminmax(int * cmin, int * cmax, const char * word, int len) {
1604
for (*cmin = 0, i = 0; (i < cpdmin) && word[*cmin]; i++) {
1605
for ((*cmin)++; (word[*cmin] & 0xc0) == 0x80; (*cmin)++);
1607
for (*cmax = len, i = 0; (i < (cpdmin - 1)) && *cmax; i++) {
1608
for ((*cmax)--; (word[*cmax] & 0xc0) == 0x80; (*cmax)--);
1612
*cmax = len - cpdmin + 1;
1617
// check if compound word is correctly spelled
1618
// hu_mov_rule = spec. Hungarian rule (XXX)
1619
struct hentry * AffixMgr::compound_check(const char * word, int len,
1620
short wordnum, short numsyllable, short maxwordnum, short wnum, hentry ** words = NULL,
1621
char hu_mov_rule = 0, char is_sug = 0, int * info = NULL)
1624
short oldnumsyllable, oldnumsyllable2, oldwordnum, oldwordnum2;
1625
struct hentry * rv = NULL;
1626
struct hentry * rv_first;
1627
struct hentry * rwords[MAXWORDLEN]; // buffer for COMPOUND pattern checking
1628
char st [MAXWORDUTF8LEN + 4];
1638
int checkedstriple = 0;
1641
hentry ** oldwords = words;
1645
setcminmax(&cmin, &cmax, word, len);
1649
for (i = cmin; i < cmax; i++) {
1650
// go to end of the UTF-8 character
1652
for (; (st[i] & 0xc0) == 0x80; i++);
1653
if (i >= cmax) return NULL;
1657
onlycpdrule = (words) ? 1 : 0;
1659
do { // onlycpdrule loop
1661
oldnumsyllable = numsyllable;
1662
oldwordnum = wordnum;
1666
do { // simplified checkcompoundpattern loop
1669
for (; scpd <= numcheckcpd && (!checkcpdtable[scpd-1].pattern3 ||
1670
strncmp(word + i, checkcpdtable[scpd-1].pattern3, strlen(checkcpdtable[scpd-1].pattern3)) != 0); scpd++);
1672
if (scpd > numcheckcpd) break; // break simplified checkcompoundpattern loop
1673
strcpy(st + i, checkcpdtable[scpd-1].pattern);
1675
i += strlen(checkcpdtable[scpd-1].pattern);
1676
strcpy(st + i, checkcpdtable[scpd-1].pattern2);
1677
strcpy(st + i + strlen(checkcpdtable[scpd-1].pattern2), word + soldi + strlen(checkcpdtable[scpd-1].pattern3));
1680
len += strlen(checkcpdtable[scpd-1].pattern) + strlen(checkcpdtable[scpd-1].pattern2) - strlen(checkcpdtable[scpd-1].pattern3);
1683
setcminmax(&cmin, &cmax, st, len);
1685
cmax = len - cpdmin + 1;
1697
rv = lookup(st); // perhaps without prefix
1699
// search homonym with compound flag
1700
while ((rv) && !hu_mov_rule &&
1701
((needaffix && TESTAFF(rv->astr, needaffix, rv->alen)) ||
1702
!((compoundflag && !words && !onlycpdrule && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
1703
(compoundbegin && !wordnum && !onlycpdrule &&
1704
TESTAFF(rv->astr, compoundbegin, rv->alen)) ||
1705
(compoundmiddle && wordnum && !words && !onlycpdrule &&
1706
TESTAFF(rv->astr, compoundmiddle, rv->alen)) ||
1707
(numdefcpd && onlycpdrule &&
1708
((!words && !wordnum && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0)) ||
1709
(words && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0))))) ||
1710
(scpd != 0 && checkcpdtable[scpd-1].cond != FLAG_NULL &&
1711
!TESTAFF(rv->astr, checkcpdtable[scpd-1].cond, rv->alen)))
1713
rv = rv->next_homonym;
1716
if (rv) affixed = 0;
1719
if (onlycpdrule) break;
1721
!(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundflag))) {
1722
if (((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL,
1723
FLAG_NULL, compoundflag, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
1724
(compoundmoresuffixes && (rv = suffix_check_twosfx(st, i, 0, NULL, compoundflag)))) && !hu_mov_rule &&
1726
((compoundforbidflag && TESTAFF(sfx->getCont(), compoundforbidflag,
1727
sfx->getContLen())) || (compoundend &&
1728
TESTAFF(sfx->getCont(), compoundend,
1729
sfx->getContLen())))) {
1735
(((wordnum == 0) && compoundbegin &&
1736
((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL, FLAG_NULL, compoundbegin, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
1737
(compoundmoresuffixes && (rv = suffix_check_twosfx(st, i, 0, NULL, compoundbegin))) || // twofold suffixes + compound
1738
(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundbegin)))) ||
1739
((wordnum > 0) && compoundmiddle &&
1740
((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL, FLAG_NULL, compoundmiddle, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
1741
(compoundmoresuffixes && (rv = suffix_check_twosfx(st, i, 0, NULL, compoundmiddle))) || // twofold suffixes + compound
1742
(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundmiddle)))))
1743
) checked_prefix = 1;
1744
// else check forbiddenwords and needaffix
1745
} else if (rv->astr && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
1746
TESTAFF(rv->astr, needaffix, rv->alen) ||
1747
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
1748
(is_sug && nosuggest && TESTAFF(rv->astr, nosuggest, rv->alen))
1755
// check non_compound flag in suffix and prefix
1756
if ((rv) && !hu_mov_rule &&
1757
((pfx && pfx->getCont() &&
1758
TESTAFF(pfx->getCont(), compoundforbidflag,
1759
pfx->getContLen())) ||
1760
(sfx && sfx->getCont() &&
1761
TESTAFF(sfx->getCont(), compoundforbidflag,
1762
sfx->getContLen())))) {
1766
// check compoundend flag in suffix and prefix
1767
if ((rv) && !checked_prefix && compoundend && !hu_mov_rule &&
1768
((pfx && pfx->getCont() &&
1769
TESTAFF(pfx->getCont(), compoundend,
1770
pfx->getContLen())) ||
1771
(sfx && sfx->getCont() &&
1772
TESTAFF(sfx->getCont(), compoundend,
1773
sfx->getContLen())))) {
1777
// check compoundmiddle flag in suffix and prefix
1778
if ((rv) && !checked_prefix && (wordnum==0) && compoundmiddle && !hu_mov_rule &&
1779
((pfx && pfx->getCont() &&
1780
TESTAFF(pfx->getCont(), compoundmiddle,
1781
pfx->getContLen())) ||
1782
(sfx && sfx->getCont() &&
1783
TESTAFF(sfx->getCont(), compoundmiddle,
1784
sfx->getContLen())))) {
1788
// check forbiddenwords
1789
if ((rv) && (rv->astr) && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
1790
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
1791
(is_sug && nosuggest && TESTAFF(rv->astr, nosuggest, rv->alen)))) {
1795
// increment word number, if the second root has a compoundroot flag
1796
if ((rv) && compoundroot &&
1797
(TESTAFF(rv->astr, compoundroot, rv->alen))) {
1801
// first word is acceptable in compound words?
1803
( checked_prefix || (words && words[wnum]) ||
1804
(compoundflag && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
1805
((oldwordnum == 0) && compoundbegin && TESTAFF(rv->astr, compoundbegin, rv->alen)) ||
1806
((oldwordnum > 0) && compoundmiddle && TESTAFF(rv->astr, compoundmiddle, rv->alen))// ||
1809
// LANG_hu section: spec. Hungarian rule
1810
|| ((langnum == LANG_hu) && hu_mov_rule && (
1811
TESTAFF(rv->astr, 'F', rv->alen) || // XXX hardwired Hungarian dictionary codes
1812
TESTAFF(rv->astr, 'G', rv->alen) ||
1813
TESTAFF(rv->astr, 'H', rv->alen)
1816
// END of LANG_hu section
1819
// test CHECKCOMPOUNDPATTERN conditions
1820
scpd == 0 || checkcpdtable[scpd-1].cond == FLAG_NULL ||
1821
TESTAFF(rv->astr, checkcpdtable[scpd-1].cond, rv->alen)
1823
&& ! (( checkcompoundtriple && scpd == 0 && !words && // test triple letters
1824
(word[i-1]==word[i]) && (
1825
((i>1) && (word[i-1]==word[i-2])) ||
1826
((word[i-1]==word[i+1])) // may be word[i+1] == '\0'
1830
checkcompoundcase && scpd == 0 && !words && cpdcase_check(word, i)
1833
// LANG_hu section: spec. Hungarian rule
1834
|| ((!rv) && (langnum == LANG_hu) && hu_mov_rule && (rv = affix_check(st,i)) &&
1835
(sfx && sfx->getCont() && ( // XXX hardwired Hungarian dic. codes
1836
TESTAFF(sfx->getCont(), (unsigned short) 'x', sfx->getContLen()) ||
1837
TESTAFF(sfx->getCont(), (unsigned short) '%', sfx->getContLen())
1841
) { // first word is ok condition
1843
// LANG_hu section: spec. Hungarian rule
1844
if (langnum == LANG_hu) {
1845
// calculate syllable number of the word
1846
numsyllable += get_syllable(st, i);
1847
// + 1 word, if syllable number of the prefix > 1 (hungarian convention)
1848
if (pfx && (get_syllable(pfx->getKey(),strlen(pfx->getKey())) > 1)) wordnum++;
1850
// END of LANG_hu section
1856
do { // striple loop
1858
// check simplifiedtriple
1859
if (simplifiedtriple) {
1862
i--; // check "fahrt" instead of "ahrt" in "Schiffahrt"
1863
} else if (i > 2 && *(word+i - 1) == *(word + i - 2)) striple = 1;
1866
rv = lookup((st+i)); // perhaps without prefix
1868
// search homonym with compound flag
1869
while ((rv) && ((needaffix && TESTAFF(rv->astr, needaffix, rv->alen)) ||
1870
!((compoundflag && !words && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
1871
(compoundend && !words && TESTAFF(rv->astr, compoundend, rv->alen)) ||
1872
(numdefcpd && words && defcpd_check(&words, wnum + 1, rv, NULL,1))) ||
1873
(scpd != 0 && checkcpdtable[scpd-1].cond2 != FLAG_NULL &&
1874
!TESTAFF(rv->astr, checkcpdtable[scpd-1].cond2, rv->alen))
1876
rv = rv->next_homonym;
1880
if (rv && forceucase && (rv) &&
1881
(TESTAFF(rv->astr, forceucase, rv->alen)) && !(info && *info & SPELL_ORIGCAP)) rv = NULL;
1883
if (rv && words && words[wnum + 1]) return rv_first;
1885
oldnumsyllable2 = numsyllable;
1886
oldwordnum2 = wordnum;
1889
// LANG_hu section: spec. Hungarian rule, XXX hardwired dictionary code
1890
if ((rv) && (langnum == LANG_hu) && (TESTAFF(rv->astr, 'I', rv->alen)) && !(TESTAFF(rv->astr, 'J', rv->alen))) {
1893
// END of LANG_hu section
1895
// increment word number, if the second root has a compoundroot flag
1896
if ((rv) && (compoundroot) &&
1897
(TESTAFF(rv->astr, compoundroot, rv->alen))) {
1901
// check forbiddenwords
1902
if ((rv) && (rv->astr) && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
1903
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
1904
(is_sug && nosuggest && TESTAFF(rv->astr, nosuggest, rv->alen)))) return NULL;
1906
// second word is acceptable, as a root?
1907
// hungarian conventions: compounding is acceptable,
1908
// when compound forms consist of 2 words, or if more,
1909
// then the syllable number of root words must be 6, or lesser.
1912
(compoundflag && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
1913
(compoundend && TESTAFF(rv->astr, compoundend, rv->alen))
1916
((cpdwordmax==-1) || (wordnum+1<cpdwordmax)) ||
1917
((cpdmaxsyllable!=0) &&
1918
(numsyllable + get_syllable(HENTRY_WORD(rv), rv->clen)<=cpdmaxsyllable))
1921
// test CHECKCOMPOUNDPATTERN
1922
!numcheckcpd || scpd != 0 || !cpdpat_check(word, i, rv_first, rv, 0)
1925
(!checkcompounddup || (rv != rv_first))
1927
// test CHECKCOMPOUNDPATTERN conditions
1928
&& (scpd == 0 || checkcpdtable[scpd-1].cond2 == FLAG_NULL ||
1929
TESTAFF(rv->astr, checkcpdtable[scpd-1].cond2, rv->alen))
1932
// forbid compound word, if it is a non compound word with typical fault
1933
if (checkcompoundrep && cpdrep_check(word,len)) return NULL;
1937
numsyllable = oldnumsyllable2;
1938
wordnum = oldwordnum2;
1940
// perhaps second word has prefix or/and suffix
1942
sfxflag = FLAG_NULL;
1943
rv = (compoundflag && !onlycpdrule) ? affix_check((word+i),strlen(word+i), compoundflag, IN_CPD_END) : NULL;
1944
if (!rv && compoundend && !onlycpdrule) {
1947
rv = affix_check((word+i),strlen(word+i), compoundend, IN_CPD_END);
1950
if (!rv && numdefcpd && words) {
1951
rv = affix_check((word+i),strlen(word+i), 0, IN_CPD_END);
1952
if (rv && defcpd_check(&words, wnum + 1, rv, NULL, 1)) return rv_first;
1956
// test CHECKCOMPOUNDPATTERN conditions (allowed forms)
1957
if (rv && !(scpd == 0 || checkcpdtable[scpd-1].cond2 == FLAG_NULL ||
1958
TESTAFF(rv->astr, checkcpdtable[scpd-1].cond2, rv->alen))) rv = NULL;
1960
// test CHECKCOMPOUNDPATTERN conditions (forbidden compounds)
1961
if (rv && numcheckcpd && scpd == 0 && cpdpat_check(word, i, rv_first, rv, affixed)) rv = NULL;
1963
// check non_compound flag in suffix and prefix
1965
((pfx && pfx->getCont() &&
1966
TESTAFF(pfx->getCont(), compoundforbidflag,
1967
pfx->getContLen())) ||
1968
(sfx && sfx->getCont() &&
1969
TESTAFF(sfx->getCont(), compoundforbidflag,
1970
sfx->getContLen())))) {
1975
if (rv && forceucase && (rv) &&
1976
(TESTAFF(rv->astr, forceucase, rv->alen)) && !(info && *info & SPELL_ORIGCAP)) rv = NULL;
1978
// check forbiddenwords
1979
if ((rv) && (rv->astr) && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
1980
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
1981
(is_sug && nosuggest && TESTAFF(rv->astr, nosuggest, rv->alen)))) return NULL;
1983
// pfxappnd = prefix of word+i, or NULL
1984
// calculate syllable number of prefix.
1985
// hungarian convention: when syllable number of prefix is more,
1986
// than 1, the prefix+word counts as two words.
1988
if (langnum == LANG_hu) {
1989
// calculate syllable number of the word
1990
numsyllable += get_syllable(word + i, strlen(word + i));
1992
// - affix syllable num.
1993
// XXX only second suffix (inflections, not derivations)
1995
char * tmp = myrevstrdup(sfxappnd);
1996
numsyllable -= get_syllable(tmp, strlen(tmp));
2000
// + 1 word, if syllable number of the prefix > 1 (hungarian convention)
2001
if (pfx && (get_syllable(pfx->getKey(),strlen(pfx->getKey())) > 1)) wordnum++;
2003
// increment syllable num, if last word has a SYLLABLENUM flag
2004
// and the suffix is beginning `s'
2006
if (cpdsyllablenum) {
2008
case 'c': { numsyllable+=2; break; }
2009
case 'J': { numsyllable += 1; break; }
2010
case 'I': { if (rv && TESTAFF(rv->astr, 'J', rv->alen)) numsyllable += 1; break; }
2015
// increment word number, if the second word has a compoundroot flag
2016
if ((rv) && (compoundroot) &&
2017
(TESTAFF(rv->astr, compoundroot, rv->alen))) {
2021
// second word is acceptable, as a word with prefix or/and suffix?
2022
// hungarian conventions: compounding is acceptable,
2023
// when compound forms consist 2 word, otherwise
2024
// the syllable number of root words is 6, or lesser.
2027
((cpdwordmax == -1) || (wordnum + 1 < cpdwordmax)) ||
2028
((cpdmaxsyllable != 0) &&
2029
(numsyllable <= cpdmaxsyllable))
2032
(!checkcompounddup || (rv != rv_first))
2034
// forbid compound word, if it is a non compound word with typical fault
2035
if (checkcompoundrep && cpdrep_check(word, len)) return NULL;
2039
numsyllable = oldnumsyllable2;
2040
wordnum = oldwordnum2;
2042
// perhaps second word is a compound word (recursive call)
2043
if (wordnum < maxwordnum) {
2044
rv = compound_check((st+i),strlen(st+i), wordnum+1,
2045
numsyllable, maxwordnum, wnum + 1, words, 0, is_sug, info);
2047
if (rv && numcheckcpd && ((scpd == 0 && cpdpat_check(word, i, rv_first, rv, affixed)) ||
2048
(scpd != 0 && !cpdpat_check(word, i, rv_first, rv, affixed)))) rv = NULL;
2053
// forbid compound word, if it is a non compound word with typical fault
2054
if (checkcompoundrep || forbiddenword) {
2055
struct hentry * rv2 = NULL;
2057
if (checkcompoundrep && cpdrep_check(word, len)) return NULL;
2060
if (strncmp(rv->word, word + i, rv->blen) == 0) {
2061
char r = *(st + i + rv->blen);
2062
*(st + i + rv->blen) = '\0';
2064
if (checkcompoundrep && cpdrep_check(st, i + rv->blen)) {
2065
*(st + i + rv->blen) = r;
2069
if (forbiddenword) {
2071
if (!rv2) rv2 = affix_check(word, len);
2072
if (rv2 && rv2->astr && TESTAFF(rv2->astr, forbiddenword, rv2->alen) &&
2073
(strncmp(rv2->word, st, i + rv->blen) == 0)) {
2077
*(st + i + rv->blen) = r;
2082
} while (striple && !checkedstriple); // end of striple loop
2084
if (checkedstriple) {
2090
} // first word is ok condition
2102
} while (!onlycpdrule && simplifiedcpd && scpd <= numcheckcpd); // end of simplifiedcpd loop
2105
wordnum = oldwordnum;
2106
numsyllable = oldnumsyllable;
2110
strcpy(st, word); // XXX add more optim.
2114
} while (numdefcpd && oldwordnum == 0 && onlycpdrule++ < 1); // end of onlycpd loop
2120
// check if compound word is correctly spelled
2121
// hu_mov_rule = spec. Hungarian rule (XXX)
2122
int AffixMgr::compound_check_morph(const char * word, int len,
2123
short wordnum, short numsyllable, short maxwordnum, short wnum, hentry ** words,
2124
char hu_mov_rule = 0, char ** result = NULL, char * partresult = NULL)
2127
short oldnumsyllable, oldnumsyllable2, oldwordnum, oldwordnum2;
2130
struct hentry * rv = NULL;
2131
struct hentry * rv_first;
2132
struct hentry * rwords[MAXWORDLEN]; // buffer for COMPOUND pattern checking
2133
char st [MAXWORDUTF8LEN + 4];
2137
char presult[MAXLNLEN];
2144
hentry ** oldwords = words;
2146
setcminmax(&cmin, &cmax, word, len);
2150
for (i = cmin; i < cmax; i++) {
2151
// go to end of the UTF-8 character
2153
for (; (st[i] & 0xc0) == 0x80; i++);
2154
if (i >= cmax) return 0;
2158
onlycpdrule = (words) ? 1 : 0;
2160
do { // onlycpdrule loop
2162
oldnumsyllable = numsyllable;
2163
oldwordnum = wordnum;
2175
if (partresult) mystrcat(presult, partresult, MAXLNLEN);
2177
rv = lookup(st); // perhaps without prefix
2179
// search homonym with compound flag
2180
while ((rv) && !hu_mov_rule &&
2181
((needaffix && TESTAFF(rv->astr, needaffix, rv->alen)) ||
2182
!((compoundflag && !words && !onlycpdrule && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
2183
(compoundbegin && !wordnum && !onlycpdrule &&
2184
TESTAFF(rv->astr, compoundbegin, rv->alen)) ||
2185
(compoundmiddle && wordnum && !words && !onlycpdrule &&
2186
TESTAFF(rv->astr, compoundmiddle, rv->alen)) ||
2187
(numdefcpd && onlycpdrule &&
2188
((!words && !wordnum && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0)) ||
2189
(words && defcpd_check(&words, wnum, rv, (hentry **) &rwords, 0))))
2191
rv = rv->next_homonym;
2194
if (rv) affixed = 0;
2197
sprintf(presult + strlen(presult), "%c%s%s", MSEP_FLD, MORPH_PART, st);
2198
if (!HENTRY_FIND(rv, MORPH_STEM)) {
2199
sprintf(presult + strlen(presult), "%c%s%s", MSEP_FLD, MORPH_STEM, st);
2201
// store the pointer of the hash entry
2202
// sprintf(presult + strlen(presult), "%c%s%p", MSEP_FLD, MORPH_HENTRY, rv);
2203
if (HENTRY_DATA(rv)) {
2204
sprintf(presult + strlen(presult), "%c%s", MSEP_FLD, HENTRY_DATA2(rv));
2209
if (onlycpdrule && strlen(*result) > MAXLNLEN/10) break;
2211
!(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundflag))) {
2212
if (((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL,
2213
FLAG_NULL, compoundflag, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
2214
(compoundmoresuffixes && (rv = suffix_check_twosfx(st, i, 0, NULL, compoundflag)))) && !hu_mov_rule &&
2216
((compoundforbidflag && TESTAFF(sfx->getCont(), compoundforbidflag,
2217
sfx->getContLen())) || (compoundend &&
2218
TESTAFF(sfx->getCont(), compoundend,
2219
sfx->getContLen())))) {
2225
(((wordnum == 0) && compoundbegin &&
2226
((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL, FLAG_NULL, compoundbegin, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
2227
(compoundmoresuffixes && (rv = suffix_check_twosfx(st, i, 0, NULL, compoundbegin))) || // twofold suffix+compound
2228
(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundbegin)))) ||
2229
((wordnum > 0) && compoundmiddle &&
2230
((rv = suffix_check(st, i, 0, NULL, NULL, 0, NULL, FLAG_NULL, compoundmiddle, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN)) ||
2231
(compoundmoresuffixes && (rv = suffix_check_twosfx(st, i, 0, NULL, compoundmiddle))) || // twofold suffix+compound
2232
(rv = prefix_check(st, i, hu_mov_rule ? IN_CPD_OTHER : IN_CPD_BEGIN, compoundmiddle)))))
2234
// char * p = prefix_check_morph(st, i, 0, compound);
2236
if (compoundflag) p = affix_check_morph(st, i, compoundflag);
2237
if (!p || (*p == '\0')) {
2240
if ((wordnum == 0) && compoundbegin) {
2241
p = affix_check_morph(st, i, compoundbegin);
2242
} else if ((wordnum > 0) && compoundmiddle) {
2243
p = affix_check_morph(st, i, compoundmiddle);
2246
if (p && (*p != '\0')) {
2247
sprintf(presult + strlen(presult), "%c%s%s%s", MSEP_FLD,
2248
MORPH_PART, st, line_uniq_app(&p, MSEP_REC));
2253
// else check forbiddenwords
2254
} else if (rv->astr && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
2255
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
2256
TESTAFF(rv->astr, needaffix, rv->alen))) {
2261
// check non_compound flag in suffix and prefix
2262
if ((rv) && !hu_mov_rule &&
2263
((pfx && pfx->getCont() &&
2264
TESTAFF(pfx->getCont(), compoundforbidflag,
2265
pfx->getContLen())) ||
2266
(sfx && sfx->getCont() &&
2267
TESTAFF(sfx->getCont(), compoundforbidflag,
2268
sfx->getContLen())))) {
2272
// check compoundend flag in suffix and prefix
2273
if ((rv) && !checked_prefix && compoundend && !hu_mov_rule &&
2274
((pfx && pfx->getCont() &&
2275
TESTAFF(pfx->getCont(), compoundend,
2276
pfx->getContLen())) ||
2277
(sfx && sfx->getCont() &&
2278
TESTAFF(sfx->getCont(), compoundend,
2279
sfx->getContLen())))) {
2283
// check compoundmiddle flag in suffix and prefix
2284
if ((rv) && !checked_prefix && (wordnum==0) && compoundmiddle && !hu_mov_rule &&
2285
((pfx && pfx->getCont() &&
2286
TESTAFF(pfx->getCont(), compoundmiddle,
2287
pfx->getContLen())) ||
2288
(sfx && sfx->getCont() &&
2289
TESTAFF(sfx->getCont(), compoundmiddle,
2290
sfx->getContLen())))) {
2294
// check forbiddenwords
2295
if ((rv) && (rv->astr) && (TESTAFF(rv->astr, forbiddenword, rv->alen)
2296
|| TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen))) continue;
2298
// increment word number, if the second root has a compoundroot flag
2299
if ((rv) && (compoundroot) &&
2300
(TESTAFF(rv->astr, compoundroot, rv->alen))) {
2304
// first word is acceptable in compound words?
2306
( checked_prefix || (words && words[wnum]) ||
2307
(compoundflag && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
2308
((oldwordnum == 0) && compoundbegin && TESTAFF(rv->astr, compoundbegin, rv->alen)) ||
2309
((oldwordnum > 0) && compoundmiddle && TESTAFF(rv->astr, compoundmiddle, rv->alen))
2310
// LANG_hu section: spec. Hungarian rule
2311
|| ((langnum == LANG_hu) && // hu_mov_rule
2313
TESTAFF(rv->astr, 'F', rv->alen) ||
2314
TESTAFF(rv->astr, 'G', rv->alen) ||
2315
TESTAFF(rv->astr, 'H', rv->alen)
2318
// END of LANG_hu section
2320
&& ! (( checkcompoundtriple && !words && // test triple letters
2321
(word[i-1]==word[i]) && (
2322
((i>1) && (word[i-1]==word[i-2])) ||
2323
((word[i-1]==word[i+1])) // may be word[i+1] == '\0'
2327
// test CHECKCOMPOUNDPATTERN
2328
numcheckcpd && !words && cpdpat_check(word, i, rv, NULL, affixed)
2331
checkcompoundcase && !words && cpdcase_check(word, i)
2334
// LANG_hu section: spec. Hungarian rule
2335
|| ((!rv) && (langnum == LANG_hu) && hu_mov_rule && (rv = affix_check(st,i)) &&
2336
(sfx && sfx->getCont() && (
2337
TESTAFF(sfx->getCont(), (unsigned short) 'x', sfx->getContLen()) ||
2338
TESTAFF(sfx->getCont(), (unsigned short) '%', sfx->getContLen())
2342
// END of LANG_hu section
2345
// LANG_hu section: spec. Hungarian rule
2346
if (langnum == LANG_hu) {
2347
// calculate syllable number of the word
2348
numsyllable += get_syllable(st, i);
2350
// + 1 word, if syllable number of the prefix > 1 (hungarian convention)
2351
if (pfx && (get_syllable(pfx->getKey(),strlen(pfx->getKey())) > 1)) wordnum++;
2353
// END of LANG_hu section
2357
rv = lookup((word+i)); // perhaps without prefix
2359
// search homonym with compound flag
2360
while ((rv) && ((needaffix && TESTAFF(rv->astr, needaffix, rv->alen)) ||
2361
!((compoundflag && !words && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
2362
(compoundend && !words && TESTAFF(rv->astr, compoundend, rv->alen)) ||
2363
(numdefcpd && words && defcpd_check(&words, wnum + 1, rv, NULL,1))))) {
2364
rv = rv->next_homonym;
2367
if (rv && words && words[wnum + 1]) {
2368
mystrcat(*result, presult, MAXLNLEN);
2369
mystrcat(*result, " ", MAXLNLEN);
2370
mystrcat(*result, MORPH_PART, MAXLNLEN);
2371
mystrcat(*result, word+i, MAXLNLEN);
2372
if (complexprefixes && HENTRY_DATA(rv)) mystrcat(*result, HENTRY_DATA2(rv), MAXLNLEN);
2373
if (!HENTRY_FIND(rv, MORPH_STEM)) {
2374
mystrcat(*result, " ", MAXLNLEN);
2375
mystrcat(*result, MORPH_STEM, MAXLNLEN);
2376
mystrcat(*result, HENTRY_WORD(rv), MAXLNLEN);
2378
// store the pointer of the hash entry
2379
// sprintf(*result + strlen(*result), " %s%p", MORPH_HENTRY, rv);
2380
if (!complexprefixes && HENTRY_DATA(rv)) {
2381
mystrcat(*result, " ", MAXLNLEN);
2382
mystrcat(*result, HENTRY_DATA2(rv), MAXLNLEN);
2384
mystrcat(*result, "\n", MAXLNLEN);
2388
oldnumsyllable2 = numsyllable;
2389
oldwordnum2 = wordnum;
2391
// LANG_hu section: spec. Hungarian rule
2392
if ((rv) && (langnum == LANG_hu) && (TESTAFF(rv->astr, 'I', rv->alen)) && !(TESTAFF(rv->astr, 'J', rv->alen))) {
2395
// END of LANG_hu section
2396
// increment word number, if the second root has a compoundroot flag
2397
if ((rv) && (compoundroot) &&
2398
(TESTAFF(rv->astr, compoundroot, rv->alen))) {
2402
// check forbiddenwords
2403
if ((rv) && (rv->astr) && (TESTAFF(rv->astr, forbiddenword, rv->alen) ||
2404
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen))) {
2409
// second word is acceptable, as a root?
2410
// hungarian conventions: compounding is acceptable,
2411
// when compound forms consist of 2 words, or if more,
2412
// then the syllable number of root words must be 6, or lesser.
2414
(compoundflag && TESTAFF(rv->astr, compoundflag, rv->alen)) ||
2415
(compoundend && TESTAFF(rv->astr, compoundend, rv->alen))
2418
((cpdwordmax==-1) || (wordnum+1<cpdwordmax)) ||
2419
((cpdmaxsyllable!=0) &&
2420
(numsyllable+get_syllable(HENTRY_WORD(rv),rv->blen)<=cpdmaxsyllable))
2423
(!checkcompounddup || (rv != rv_first))
2427
// bad compound word
2428
mystrcat(*result, presult, MAXLNLEN);
2429
mystrcat(*result, " ", MAXLNLEN);
2430
mystrcat(*result, MORPH_PART, MAXLNLEN);
2431
mystrcat(*result, word+i, MAXLNLEN);
2433
if (HENTRY_DATA(rv)) {
2434
if (complexprefixes) mystrcat(*result, HENTRY_DATA2(rv), MAXLNLEN);
2435
if (! HENTRY_FIND(rv, MORPH_STEM)) {
2436
mystrcat(*result, " ", MAXLNLEN);
2437
mystrcat(*result, MORPH_STEM, MAXLNLEN);
2438
mystrcat(*result, HENTRY_WORD(rv), MAXLNLEN);
2440
// store the pointer of the hash entry
2441
// sprintf(*result + strlen(*result), " %s%p", MORPH_HENTRY, rv);
2442
if (!complexprefixes) {
2443
mystrcat(*result, " ", MAXLNLEN);
2444
mystrcat(*result, HENTRY_DATA2(rv), MAXLNLEN);
2447
mystrcat(*result, "\n", MAXLNLEN);
2451
numsyllable = oldnumsyllable2 ;
2452
wordnum = oldwordnum2;
2454
// perhaps second word has prefix or/and suffix
2456
sfxflag = FLAG_NULL;
2458
if (compoundflag && !onlycpdrule) rv = affix_check((word+i),strlen(word+i), compoundflag); else rv = NULL;
2460
if (!rv && compoundend && !onlycpdrule) {
2463
rv = affix_check((word+i),strlen(word+i), compoundend);
2466
if (!rv && numdefcpd && words) {
2467
rv = affix_check((word+i),strlen(word+i), 0, IN_CPD_END);
2468
if (rv && words && defcpd_check(&words, wnum + 1, rv, NULL, 1)) {
2470
if (compoundflag) m = affix_check_morph((word+i),strlen(word+i), compoundflag);
2471
if ((!m || *m == '\0') && compoundend) {
2473
m = affix_check_morph((word+i),strlen(word+i), compoundend);
2475
mystrcat(*result, presult, MAXLNLEN);
2476
if (m || (*m != '\0')) {
2478
sprintf(m2, "%c%s%s%s", MSEP_FLD,
2479
MORPH_PART, word + i, line_uniq_app(&m, MSEP_REC));
2480
mystrcat(*result, m2, MAXLNLEN);
2483
mystrcat(*result, "\n", MAXLNLEN);
2488
// check non_compound flag in suffix and prefix
2490
((pfx && pfx->getCont() &&
2491
TESTAFF(pfx->getCont(), compoundforbidflag,
2492
pfx->getContLen())) ||
2493
(sfx && sfx->getCont() &&
2494
TESTAFF(sfx->getCont(), compoundforbidflag,
2495
sfx->getContLen())))) {
2499
// check forbiddenwords
2500
if ((rv) && (rv->astr) && (TESTAFF(rv->astr,forbiddenword,rv->alen) ||
2501
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen))
2502
&& (! TESTAFF(rv->astr, needaffix, rv->alen))) {
2507
if (langnum == LANG_hu) {
2508
// calculate syllable number of the word
2509
numsyllable += get_syllable(word + i, strlen(word + i));
2511
// - affix syllable num.
2512
// XXX only second suffix (inflections, not derivations)
2514
char * tmp = myrevstrdup(sfxappnd);
2515
numsyllable -= get_syllable(tmp, strlen(tmp));
2519
// + 1 word, if syllable number of the prefix > 1 (hungarian convention)
2520
if (pfx && (get_syllable(pfx->getKey(),strlen(pfx->getKey())) > 1)) wordnum++;
2522
// increment syllable num, if last word has a SYLLABLENUM flag
2523
// and the suffix is beginning `s'
2525
if (cpdsyllablenum) {
2527
case 'c': { numsyllable+=2; break; }
2528
case 'J': { numsyllable += 1; break; }
2529
case 'I': { if (rv && TESTAFF(rv->astr, 'J', rv->alen)) numsyllable += 1; break; }
2534
// increment word number, if the second word has a compoundroot flag
2535
if ((rv) && (compoundroot) &&
2536
(TESTAFF(rv->astr, compoundroot, rv->alen))) {
2539
// second word is acceptable, as a word with prefix or/and suffix?
2540
// hungarian conventions: compounding is acceptable,
2541
// when compound forms consist 2 word, otherwise
2542
// the syllable number of root words is 6, or lesser.
2545
((cpdwordmax==-1) || (wordnum+1<cpdwordmax)) ||
2546
((cpdmaxsyllable!=0) &&
2547
(numsyllable <= cpdmaxsyllable))
2550
(!checkcompounddup || (rv != rv_first))
2553
if (compoundflag) m = affix_check_morph((word+i),strlen(word+i), compoundflag);
2554
if ((!m || *m == '\0') && compoundend) {
2556
m = affix_check_morph((word+i),strlen(word+i), compoundend);
2558
mystrcat(*result, presult, MAXLNLEN);
2559
if (m && (*m != '\0')) {
2561
sprintf(m2, "%c%s%s%s", MSEP_FLD,
2562
MORPH_PART, word + i, line_uniq_app(&m, MSEP_REC));
2563
mystrcat(*result, m2, MAXLNLEN);
2566
if (strlen(*result) + 1 < MAXLNLEN) sprintf(*result + strlen(*result), "%c", MSEP_REC);
2570
numsyllable = oldnumsyllable2;
2571
wordnum = oldwordnum2;
2573
// perhaps second word is a compound word (recursive call)
2574
if ((wordnum < maxwordnum) && (ok == 0)) {
2575
compound_check_morph((word+i),strlen(word+i), wordnum+1,
2576
numsyllable, maxwordnum, wnum + 1, words, 0, result, presult);
2582
wordnum = oldwordnum;
2583
numsyllable = oldnumsyllable;
2585
} while (numdefcpd && oldwordnum == 0 && onlycpdrule++ < 1); // end of onlycpd loop
2591
// return 1 if s1 (reversed) is a leading subset of end of s2
2592
/* inline int AffixMgr::isRevSubset(const char * s1, const char * end_of_s2, int len)
2594
while ((len > 0) && *s1 && (*s1 == *end_of_s2)) {
2599
return (*s1 == '\0');
2603
inline int AffixMgr::isRevSubset(const char * s1, const char * end_of_s2, int len)
2605
while ((len > 0) && (*s1 != '\0') && ((*s1 == *end_of_s2) || (*s1 == '.'))) {
2610
return (*s1 == '\0');
2613
// check word for suffixes
2615
struct hentry * AffixMgr::suffix_check (const char * word, int len,
2616
int sfxopts, PfxEntry * ppfx, char ** wlst, int maxSug, int * ns,
2617
const FLAG cclass, const FLAG needflag, char in_compound)
2619
struct hentry * rv = NULL;
2620
PfxEntry* ep = ppfx;
2622
// first handle the special case of 0 length suffixes
2623
SfxEntry * se = sStart[0];
2626
if (!cclass || se->getCont()) {
2627
// suffixes are not allowed in beginning of compounds
2628
if ((((in_compound != IN_CPD_BEGIN)) || // && !cclass
2629
// except when signed with compoundpermitflag flag
2630
(se->getCont() && compoundpermitflag &&
2631
TESTAFF(se->getCont(),compoundpermitflag,se->getContLen()))) && (!circumfix ||
2632
// no circumfix flag in prefix and suffix
2633
((!ppfx || !(ep->getCont()) || !TESTAFF(ep->getCont(),
2634
circumfix, ep->getContLen())) &&
2635
(!se->getCont() || !(TESTAFF(se->getCont(),circumfix,se->getContLen())))) ||
2636
// circumfix flag in prefix AND suffix
2637
((ppfx && (ep->getCont()) && TESTAFF(ep->getCont(),
2638
circumfix, ep->getContLen())) &&
2639
(se->getCont() && (TESTAFF(se->getCont(),circumfix,se->getContLen()))))) &&
2642
!(se->getCont() && (TESTAFF(se->getCont(), onlyincompound, se->getContLen())))) &&
2643
// needaffix on prefix or first suffix
2645
!(se->getCont() && TESTAFF(se->getCont(), needaffix, se->getContLen())) ||
2646
(ppfx && !((ep->getCont()) &&
2647
TESTAFF(ep->getCont(), needaffix,
2650
rv = se->checkword(word,len, sfxopts, ppfx, wlst, maxSug, ns, (FLAG) cclass,
2651
needflag, (in_compound ? 0 : onlyincompound));
2653
sfx=se; // BUG: sfx not stateless
2661
// now handle the general case
2662
if (len == 0) return NULL; // FULLSTRIP
2663
unsigned char sp= *((const unsigned char *)(word + len - 1));
2664
SfxEntry * sptr = sStart[sp];
2667
if (isRevSubset(sptr->getKey(), word + len - 1, len)
2669
// suffixes are not allowed in beginning of compounds
2670
if ((((in_compound != IN_CPD_BEGIN)) || // && !cclass
2671
// except when signed with compoundpermitflag flag
2672
(sptr->getCont() && compoundpermitflag &&
2673
TESTAFF(sptr->getCont(),compoundpermitflag,sptr->getContLen()))) && (!circumfix ||
2674
// no circumfix flag in prefix and suffix
2675
((!ppfx || !(ep->getCont()) || !TESTAFF(ep->getCont(),
2676
circumfix, ep->getContLen())) &&
2677
(!sptr->getCont() || !(TESTAFF(sptr->getCont(),circumfix,sptr->getContLen())))) ||
2678
// circumfix flag in prefix AND suffix
2679
((ppfx && (ep->getCont()) && TESTAFF(ep->getCont(),
2680
circumfix, ep->getContLen())) &&
2681
(sptr->getCont() && (TESTAFF(sptr->getCont(),circumfix,sptr->getContLen()))))) &&
2684
!((sptr->getCont() && (TESTAFF(sptr->getCont(), onlyincompound, sptr->getContLen()))))) &&
2685
// needaffix on prefix or first suffix
2687
!(sptr->getCont() && TESTAFF(sptr->getCont(), needaffix, sptr->getContLen())) ||
2688
(ppfx && !((ep->getCont()) &&
2689
TESTAFF(ep->getCont(), needaffix,
2692
) if (in_compound != IN_CPD_END || ppfx || !(sptr->getCont() && TESTAFF(sptr->getCont(), onlyincompound, sptr->getContLen()))) {
2693
rv = sptr->checkword(word,len, sfxopts, ppfx, wlst,
2694
maxSug, ns, cclass, needflag, (in_compound ? 0 : onlyincompound));
2696
sfx=sptr; // BUG: sfx not stateless
2697
sfxflag = sptr->getFlag(); // BUG: sfxflag not stateless
2698
if (!sptr->getCont()) sfxappnd=sptr->getKey(); // BUG: sfxappnd not stateless
2702
sptr = sptr->getNextEQ();
2704
sptr = sptr->getNextNE();
2711
// check word for two-level suffixes
2713
struct hentry * AffixMgr::suffix_check_twosfx(const char * word, int len,
2714
int sfxopts, PfxEntry * ppfx, const FLAG needflag)
2716
struct hentry * rv = NULL;
2718
// first handle the special case of 0 length suffixes
2719
SfxEntry * se = sStart[0];
2721
if (contclasses[se->getFlag()])
2723
rv = se->check_twosfx(word,len, sfxopts, ppfx, needflag);
2729
// now handle the general case
2730
if (len == 0) return NULL; // FULLSTRIP
2731
unsigned char sp = *((const unsigned char *)(word + len - 1));
2732
SfxEntry * sptr = sStart[sp];
2735
if (isRevSubset(sptr->getKey(), word + len - 1, len)) {
2736
if (contclasses[sptr->getFlag()])
2738
rv = sptr->check_twosfx(word,len, sfxopts, ppfx, needflag);
2740
sfxflag = sptr->getFlag(); // BUG: sfxflag not stateless
2741
if (!sptr->getCont()) sfxappnd=sptr->getKey(); // BUG: sfxappnd not stateless
2745
sptr = sptr->getNextEQ();
2747
sptr = sptr->getNextNE();
2754
char * AffixMgr::suffix_check_twosfx_morph(const char * word, int len,
2755
int sfxopts, PfxEntry * ppfx, const FLAG needflag)
2758
std::string result2;
2759
std::string result3;
2763
// first handle the special case of 0 length suffixes
2764
SfxEntry * se = sStart[0];
2766
if (contclasses[se->getFlag()])
2768
st = se->check_twosfx_morph(word,len, sfxopts, ppfx, needflag);
2771
if (ppfx->getMorph()) {
2772
result.append(ppfx->getMorph());
2774
} else debugflag(result, ppfx->getFlag());
2778
if (se->getMorph()) {
2780
result.append(se->getMorph());
2781
} else debugflag(result, se->getFlag());
2782
result.append("\n");
2788
// now handle the general case
2789
if (len == 0) return NULL; // FULLSTRIP
2790
unsigned char sp = *((const unsigned char *)(word + len - 1));
2791
SfxEntry * sptr = sStart[sp];
2794
if (isRevSubset(sptr->getKey(), word + len - 1, len)) {
2795
if (contclasses[sptr->getFlag()])
2797
st = sptr->check_twosfx_morph(word,len, sfxopts, ppfx, needflag);
2799
sfxflag = sptr->getFlag(); // BUG: sfxflag not stateless
2800
if (!sptr->getCont()) sfxappnd=sptr->getKey(); // BUG: sfxappnd not stateless
2806
if (sptr->getMorph()) {
2807
result3.append(" ");
2808
result3.append(sptr->getMorph());
2809
} else debugflag(result3, sptr->getFlag());
2810
strlinecat(result2, result3);
2811
result2.append("\n");
2812
result.append(result2);
2815
sptr = sptr->getNextEQ();
2817
sptr = sptr->getNextNE();
2821
if (!result.empty())
2822
return mystrdup(result.c_str());
2827
char * AffixMgr::suffix_check_morph(const char * word, int len,
2828
int sfxopts, PfxEntry * ppfx, const FLAG cclass, const FLAG needflag, char in_compound)
2830
char result[MAXLNLEN];
2832
struct hentry * rv = NULL;
2836
PfxEntry* ep = ppfx;
2838
// first handle the special case of 0 length suffixes
2839
SfxEntry * se = sStart[0];
2841
if (!cclass || se->getCont()) {
2842
// suffixes are not allowed in beginning of compounds
2843
if (((((in_compound != IN_CPD_BEGIN)) || // && !cclass
2844
// except when signed with compoundpermitflag flag
2845
(se->getCont() && compoundpermitflag &&
2846
TESTAFF(se->getCont(),compoundpermitflag,se->getContLen()))) && (!circumfix ||
2847
// no circumfix flag in prefix and suffix
2848
((!ppfx || !(ep->getCont()) || !TESTAFF(ep->getCont(),
2849
circumfix, ep->getContLen())) &&
2850
(!se->getCont() || !(TESTAFF(se->getCont(),circumfix,se->getContLen())))) ||
2851
// circumfix flag in prefix AND suffix
2852
((ppfx && (ep->getCont()) && TESTAFF(ep->getCont(),
2853
circumfix, ep->getContLen())) &&
2854
(se->getCont() && (TESTAFF(se->getCont(),circumfix,se->getContLen()))))) &&
2857
!((se->getCont() && (TESTAFF(se->getCont(), onlyincompound, se->getContLen()))))) &&
2858
// needaffix on prefix or first suffix
2860
!(se->getCont() && TESTAFF(se->getCont(), needaffix, se->getContLen())) ||
2861
(ppfx && !((ep->getCont()) &&
2862
TESTAFF(ep->getCont(), needaffix,
2866
rv = se->checkword(word, len, sfxopts, ppfx, NULL, 0, 0, cclass, needflag);
2869
if (ppfx->getMorph()) {
2870
mystrcat(result, ppfx->getMorph(), MAXLNLEN);
2871
mystrcat(result, " ", MAXLNLEN);
2872
} else debugflag(result, ppfx->getFlag());
2874
if (complexprefixes && HENTRY_DATA(rv)) mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
2875
if (! HENTRY_FIND(rv, MORPH_STEM)) {
2876
mystrcat(result, " ", MAXLNLEN);
2877
mystrcat(result, MORPH_STEM, MAXLNLEN);
2878
mystrcat(result, HENTRY_WORD(rv), MAXLNLEN);
2880
// store the pointer of the hash entry
2881
// sprintf(result + strlen(result), " %s%p", MORPH_HENTRY, rv);
2883
if (!complexprefixes && HENTRY_DATA(rv)) {
2884
mystrcat(result, " ", MAXLNLEN);
2885
mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
2887
if (se->getMorph()) {
2888
mystrcat(result, " ", MAXLNLEN);
2889
mystrcat(result, se->getMorph(), MAXLNLEN);
2890
} else debugflag(result, se->getFlag());
2891
mystrcat(result, "\n", MAXLNLEN);
2892
rv = se->get_next_homonym(rv, sfxopts, ppfx, cclass, needflag);
2898
// now handle the general case
2899
if (len == 0) return NULL; // FULLSTRIP
2900
unsigned char sp = *((const unsigned char *)(word + len - 1));
2901
SfxEntry * sptr = sStart[sp];
2904
if (isRevSubset(sptr->getKey(), word + len - 1, len)
2906
// suffixes are not allowed in beginning of compounds
2907
if (((((in_compound != IN_CPD_BEGIN)) || // && !cclass
2908
// except when signed with compoundpermitflag flag
2909
(sptr->getCont() && compoundpermitflag &&
2910
TESTAFF(sptr->getCont(),compoundpermitflag,sptr->getContLen()))) && (!circumfix ||
2911
// no circumfix flag in prefix and suffix
2912
((!ppfx || !(ep->getCont()) || !TESTAFF(ep->getCont(),
2913
circumfix, ep->getContLen())) &&
2914
(!sptr->getCont() || !(TESTAFF(sptr->getCont(),circumfix,sptr->getContLen())))) ||
2915
// circumfix flag in prefix AND suffix
2916
((ppfx && (ep->getCont()) && TESTAFF(ep->getCont(),
2917
circumfix, ep->getContLen())) &&
2918
(sptr->getCont() && (TESTAFF(sptr->getCont(),circumfix,sptr->getContLen()))))) &&
2921
!((sptr->getCont() && (TESTAFF(sptr->getCont(), onlyincompound, sptr->getContLen()))))) &&
2922
// needaffix on first suffix
2923
(cclass || !(sptr->getCont() &&
2924
TESTAFF(sptr->getCont(), needaffix, sptr->getContLen())))
2925
)) rv = sptr->checkword(word,len, sfxopts, ppfx, NULL, 0, 0, cclass, needflag);
2928
if (ppfx->getMorph()) {
2929
mystrcat(result, ppfx->getMorph(), MAXLNLEN);
2930
mystrcat(result, " ", MAXLNLEN);
2931
} else debugflag(result, ppfx->getFlag());
2933
if (complexprefixes && HENTRY_DATA(rv)) mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
2934
if (! HENTRY_FIND(rv, MORPH_STEM)) {
2935
mystrcat(result, " ", MAXLNLEN);
2936
mystrcat(result, MORPH_STEM, MAXLNLEN);
2937
mystrcat(result, HENTRY_WORD(rv), MAXLNLEN);
2939
// store the pointer of the hash entry
2940
// sprintf(result + strlen(result), " %s%p", MORPH_HENTRY, rv);
2942
if (!complexprefixes && HENTRY_DATA(rv)) {
2943
mystrcat(result, " ", MAXLNLEN);
2944
mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
2947
if (sptr->getMorph()) {
2948
mystrcat(result, " ", MAXLNLEN);
2949
mystrcat(result, sptr->getMorph(), MAXLNLEN);
2950
} else debugflag(result, sptr->getFlag());
2951
mystrcat(result, "\n", MAXLNLEN);
2952
rv = sptr->get_next_homonym(rv, sfxopts, ppfx, cclass, needflag);
2954
sptr = sptr->getNextEQ();
2956
sptr = sptr->getNextNE();
2960
if (*result) return mystrdup(result);
2964
// check if word with affixes is correctly spelled
2965
struct hentry * AffixMgr::affix_check (const char * word, int len, const FLAG needflag, char in_compound)
2967
struct hentry * rv= NULL;
2969
// check all prefixes (also crossed with suffixes if allowed)
2970
rv = prefix_check(word, len, in_compound, needflag);
2973
// if still not found check all suffixes
2974
rv = suffix_check(word, len, 0, NULL, NULL, 0, NULL, FLAG_NULL, needflag, in_compound);
2976
if (havecontclass) {
2981
// if still not found check all two-level suffixes
2982
rv = suffix_check_twosfx(word, len, 0, NULL, needflag);
2985
// if still not found check all two-level suffixes
2986
rv = prefix_check_twosfx(word, len, IN_CPD_NOT, needflag);
2992
// check if word with affixes is correctly spelled
2993
char * AffixMgr::affix_check_morph(const char * word, int len, const FLAG needflag, char in_compound)
2995
char result[MAXLNLEN];
3000
// check all prefixes (also crossed with suffixes if allowed)
3001
st = prefix_check_morph(word, len, in_compound);
3003
mystrcat(result, st, MAXLNLEN);
3007
// if still not found check all suffixes
3008
st = suffix_check_morph(word, len, 0, NULL, '\0', needflag, in_compound);
3010
mystrcat(result, st, MAXLNLEN);
3014
if (havecontclass) {
3017
// if still not found check all two-level suffixes
3018
st = suffix_check_twosfx_morph(word, len, 0, NULL, needflag);
3020
mystrcat(result, st, MAXLNLEN);
3024
// if still not found check all two-level suffixes
3025
st = prefix_check_twosfx_morph(word, len, IN_CPD_NOT, needflag);
3027
mystrcat(result, st, MAXLNLEN);
3032
return mystrdup(result);
3035
char * AffixMgr::morphgen(const char * ts, int wl, const unsigned short * ap,
3036
unsigned short al, const char * morph, const char * targetmorph, int level)
3039
if (!morph) return NULL;
3041
// check substandard flag
3042
if (TESTAFF(ap, substandard, al)) return NULL;
3044
if (morphcmp(morph, targetmorph) == 0) return mystrdup(ts);
3046
size_t stemmorphcatpos;
3047
std::string mymorph;
3049
// use input suffix fields, if exist
3050
if (strstr(morph, MORPH_INFL_SFX) || strstr(morph, MORPH_DERI_SFX)) {
3051
mymorph.assign(morph);
3052
mymorph.append(" ");
3053
stemmorphcatpos = mymorph.size();
3055
stemmorphcatpos = std::string::npos;
3058
for (int i = 0; i < al; i++) {
3059
const unsigned char c = (unsigned char) (ap[i] & 0x00FF);
3060
SfxEntry * sptr = sFlag[c];
3062
if (sptr->getFlag() == ap[i] && sptr->getMorph() && ((sptr->getContLen() == 0) ||
3063
// don't generate forms with substandard affixes
3064
!TESTAFF(sptr->getCont(), substandard, sptr->getContLen()))) {
3066
const char * stemmorph;
3067
if (stemmorphcatpos != std::string::npos) {
3068
mymorph.replace(stemmorphcatpos, std::string::npos, sptr->getMorph());
3069
stemmorph = mymorph.c_str();
3072
stemmorph = sptr->getMorph();
3075
int cmp = morphcmp(stemmorph, targetmorph);
3078
char * newword = sptr->add(ts, wl);
3080
hentry * check = pHMgr->lookup(newword); // XXX extra dic
3081
if (!check || !check->astr ||
3082
!(TESTAFF(check->astr, forbiddenword, check->alen) ||
3083
TESTAFF(check->astr, ONLYUPCASEFLAG, check->alen))) {
3090
// recursive call for secondary suffixes
3091
if ((level == 0) && (cmp == 1) && (sptr->getContLen() > 0) &&
3092
// (get_sfxcount(stemmorph) < targetcount) &&
3093
!TESTAFF(sptr->getCont(), substandard, sptr->getContLen())) {
3094
char * newword = sptr->add(ts, wl);
3096
char * newword2 = morphgen(newword, strlen(newword), sptr->getCont(),
3097
sptr->getContLen(), stemmorph, targetmorph, 1);
3108
sptr = sptr->getFlgNxt();
3114
int AffixMgr::expand_rootword(struct guessword * wlst, int maxn, const char * ts,
3115
int wl, const unsigned short * ap, unsigned short al, const char * bad, int badl,
3119
// first add root word to list
3120
if ((nh < maxn) && !(al && ((needaffix && TESTAFF(ap, needaffix, al)) ||
3121
(onlyincompound && TESTAFF(ap, onlyincompound, al))))) {
3122
wlst[nh].word = mystrdup(ts);
3123
if (!wlst[nh].word) return 0;
3124
wlst[nh].allow = (1 == 0);
3125
wlst[nh].orig = NULL;
3127
// add special phonetic version
3128
if (phon && (nh < maxn)) {
3129
wlst[nh].word = mystrdup(phon);
3130
if (!wlst[nh].word) return nh - 1;
3131
wlst[nh].allow = (1 == 0);
3132
wlst[nh].orig = mystrdup(ts);
3133
if (!wlst[nh].orig) return nh - 1;
3139
for (int i = 0; i < al; i++) {
3140
const unsigned char c = (unsigned char) (ap[i] & 0x00FF);
3141
SfxEntry * sptr = sFlag[c];
3143
if ((sptr->getFlag() == ap[i]) && (!sptr->getKeyLen() || ((badl > sptr->getKeyLen()) &&
3144
(strcmp(sptr->getAffix(), bad + badl - sptr->getKeyLen()) == 0))) &&
3145
// check needaffix flag
3146
!(sptr->getCont() && ((needaffix &&
3147
TESTAFF(sptr->getCont(), needaffix, sptr->getContLen())) ||
3149
TESTAFF(sptr->getCont(), circumfix, sptr->getContLen())) ||
3151
TESTAFF(sptr->getCont(), onlyincompound, sptr->getContLen()))))
3153
char * newword = sptr->add(ts, wl);
3156
wlst[nh].word = newword;
3157
wlst[nh].allow = sptr->allowCross();
3158
wlst[nh].orig = NULL;
3160
// add special phonetic version
3161
if (phon && (nh < maxn)) {
3162
std::string prefix(phon);
3163
std::string key(sptr->getKey());
3166
wlst[nh].word = mystrdup(prefix.c_str());
3167
if (!wlst[nh].word) return nh - 1;
3168
wlst[nh].allow = (1 == 0);
3169
wlst[nh].orig = mystrdup(newword);
3170
if (!wlst[nh].orig) return nh - 1;
3178
sptr = sptr->getFlgNxt();
3184
// handle cross products of prefixes and suffixes
3185
for (int j=1;j<n ;j++)
3186
if (wlst[j].allow) {
3187
for (int k = 0; k < al; k++) {
3188
const unsigned char c = (unsigned char) (ap[k] & 0x00FF);
3189
PfxEntry * cptr = pFlag[c];
3191
if ((cptr->getFlag() == ap[k]) && cptr->allowCross() && (!cptr->getKeyLen() || ((badl > cptr->getKeyLen()) &&
3192
(strncmp(cptr->getKey(), bad, cptr->getKeyLen()) == 0)))) {
3193
int l1 = strlen(wlst[j].word);
3194
char * newword = cptr->add(wlst[j].word, l1);
3197
wlst[nh].word = newword;
3198
wlst[nh].allow = cptr->allowCross();
3199
wlst[nh].orig = NULL;
3206
cptr = cptr->getFlgNxt();
3212
// now handle pure prefixes
3213
for (int m = 0; m < al; m ++) {
3214
const unsigned char c = (unsigned char) (ap[m] & 0x00FF);
3215
PfxEntry * ptr = pFlag[c];
3217
if ((ptr->getFlag() == ap[m]) && (!ptr->getKeyLen() || ((badl > ptr->getKeyLen()) &&
3218
(strncmp(ptr->getKey(), bad, ptr->getKeyLen()) == 0))) &&
3219
// check needaffix flag
3220
!(ptr->getCont() && ((needaffix &&
3221
TESTAFF(ptr->getCont(), needaffix, ptr->getContLen())) ||
3223
TESTAFF(ptr->getCont(), circumfix, ptr->getContLen())) ||
3225
TESTAFF(ptr->getCont(), onlyincompound, ptr->getContLen()))))
3227
char * newword = ptr->add(ts, wl);
3230
wlst[nh].word = newword;
3231
wlst[nh].allow = ptr->allowCross();
3232
wlst[nh].orig = NULL;
3239
ptr = ptr->getFlgNxt();
3246
// return length of replacing table
3247
int AffixMgr::get_numrep() const
3252
// return replacing table
3253
struct replentry * AffixMgr::get_reptable() const
3255
if (! reptable ) return NULL;
3259
// return iconv table
3260
RepList * AffixMgr::get_iconvtable() const
3262
if (! iconvtable ) return NULL;
3266
// return oconv table
3267
RepList * AffixMgr::get_oconvtable() const
3269
if (! oconvtable ) return NULL;
3273
// return replacing table
3274
struct phonetable * AffixMgr::get_phonetable() const
3276
if (! phone ) return NULL;
3280
// return length of character map table
3281
int AffixMgr::get_nummap() const
3286
// return character map table
3287
struct mapentry * AffixMgr::get_maptable() const
3289
if (! maptable ) return NULL;
3293
// return length of word break table
3294
int AffixMgr::get_numbreak() const
3299
// return character map table
3300
char ** AffixMgr::get_breaktable() const
3302
if (! breaktable ) return NULL;
3306
// return text encoding of dictionary
3307
char * AffixMgr::get_encoding()
3309
if (! encoding ) encoding = mystrdup(SPELL_ENCODING);
3310
return mystrdup(encoding);
3313
// return text encoding of dictionary
3314
int AffixMgr::get_langnum() const
3319
// return double prefix option
3320
int AffixMgr::get_complexprefixes() const
3322
return complexprefixes;
3325
// return FULLSTRIP option
3326
int AffixMgr::get_fullstrip() const
3331
FLAG AffixMgr::get_keepcase() const
3336
FLAG AffixMgr::get_forceucase() const
3341
FLAG AffixMgr::get_warn() const
3346
int AffixMgr::get_forbidwarn() const
3351
int AffixMgr::get_checksharps() const
3356
char * AffixMgr::encode_flag(unsigned short aflag) const
3358
return pHMgr->encode_flag(aflag);
3362
// return the preferred ignore string for suggestions
3363
char * AffixMgr::get_ignore() const
3365
if (!ignorechars) return NULL;
3369
// return the preferred ignore string for suggestions
3370
unsigned short * AffixMgr::get_ignore_utf16(int * len) const
3372
*len = ignorechars_utf16_len;
3373
return ignorechars_utf16;
3376
// return the keyboard string for suggestions
3377
char * AffixMgr::get_key_string()
3379
if (! keystring ) keystring = mystrdup(SPELL_KEYSTRING);
3380
return mystrdup(keystring);
3383
// return the preferred try string for suggestions
3384
char * AffixMgr::get_try_string() const
3386
if (! trystring ) return NULL;
3387
return mystrdup(trystring);
3390
// return the preferred try string for suggestions
3391
const char * AffixMgr::get_wordchars() const
3396
unsigned short * AffixMgr::get_wordchars_utf16(int * len) const
3398
*len = wordchars_utf16_len;
3399
return wordchars_utf16;
3402
// is there compounding?
3403
int AffixMgr::get_compound() const
3405
return compoundflag || compoundbegin || numdefcpd;
3408
// return the compound words control flag
3409
FLAG AffixMgr::get_compoundflag() const
3411
return compoundflag;
3414
// return the forbidden words control flag
3415
FLAG AffixMgr::get_forbiddenword() const
3417
return forbiddenword;
3420
// return the forbidden words control flag
3421
FLAG AffixMgr::get_nosuggest() const
3426
// return the forbidden words control flag
3427
FLAG AffixMgr::get_nongramsuggest() const
3429
return nongramsuggest;
3432
// return the forbidden words flag modify flag
3433
FLAG AffixMgr::get_needaffix() const
3438
// return the onlyincompound flag
3439
FLAG AffixMgr::get_onlyincompound() const
3441
return onlyincompound;
3444
// return the compound word signal flag
3445
FLAG AffixMgr::get_compoundroot() const
3447
return compoundroot;
3450
// return the compound begin signal flag
3451
FLAG AffixMgr::get_compoundbegin() const
3453
return compoundbegin;
3456
// return the value of checknum
3457
int AffixMgr::get_checknum() const
3462
// return the value of prefix
3463
const char * AffixMgr::get_prefix() const
3465
if (pfx) return pfx->getKey();
3469
// return the value of suffix
3470
const char * AffixMgr::get_suffix() const
3475
// return the value of suffix
3476
const char * AffixMgr::get_version() const
3481
// return lemma_present flag
3482
FLAG AffixMgr::get_lemma_present() const
3484
return lemma_present;
3487
// utility method to look up root words in hash table
3488
struct hentry * AffixMgr::lookup(const char * word)
3491
struct hentry * he = NULL;
3492
for (i = 0; i < *maxdic && !he; i++) {
3493
he = (alldic[i])->lookup(word);
3498
// return the value of suffix
3499
int AffixMgr::have_contclass() const
3501
return havecontclass;
3505
int AffixMgr::get_utf8() const
3510
int AffixMgr::get_maxngramsugs(void) const
3512
return maxngramsugs;
3515
int AffixMgr::get_maxcpdsugs(void) const
3520
int AffixMgr::get_maxdiff(void) const
3525
int AffixMgr::get_onlymaxdiff(void) const
3530
// return nosplitsugs
3531
int AffixMgr::get_nosplitsugs(void) const
3536
// return sugswithdots
3537
int AffixMgr::get_sugswithdots(void) const
3539
return sugswithdots;
3543
int AffixMgr::parse_flag(char * line, unsigned short * out, FileMgr * af) {
3545
if (*out != FLAG_NULL && !(*out >= DEFAULTFLAGS)) {
3546
HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of an affix file parameter\n", af->getlinenum());
3549
if (parse_string(line, &s, af->getlinenum())) return 1;
3550
*out = pHMgr->decode_flag(s);
3556
int AffixMgr::parse_num(char * line, int * out, FileMgr * af) {
3559
HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of an affix file parameter\n", af->getlinenum());
3562
if (parse_string(line, &s, af->getlinenum())) return 1;
3568
/* parse in the max syllablecount of compound words and */
3569
int AffixMgr::parse_cpdsyllable(char * line, FileMgr * af)
3575
w_char w[MAXWORDLEN];
3576
piece = mystrsep(&tp, 0);
3578
if (*piece != '\0') {
3580
case 0: { np++; break; }
3581
case 1: { cpdmaxsyllable = atoi(piece); np++; break; }
3584
cpdvowels = mystrdup(piece);
3586
int n = u8_u16(w, MAXWORDLEN, piece);
3588
flag_qsort((unsigned short *) w, 0, n);
3589
cpdvowels_utf16 = (w_char *) malloc(n * sizeof(w_char));
3590
if (!cpdvowels_utf16) return 1;
3591
memcpy(cpdvowels_utf16, w, n * sizeof(w_char));
3593
cpdvowels_utf16_len = n;
3602
piece = mystrsep(&tp, 0);
3605
HUNSPELL_WARNING(stderr, "error: line %d: missing compoundsyllable information\n", af->getlinenum());
3608
if (np == 2) cpdvowels = mystrdup("aeiouAEIOU");
3612
/* parse in the typical fault correcting table */
3613
int AffixMgr::parse_reptable(char * line, FileMgr * af)
3616
HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
3623
piece = mystrsep(&tp, 0);
3625
if (*piece != '\0') {
3627
case 0: { np++; break; }
3629
numrep = atoi(piece);
3631
HUNSPELL_WARNING(stderr, "error: line %d: incorrect entry number\n", af->getlinenum());
3634
reptable = (replentry *) malloc(numrep * sizeof(struct replentry));
3635
if (!reptable) return 1;
3643
piece = mystrsep(&tp, 0);
3646
HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
3650
/* now parse the numrep lines to read in the remainder of the table */
3652
for (int j=0; j < numrep; j++) {
3653
if ((nl = af->getline()) == NULL) return 1;
3657
reptable[j].pattern = NULL;
3658
reptable[j].pattern2 = NULL;
3659
piece = mystrsep(&tp, 0);
3661
if (*piece != '\0') {
3664
if (strncmp(piece,"REP",3) != 0) {
3665
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
3672
if (*piece == '^') reptable[j].start = true; else reptable[j].start = false;
3673
reptable[j].pattern = mystrrep(mystrdup(piece + int(reptable[j].start)),"_"," ");
3674
int lr = strlen(reptable[j].pattern) - 1;
3675
if (reptable[j].pattern[lr] == '$') {
3676
reptable[j].end = true;
3677
reptable[j].pattern[lr] = '\0';
3678
} else reptable[j].end = false;
3681
case 2: { reptable[j].pattern2 = mystrrep(mystrdup(piece),"_"," "); break; }
3686
piece = mystrsep(&tp, 0);
3688
if ((!(reptable[j].pattern)) || (!(reptable[j].pattern2))) {
3689
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
3697
/* parse in the typical fault correcting table */
3698
int AffixMgr::parse_convtable(char * line, FileMgr * af, RepList ** rl, const char * keyword)
3701
HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
3709
piece = mystrsep(&tp, 0);
3711
if (*piece != '\0') {
3713
case 0: { np++; break; }
3715
numrl = atoi(piece);
3717
HUNSPELL_WARNING(stderr, "error: line %d: incorrect entry number\n", af->getlinenum());
3720
*rl = new RepList(numrl);
3729
piece = mystrsep(&tp, 0);
3732
HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
3736
/* now parse the num lines to read in the remainder of the table */
3738
for (int j=0; j < numrl; j++) {
3739
if (!(nl = af->getline())) return 1;
3743
char * pattern = NULL;
3744
char * pattern2 = NULL;
3745
piece = mystrsep(&tp, 0);
3747
if (*piece != '\0') {
3750
if (strncmp(piece, keyword, strlen(keyword)) != 0) {
3751
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
3758
case 1: { pattern = mystrrep(mystrdup(piece),"_"," "); break; }
3760
pattern2 = mystrrep(mystrdup(piece),"_"," ");
3767
piece = mystrsep(&tp, 0);
3769
if (!pattern || !pattern2) {
3774
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
3777
(*rl)->add(pattern, pattern2);
3783
/* parse in the typical fault correcting table */
3784
int AffixMgr::parse_phonetable(char * line, FileMgr * af)
3787
HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
3794
piece = mystrsep(&tp, 0);
3796
if (*piece != '\0') {
3798
case 0: { np++; break; }
3800
phone = (phonetable *) malloc(sizeof(struct phonetable));
3801
if (!phone) return 1;
3802
phone->num = atoi(piece);
3803
phone->rules = NULL;
3804
phone->utf8 = (char) utf8;
3805
if (phone->num < 1) {
3806
HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
3809
phone->rules = (char * *) malloc(2 * (phone->num + 1) * sizeof(char *));
3810
if (!phone->rules) {
3822
piece = mystrsep(&tp, 0);
3825
HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
3829
/* now parse the phone->num lines to read in the remainder of the table */
3831
for (int j=0; j < phone->num; j++) {
3832
if (!(nl = af->getline())) return 1;
3836
phone->rules[j * 2] = NULL;
3837
phone->rules[j * 2 + 1] = NULL;
3838
piece = mystrsep(&tp, 0);
3840
if (*piece != '\0') {
3843
if (strncmp(piece,"PHONE",5) != 0) {
3844
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
3850
case 1: { phone->rules[j * 2] = mystrrep(mystrdup(piece),"_",""); break; }
3851
case 2: { phone->rules[j * 2 + 1] = mystrrep(mystrdup(piece),"_",""); break; }
3856
piece = mystrsep(&tp, 0);
3858
if ((!(phone->rules[j * 2])) || (!(phone->rules[j * 2 + 1]))) {
3859
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
3864
phone->rules[phone->num * 2] = mystrdup("");
3865
phone->rules[phone->num * 2 + 1] = mystrdup("");
3866
init_phonet_hash(*phone);
3870
/* parse in the checkcompoundpattern table */
3871
int AffixMgr::parse_checkcpdtable(char * line, FileMgr * af)
3873
if (numcheckcpd != 0) {
3874
HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
3881
piece = mystrsep(&tp, 0);
3883
if (*piece != '\0') {
3885
case 0: { np++; break; }
3887
numcheckcpd = atoi(piece);
3888
if (numcheckcpd < 1) {
3889
HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
3892
checkcpdtable = (patentry *) malloc(numcheckcpd * sizeof(struct patentry));
3893
if (!checkcpdtable) return 1;
3901
piece = mystrsep(&tp, 0);
3904
HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
3908
/* now parse the numcheckcpd lines to read in the remainder of the table */
3910
for (int j=0; j < numcheckcpd; j++) {
3911
if (!(nl = af->getline())) return 1;
3915
checkcpdtable[j].pattern = NULL;
3916
checkcpdtable[j].pattern2 = NULL;
3917
checkcpdtable[j].pattern3 = NULL;
3918
checkcpdtable[j].cond = FLAG_NULL;
3919
checkcpdtable[j].cond2 = FLAG_NULL;
3920
piece = mystrsep(&tp, 0);
3922
if (*piece != '\0') {
3925
if (strncmp(piece,"CHECKCOMPOUNDPATTERN",20) != 0) {
3926
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
3933
checkcpdtable[j].pattern = mystrdup(piece);
3934
char * p = strchr(checkcpdtable[j].pattern, '/');
3937
checkcpdtable[j].cond = pHMgr->decode_flag(p + 1);
3941
checkcpdtable[j].pattern2 = mystrdup(piece);
3942
char * p = strchr(checkcpdtable[j].pattern2, '/');
3945
checkcpdtable[j].cond2 = pHMgr->decode_flag(p + 1);
3949
case 3: { checkcpdtable[j].pattern3 = mystrdup(piece); simplifiedcpd = 1; break; }
3954
piece = mystrsep(&tp, 0);
3956
if ((!(checkcpdtable[j].pattern)) || (!(checkcpdtable[j].pattern2))) {
3957
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
3965
/* parse in the compound rule table */
3966
int AffixMgr::parse_defcpdtable(char * line, FileMgr * af)
3968
if (numdefcpd != 0) {
3969
HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
3976
piece = mystrsep(&tp, 0);
3978
if (*piece != '\0') {
3980
case 0: { np++; break; }
3982
numdefcpd = atoi(piece);
3983
if (numdefcpd < 1) {
3984
HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
3987
defcpdtable = (flagentry *) malloc(numdefcpd * sizeof(flagentry));
3988
if (!defcpdtable) return 1;
3996
piece = mystrsep(&tp, 0);
3999
HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
4003
/* now parse the numdefcpd lines to read in the remainder of the table */
4005
for (int j=0; j < numdefcpd; j++) {
4006
if (!(nl = af->getline())) return 1;
4010
defcpdtable[j].def = NULL;
4011
defcpdtable[j].len = 0;
4012
piece = mystrsep(&tp, 0);
4014
if (*piece != '\0') {
4017
if (strncmp(piece, "COMPOUNDRULE", 12) != 0) {
4018
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
4024
case 1: { // handle parenthesized flags
4025
if (strchr(piece, '(')) {
4026
defcpdtable[j].def = (FLAG *) malloc(strlen(piece) * sizeof(FLAG));
4027
defcpdtable[j].len = 0;
4031
char * par = piece + 1;
4032
while (*par != '(' && *par != ')' && *par != '\0') par++;
4033
if (*par == '\0') end = 1; else *par = '\0';
4034
if (*piece == '(') piece++;
4035
if (*piece == '*' || *piece == '?') {
4036
defcpdtable[j].def[defcpdtable[j].len++] = (FLAG) *piece;
4037
} else if (*piece != '\0') {
4038
int l = pHMgr->decode_flags(&conv, piece, af);
4039
for (int k = 0; k < l; k++) defcpdtable[j].def[defcpdtable[j].len++] = conv[k];
4045
defcpdtable[j].len = pHMgr->decode_flags(&(defcpdtable[j].def), piece, af);
4053
piece = mystrsep(&tp, 0);
4055
if (!defcpdtable[j].len) {
4056
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
4065
/* parse in the character map table */
4066
int AffixMgr::parse_maptable(char * line, FileMgr * af)
4069
HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
4076
piece = mystrsep(&tp, 0);
4078
if (*piece != '\0') {
4080
case 0: { np++; break; }
4082
nummap = atoi(piece);
4084
HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
4087
maptable = (mapentry *) malloc(nummap * sizeof(struct mapentry));
4088
if (!maptable) return 1;
4096
piece = mystrsep(&tp, 0);
4099
HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
4103
/* now parse the nummap lines to read in the remainder of the table */
4105
for (int j=0; j < nummap; j++) {
4106
if (!(nl = af->getline())) return 1;
4110
maptable[j].set = NULL;
4111
maptable[j].len = 0;
4112
piece = mystrsep(&tp, 0);
4114
if (*piece != '\0') {
4117
if (strncmp(piece,"MAP",3) != 0) {
4118
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
4126
maptable[j].len = strlen(piece);
4127
maptable[j].set = (char **) malloc(maptable[j].len * sizeof(char*));
4128
if (!maptable[j].set) return 1;
4129
for (int k = 0; k < maptable[j].len; k++) {
4132
if (piece[k] == '(') {
4133
char * parpos = strchr(piece + k, ')');
4134
if (parpos != NULL) {
4136
chl = (int)(parpos - piece) - k - 1;
4140
if (utf8 && (piece[k] & 0xc0) == 0xc0) {
4141
for (k++; utf8 && (piece[k] & 0xc0) == 0x80; k++);
4146
maptable[j].set[setn] = (char *) malloc(chl + 1);
4147
if (!maptable[j].set[setn]) return 1;
4148
strncpy(maptable[j].set[setn], piece + chb, chl);
4149
maptable[j].set[setn][chl] = '\0';
4152
maptable[j].len = setn;
4158
piece = mystrsep(&tp, 0);
4160
if (!maptable[j].set || !maptable[j].len) {
4161
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
4169
/* parse in the word breakpoint table */
4170
int AffixMgr::parse_breaktable(char * line, FileMgr * af)
4172
if (numbreak > -1) {
4173
HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
4180
piece = mystrsep(&tp, 0);
4182
if (*piece != '\0') {
4184
case 0: { np++; break; }
4186
numbreak = atoi(piece);
4188
HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
4191
if (numbreak == 0) return 0;
4192
breaktable = (char **) malloc(numbreak * sizeof(char *));
4193
if (!breaktable) return 1;
4201
piece = mystrsep(&tp, 0);
4204
HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
4208
/* now parse the numbreak lines to read in the remainder of the table */
4210
for (int j=0; j < numbreak; j++) {
4211
if (!(nl = af->getline())) return 1;
4215
piece = mystrsep(&tp, 0);
4217
if (*piece != '\0') {
4220
if (strncmp(piece,"BREAK",5) != 0) {
4221
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
4228
breaktable[j] = mystrdup(piece);
4235
piece = mystrsep(&tp, 0);
4238
HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
4246
void AffixMgr::reverse_condition(char * piece) {
4248
for (char * k = piece + strlen(piece) - 1; k >= piece; k--) {
4251
if (neg) *(k+1) = '['; else *k = ']';
4256
if (neg) *(k+1) = '^';
4261
if (*(k+1) == ']') neg = 1; else *(k+1) = *k;
4265
if (neg) *(k+1) = *k;
4271
int AffixMgr::parse_affix(char * line, const char at, FileMgr * af, char * dupflags)
4273
int numents = 0; // number of affentry structures to parse
4275
unsigned short aflag = 0; // affix char identifier
4278
std::vector<affentry> affentries;
4285
// checking lines with bad syntax
4287
int basefieldnum = 0;
4290
// split affix header line into pieces
4294
piece = mystrsep(&tp, 0);
4296
if (*piece != '\0') {
4298
// piece 1 - is type of affix
4299
case 0: { np++; break; }
4301
// piece 2 - is affix char
4304
aflag = pHMgr->decode_flag(piece);
4305
if (((at == 'S') && (dupflags[aflag] & dupSFX)) ||
4306
((at == 'P') && (dupflags[aflag] & dupPFX))) {
4307
HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of an affix flag\n",
4309
// return 1; XXX permissive mode for bad dictionaries
4311
dupflags[aflag] += (char) ((at == 'S') ? dupSFX : dupPFX);
4314
// piece 3 - is cross product indicator
4315
case 2: { np++; if (*piece == 'Y') ff = aeXPRODUCT; break; }
4317
// piece 4 - is number of affentries
4320
numents = atoi(piece);
4321
if ((numents <= 0) ||
4322
((::std::numeric_limits<size_t>::max()
4323
/ sizeof(struct affentry)) < numents))
4325
char * err = pHMgr->encode_flag(aflag);
4327
HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n",
4333
affentries.resize(numents);
4334
affentries[0].opts = ff;
4335
if (utf8) affentries[0].opts += aeUTF8;
4336
if (pHMgr->is_aliasf()) affentries[0].opts += aeALIASF;
4337
if (pHMgr->is_aliasm()) affentries[0].opts += aeALIASM;
4338
affentries[0].aflag = aflag;
4345
piece = mystrsep(&tp, 0);
4347
// check to make sure we parsed enough pieces
4349
char * err = pHMgr->encode_flag(aflag);
4351
HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
4357
// now parse numents affentries for this affix
4358
std::vector<affentry>::iterator start = affentries.begin();
4359
std::vector<affentry>::iterator end = affentries.end();
4360
for (std::vector<affentry>::iterator entry = start; entry != end; ++entry) {
4361
if ((nl = af->getline()) == NULL) return 1;
4367
// split line into pieces
4368
piece = mystrsep(&tp, 0);
4370
if (*piece != '\0') {
4372
// piece 1 - is type
4375
if (entry != start) entry->opts = start->opts &
4376
(char) (aeXPRODUCT + aeUTF8 + aeALIASF + aeALIASM);
4380
// piece 2 - is affix char
4383
if (pHMgr->decode_flag(piece) != aflag) {
4384
char * err = pHMgr->encode_flag(aflag);
4386
HUNSPELL_WARNING(stderr, "error: line %d: affix %s is corrupt\n",
4387
af->getlinenum(), err);
4393
if (entry != start) entry->aflag = start->aflag;
4397
// piece 3 - is string to strip or 0 for null
4400
if (complexprefixes) {
4401
if (utf8) reverseword_utf(piece); else reverseword(piece);
4403
entry->strip = mystrdup(piece);
4404
entry->stripl = (unsigned char) strlen(entry->strip);
4405
if (strcmp(entry->strip,"0") == 0) {
4407
entry->strip=mystrdup("");
4413
// piece 4 - is affix string or 0 for null
4416
entry->morphcode = NULL;
4417
entry->contclass = NULL;
4418
entry->contclasslen = 0;
4420
dash = strchr(piece, '/');
4426
remove_ignored_chars_utf(piece, ignorechars_utf16, ignorechars_utf16_len);
4428
remove_ignored_chars(piece,ignorechars);
4432
if (complexprefixes) {
4433
if (utf8) reverseword_utf(piece); else reverseword(piece);
4435
entry->appnd = mystrdup(piece);
4437
if (pHMgr->is_aliasf()) {
4438
int index = atoi(dash + 1);
4439
entry->contclasslen = (unsigned short) pHMgr->get_aliasf(index, &(entry->contclass), af);
4440
if (!entry->contclasslen) HUNSPELL_WARNING(stderr, "error: bad affix flag alias: \"%s\"\n", dash+1);
4442
entry->contclasslen = (unsigned short) pHMgr->decode_flags(&(entry->contclass), dash + 1, af);
4443
flag_qsort(entry->contclass, 0, entry->contclasslen);
4448
for (unsigned short _i = 0; _i < entry->contclasslen; _i++) {
4449
contclasses[(entry->contclass)[_i]] = 1;
4454
remove_ignored_chars_utf(piece, ignorechars_utf16, ignorechars_utf16_len);
4456
remove_ignored_chars(piece,ignorechars);
4460
if (complexprefixes) {
4461
if (utf8) reverseword_utf(piece); else reverseword(piece);
4463
entry->appnd = mystrdup(piece);
4466
entry->appndl = (unsigned char) strlen(entry->appnd);
4467
if (strcmp(entry->appnd,"0") == 0) {
4469
entry->appnd=mystrdup("");
4475
// piece 5 - is the conditions descriptions
4478
if (complexprefixes) {
4479
if (utf8) reverseword_utf(piece); else reverseword(piece);
4480
reverse_condition(piece);
4482
if (entry->stripl && (strcmp(piece, ".") != 0) &&
4483
redundant_condition(at, entry->strip, entry->stripl, piece, af->getlinenum()))
4487
reverse_condition(piece);
4489
if (encodeit(*entry, piece)) return 1;
4495
if (pHMgr->is_aliasm()) {
4496
int index = atoi(piece);
4497
entry->morphcode = pHMgr->get_aliasm(index);
4499
if (complexprefixes) { // XXX - fix me for morph. gen.
4500
if (utf8) reverseword_utf(piece); else reverseword(piece);
4502
// add the remaining of the line
4505
tp = tp + strlen(tp);
4507
entry->morphcode = mystrdup(piece);
4508
if (!entry->morphcode) return 1;
4516
piece = mystrsep(&tp, 0);
4518
// check to make sure we parsed enough pieces
4520
char * err = pHMgr->encode_flag(aflag);
4522
HUNSPELL_WARNING(stderr, "error: line %d: affix %s is corrupt\n",
4523
af->getlinenum(), err);
4530
// detect unnecessary fields, excepting comments
4532
int fieldnum = !(entry->morphcode) ? 5 : ((*(entry->morphcode)=='#') ? 5 : 6);
4533
if (fieldnum != basefieldnum)
4534
HUNSPELL_WARNING(stderr, "warning: line %d: bad field number\n", af->getlinenum());
4536
basefieldnum = !(entry->morphcode) ? 5 : ((*(entry->morphcode)=='#') ? 5 : 6);
4541
// now create SfxEntry or PfxEntry objects and use links to
4542
// build an ordered (sorted by affix string) list
4543
for (std::vector<affentry>::iterator entry = start; entry != end; ++entry) {
4545
PfxEntry * pfxptr = new PfxEntry(this,&(*entry));
4546
build_pfxtree(pfxptr);
4548
SfxEntry * sfxptr = new SfxEntry(this,&(*entry));
4549
build_sfxtree(sfxptr);
4555
int AffixMgr::redundant_condition(char ft, char * strip, int stripl, const char * cond, int linenum) {
4556
int condl = strlen(cond);
4561
if (ft == 'P') { // prefix
4562
if (strncmp(strip, cond, condl) == 0) return 1;
4565
for (i = 0, j = 0; (i < stripl) && (j < condl); i++, j++) {
4566
if (cond[j] != '[') {
4567
if (cond[j] != strip[i]) {
4568
HUNSPELL_WARNING(stderr, "warning: line %d: incompatible stripping characters and condition\n", linenum);
4572
neg = (cond[j+1] == '^') ? 1 : 0;
4576
if (strip[i] == cond[j]) in = 1;
4577
} while ((j < (condl - 1)) && (cond[j] != ']'));
4578
if (j == (condl - 1) && (cond[j] != ']')) {
4579
HUNSPELL_WARNING(stderr, "error: line %d: missing ] in condition:\n%s\n", linenum, cond);
4582
if ((!neg && !in) || (neg && in)) {
4583
HUNSPELL_WARNING(stderr, "warning: line %d: incompatible stripping characters and condition\n", linenum);
4588
if (j >= condl) return 1;
4591
if ((stripl >= condl) && strcmp(strip + stripl - condl, cond) == 0) return 1;
4594
for (i = stripl - 1, j = condl - 1; (i >= 0) && (j >= 0); i--, j--) {
4595
if (cond[j] != ']') {
4596
if (cond[j] != strip[i]) {
4597
HUNSPELL_WARNING(stderr, "warning: line %d: incompatible stripping characters and condition\n", linenum);
4604
if (strip[i] == cond[j]) in = 1;
4605
} while ((j > 0) && (cond[j] != '['));
4606
if ((j == 0) && (cond[j] != '[')) {
4607
HUNSPELL_WARNING(stderr, "error: line: %d: missing ] in condition:\n%s\n", linenum, cond);
4610
neg = (cond[j+1] == '^') ? 1 : 0;
4611
if ((!neg && !in) || (neg && in)) {
4612
HUNSPELL_WARNING(stderr, "warning: line %d: incompatible stripping characters and condition\n", linenum);
4617
if (j < 0) return 1;
4623
int AffixMgr::get_suffix_words(short unsigned *suff, int len,const char * root_word,char **slst){
4624
int suff_words_cnt = 0;
4625
short unsigned * start_ptr = suff;
4626
for (int j=0; j < SETSIZE ; j++) {
4627
SfxEntry * ptr = sStart[j];
4630
for (int i=0;i<len;i++){
4631
if ( (*suff) == ptr->getFlag() ){
4632
std::string nw(root_word);
4633
nw.append(ptr->getAffix());
4634
hentry * ht = ptr->checkword(nw.c_str(),nw.size(),0,NULL,NULL,0,NULL,0,0,0);
4636
slst[suff_words_cnt] = (char *) malloc(MAXWORDUTF8LEN * sizeof(char));
4637
if(slst[suff_words_cnt]){
4638
strcpy(slst[suff_words_cnt],nw.c_str());
4645
ptr = ptr->getNext();
4648
return suff_words_cnt;
added
removed
src/plugins/contrib/SpellChecker/hunspell/src/hunspell/affixmgr.cxx
