« back to all changes in this revision
Viewing changes to estraier.c
- browse files at revision 3
- compare with another revision
- download diff
- download tarball
- view history from revision 3
- Committer: Bazaar Package Importer
- Author(s): Steve Langasek
- Date: 2006-11-14 05:28:32 UTC
- mfrom: (2.1.4 feisty)
- Revision ID: james.westby@ubuntu.com-20061114052832-0lzqzcefn8mt4yqe
Tags: 1.4.9-1.1
* Non-maintainer upload.
* High-urgency upload for RC bugfix.
* Set HOME=$(CURDIR)/junkhome when building, otherwise the package build
will incorrectly look for headers there -- and fail when the directory
exists and is unreadable, as happens sometimes on sudo-using
autobuilders!
* High-urgency upload for RC bugfix.
* Set HOME=$(CURDIR)/junkhome when building, otherwise the package build
will incorrectly look for headers there -- and fail when the directory
exists and is unreadable, as happens sometimes on sudo-using
autobuilders!
- files added:
- debian/libestraier8.files
- doc/perlnativeapi
- perlnative
- perlnative/example
- perlnative/src
- files removed:
- debian/estcall.1
- files modified:
- ChangeLog
added
removed
estraier.c
1
1
/*************************************************************************************************
2
2
* Implementation of the core API
3
* Copyright (C) 2004-2005 Mikio Hirabayashi
3
* Copyright (C) 2004-2006 Mikio Hirabayashi
4
4
* This file is part of Hyper Estraier.
5
5
* Hyper Estraier is free software; you can redistribute it and/or modify it under the terms of
6
6
* the GNU Lesser General Public License as published by the Free Software Foundation; either
14
14
*************************************************************************************************/
15
15
16
16
17
#if defined(_MYVISTA)
18
#include <vista.h>
19
#endif
20
17
21
#include "estraier.h"
18
22
#include "myconf.h"
19
23
30
34
#define ESTKEYIDXNUM "_idxnum" /* key for the number of inverted indexes */
31
35
#define ESTKEYDSEQ "_dseq" /* key for the sequence for document IDs */
32
36
#define ESTKEYDNUM "_dnum" /* key for the number of documents */
33
#define ESTKEYAMODE "_amode" /* key for the mode of text analyzer */
34
37
#define ESTKEYMETA "_meta" /* key for meta data */
35
38
36
39
#define ESTIDXDBNAME "_idx" /* name of the inverted index */
37
#define ESTIDXDBLRM 55 /* records in a leaf node of the inverted index */
40
#define ESTIDXDBLRM 109 /* records in a leaf node of the inverted index */
41
#define ESTIDXDBLRMA 17 /* records in a leaf node of the index in APN mode */
38
42
#define ESTIDXDBNIM 160 /* records in a non-leaf node of the inverted index */
39
43
#define ESTIDXDBLCN 16 /* number of leaf cache of the inverted index */
40
44
#define ESTIDXDBNCN 16 /* number of non-leaf cache of the inverted index */
41
45
#define ESTIDXDBRLCN 128 /* number of leaf cache of the index reader */
42
#define ESTIDXDBRNCN 64 /* number of non-leaf cache of the index reader */
43
#define ESTIDXDBMIN (1048576*768) /* minimum size of a database file */
46
#define ESTIDXDBRLCNA 32 /* number of leaf cache of the reader in APN mode */
47
#define ESTIDXDBRNCN 256 /* number of non-leaf cache of the index reader */
48
#define ESTIDXDBFBP 512 /* size of free block pool of the inverted index */
49
#define ESTIDXDBMIN (1048576*512) /* minimum size of a database file */
44
50
#define ESTIDXDBMAX (1048576*1536) /* maximum size of a database file */
45
51
46
52
#define ESTFWMDBNAME "_fwm" /* name of the database for forward matching */
47
#define ESTFWMDBLRM 111 /* records in a leaf node of forward matching DB */
53
#define ESTFWMDBLRM 251 /* records in a leaf node of forward matching DB */
48
54
#define ESTFWMDBNIM 110 /* records in a non-leaf node of forward matching DB */
49
55
#define ESTFWMDBLCN 32 /* number of leaf cache of forward matching DB */
50
56
#define ESTFWMDBNCN 16 /* number of non-leaf cache of forward matching DB */
51
52
#define ESTATTRDBNAME "_attr" /* name of the database for attrutes */
53
#define ESTATTRDBBNUM 245759 /* bucket number of the database for attrutes */
54
#define ESTATTRDBDNUM 3 /* division number of the database for attrutes */
55
#define ESTATTRDBALN -5 /* alignment of the database for attrutes */
57
#define ESTFWMDBFBP 128 /* size of free block pool of forward matching DB */
58
59
#define ESTAUXDBNAME "_aux" /* name of the auxiliary index */
60
#define ESTAUXDBLRM 23 /* records in a leaf node of the auxiliary index */
61
#define ESTAUXDBNIM 160 /* records in a non-leaf node of the auxiliary index */
62
#define ESTAUXDBLCN 16 /* number of leaf cache of the auxiliary index */
63
#define ESTAUXDBNCN 16 /* number of non-leaf cache of the auxiliary index */
64
#define ESTAUXDBRLCN 256 /* number of leaf cache of the auxiliary reader */
65
#define ESTAUXDBRNCN 64 /* number of non-leaf cache of the auxiliary reader */
66
#define ESTAUXDBFBP 256 /* size of free block pool of the auxiliary index */
67
68
#define ESTXFMDBNAME "_xfm" /* name of the database for auxiliary forward matching */
69
#define ESTXFMDBLRM 111 /* records in a leaf node of xfm DB */
70
#define ESTXFMDBNIM 110 /* records in a non-leaf node of xfm DB */
71
#define ESTXFMDBLCN 32 /* number of leaf cache of xfm DB */
72
#define ESTXFMDBNCN 16 /* number of non-leaf cache of xfm DB */
73
#define ESTXFMDBFBP 128 /* size of free block pool of xfm DB */
74
75
#define ESTATTRDBNAME "_attr" /* name of the database for attributes */
76
#define ESTATTRDBBNUM 212987 /* bucket number of the database for attributes */
77
#define ESTATTRDBDNUM 3 /* division number of the database for attributes */
78
#define ESTATTRDBALN -5 /* alignment of the database for attributes */
79
#define ESTATTRDBFBP 64 /* size of free block pool of the attribute DB */
56
80
57
81
#define ESTTEXTDBNAME "_text" /* name of the database of texts */
58
82
#define ESTTEXTDBBNUM 61417 /* bucket number of the database for texts */
59
83
#define ESTTEXTDBDNUM 7 /* division number of the database for texts */
60
84
#define ESTTEXTDBALN -5 /* alignment of the database for texts */
85
#define ESTTEXTDBFBP 128 /* size of free block pool of the text DB */
61
86
62
87
#define ESTKWDDBNAME "_kwd" /* name of the database of keywords */
63
88
#define ESTKWDDBBNUM 163819 /* bucket number of the database for keywords */
64
89
#define ESTKWDDBDNUM 3 /* division number of the database for keywords */
65
90
#define ESTKWDDBALN -5 /* alignment of the database for keywords */
91
#define ESTKWDDBFBP 64 /* size of free block pool of the keyword DB */
66
92
67
93
#define ESTLISTDBNAME "_list" /* name of the database of document list */
68
94
#define ESTLISTDBLRM 99 /* records in a leaf node of document list DB */
69
95
#define ESTLISTDBNIM 200 /* records in a non-leaf node of document list DB */
70
#define ESTLISTDBLCN 32 /* number of leaf cache of document list DB */
96
#define ESTLISTDBLCN 64 /* number of leaf cache of document list DB */
71
97
#define ESTLISTDBNCN 16 /* number of non-leaf cache of document list DB */
98
#define ESTLISTDBFBP 128 /* size of free block pool of document list DB */
99
100
#define ESTAISEQPREF "__seq_" /* prefix of the database for sequencial access */
101
#define ESTAISTRPREF "__str_" /* prefix of the database for string narrowing */
102
#define ESTAINUMPREF "__num_" /* prefix of the database for number narrowing */
103
#define ESTAIBDIAM 0.8 /* diameter of the bucket number */
104
#define ESTAIDXLRM 99 /* records in a leaf node of narrowing index */
105
#define ESTAIDXNIM 120 /* records in a non-leaf node of narrowing index */
106
#define ESTAIDXLCN 1024 /* number of leaf cache of narrowing index */
107
#define ESTAIDXNCN 256 /* number of non-leaf cache of narrowing index */
108
#define ESTAIDXDPFBP 32 /* size of free block pool of sequencial DB */
109
#define ESTAIDXVLFBP 128 /* size of free block pool of narrowing DB */
110
#define ESTAIKBUFSIZ 8192 /* size of a buffer for a key */
111
#define ESTOPDUMMY "[DUMMY]" /* dummy operator */
112
113
#define ESTDBSBRAT 0.3 /* ratio of bucket numbers of large mode */
114
#define ESTDBSDRAT 0.4 /* ratio of the division number of large mode */
115
#define ESTDBLBRAT 3.0 /* ratio of bucket numbers of large mode */
116
#define ESTDBLDRAT 1.0 /* ratio of the division number of large mode */
117
#define ESTDBHBRAT 5.0 /* ratio of bucket numbers of huge mode */
118
#define ESTDBHDRAT 2.0 /* ratio of the division number of huge mode */
119
#define ESTDBH2RAT 1.4 /* ratio of huge mode second */
120
#define ESTDBH3RAT 2.0 /* ratio of huge mode third */
121
122
#define ESTVLCRDNUM 2 /* division number of usual Villa databases */
123
#define ESTVLCRDNAUX 7 /* division number of the auxiliary index */
72
124
73
125
#define ESTIDXCCBNUM 524288 /* bucket number of cache for the inverted index */
126
#define ESTAUXCCBNUM 65521 /* bucket number of cache for the auxiliary index */
74
127
#define ESTIDXCCMAX (1048576*64) /* max size of the cache */
75
128
#define ESTOUTCCBNUM 131072 /* bucket number of cache for deleted documents */
76
129
#define ESTKEYCCMNUM 65536 /* bucket number of cache for keys for TF-IDF */
84
137
#define ESTICCHECKSIZ 32768 /* size of checking character code */
85
138
#define ESTICMISSMAX 256 /* allowance number of missing characters */
86
139
#define ESTICALLWRAT 0.001 /* allowance ratio of missing characters */
87
#define ESTOCPOINT 10 /* point per occurrence */
140
#define ESTOCPOINT 16 /* point per occurrence */
88
141
#define ESTJHASHNUM 251 /* hash number for a junction */
89
142
#define ESTWORDMAXLEN 48 /* maximum length of a word */
90
143
#define ESTWORDAVGLEN 8 /* average length of a word */
92
145
#define ESTKEYSCALW 3 /* allowance ratio of TF-IDF for keywords */
93
146
#define ESTMEMIRATIO 1.1 /* incremental ratio of memory allocation */
94
147
95
#define ESTECLKNUM 32 /* number of keywords to esclipse candidates */
148
#define ESTSCOREUNIT 1000 /* unit of standard deviation of scoring */
149
#define ESTAUXMIN 32 /* minimum hits to adopt the auxiliary index */
150
#define ESTAUXEXRAT 16 /* ratio of hits of keywords expansion */
151
#define ESTWILDMAX 256 /* maximum number of expansion of wild cards */
152
#define ESTECLKNUM 32 /* number of keywords to eclipse candidates */
96
153
#define ESTSMLRKNUM 16 /* number of keywords to get candidates */
97
154
#define ESTSMLRUNUM 1024 /* number of adopted documents for a keyword */
155
#define ESTSMLRMNUM 4096 /* maximum number of candidates to be checked */
98
156
#define ESTSMLRNMIN 0.5 /* the minimum value for narrowing */
99
157
158
/* set a buffer for a variable length number */
159
#define EST_SET_VNUMBUF(EST_len, EST_buf, EST_num) \
160
do { \
161
int _EST_num = (EST_num); \
162
div_t EST_d; \
163
if(_EST_num == 0){ \
164
((signed char *)(EST_buf))[0] = 0; \
165
(EST_len) = 1; \
166
} else { \
167
(EST_len) = 0; \
168
while(_EST_num > 0){ \
169
EST_d = div(_EST_num, 128); \
170
_EST_num = EST_d.quot; \
171
if(_EST_num > 0){ \
172
((signed char *)(EST_buf))[(EST_len)] = -EST_d.rem - 1; \
173
} else { \
174
((signed char *)(EST_buf))[(EST_len)] = EST_d.rem; \
175
} \
176
(EST_len)++; \
177
} \
178
} \
179
} while(FALSE)
180
181
/* read a variable length buffer */
182
#define EST_READ_VNUMBUF(EST_buf, EST_num, EST_step) \
183
do { \
184
int _EST_i, _EST_base; \
185
(EST_num) = 0; \
186
_EST_base = 1; \
187
for(_EST_i = 0; TRUE; _EST_i++){ \
188
if(((signed char *)(EST_buf))[_EST_i] >= 0){ \
189
(EST_num) += ((signed char *)(EST_buf))[_EST_i] * _EST_base; \
190
break; \
191
} \
192
(EST_num) += _EST_base * (((signed char *)(EST_buf))[_EST_i] + 1) * -1; \
193
_EST_base *= 128; \
194
} \
195
EST_step = _EST_i + 1; \
196
} while(FALSE)
197
198
typedef struct { /* type of structure for an attribute database */
199
void *db; /* handle of the database */
200
int type; /* data type */
201
} ESTATTRIDX;
202
100
203
enum { /* enumeration for character categories */
101
204
ESTSPACECHR, /* space characters */
102
205
ESTDELIMCHR, /* delimiter characters */
103
206
ESTWESTALPH, /* west alphabets */
104
ESTEASTALPH /* east alphabets */
105
};
106
107
enum { /* enumeration for text analizer modes */
108
ESTAMNORMAL, /* normal */
109
ESTAMPERFNG /* perfect N-gram */
207
ESTEASTALPH, /* east alphabets */
208
ESTHIRAGANA, /* east alphabets: hiragana */
209
ESTKATAKANA, /* east alphabets: katakana */
210
ESTHANGUL, /* east alphabets: hangul */
211
ESTKANJI /* east alphabets: kanji */
212
};
213
214
enum { /* enumeration for flags for databases */
215
ESTDFPERFNG = 1 << 10, /* use perfect N-gram analizer */
216
ESTDFCHRCAT = 1 << 11, /* use character category analizer */
217
ESTDFZLIB = 1 << 15, /* compress records with ZLIB */
218
ESTDFLZO = 1 << 16, /* compress records with LZO */
219
ESTDFBZIP = 1 << 17, /* compress records with BZIP2 */
220
ESTDFSCVOID = 1 << 20, /* store scores as void */
221
ESTDFSCINT = 1 << 21, /* store scores as integer */
222
ESTDFSCASIS = 1 << 22 /* refrain from adjustment of scores */
223
};
224
225
enum { /* enumration for phrase format */
226
ESTPMUSUAL, /* usual phrase */
227
ESTPMSIMPLE, /* simplified phrase */
228
ESTPMROUGH, /* rough phrase */
229
ESTPMUNION, /* union phrase */
230
ESTPMISECT /* intersection phrase */
110
231
};
111
232
112
233
typedef struct { /* type of structure for a hitting object */
118
239
typedef struct { /* type of structure for a conditional attribute */
119
240
char *name; /* name */
120
241
int nsiz; /* size of the name */
242
CBLIST *nlist; /* list of plural names */
121
243
char *oper; /* operator */
122
244
char *val; /* value */
123
245
int vsiz; /* size of the value */
135
257
int pt; /* score tuned by TF-IDF */
136
258
} ESTKEYSC;
137
259
260
typedef struct { /* type of structure for a meta hitting object */
261
int db; /* index of a container database */
262
int id; /* ID of a document */
263
int score; /* score tuned by TF-IDF */
264
char *value; /* value of an attribute for sorting */
265
} ESTMETASCORE;
266
138
267
139
268
/* private function prototypes */
269
static char *est_hex_encode(const char *str);
270
static char *est_hex_decode(const char *str);
140
271
static int est_enc_miss(const char *ptr, int size, const char *icode, const char *ocode);
141
272
static void est_normalize_text(unsigned char *utext, int size, int *sp);
142
273
static void est_canonicalize_text(unsigned char *utext, int size, int funcspc);
143
274
static int est_char_category(int c);
144
275
static int est_char_category_perfng(int c);
145
static char *est_phrase_from_thumb(const char *sphrase);
276
static int est_char_category_chrcat(int c);
277
static char *est_make_snippet(const char *str, int len, const CBLIST *words,
278
int wwidth, int hwidth, int awidth);
279
static int est_check_cjk_only(const char *str);
280
static char *est_phrase_from_simple(const char *sphrase);
281
static char *est_phrase_from_rough(const char *rphrase);
282
static char *est_phrase_from_union(const char *uphrase);
283
static char *est_phrase_from_isect(const char *iphrase);
146
284
static void est_snippet_add_text(const unsigned char *rtext, const unsigned char *ctext,
147
285
int size, int awsiz, CBDATUM *res, const CBLIST *rwords);
148
286
static int est_str_fwmatch_wide(const unsigned char *haystack, int hsiz,
149
287
const unsigned char *needle, int nsiz);
288
static char *est_strstr_sparse(const char *haystack, const char *needle);
289
static int est_idx_rec_last_id(const char *vbuf, int vsiz, int smode);
290
static void est_encode_idx_rec(CBDATUM *datum, const char *vbuf, int vsiz, int lid, int smode);
291
static void est_decode_idx_rec(CBDATUM *datum, const char *vbuf, int vsiz, int smode);
150
292
static ESTIDX *est_idx_open(const char *name, int omode, int dnum);
151
293
static int est_idx_close(ESTIDX *idx);
152
static void est_idx_set_tuning(ESTIDX *idx, int lrecmax, int nidxmax, int lcnum, int ncnum);
294
static void est_idx_set_tuning(ESTIDX *idx, int lrecmax, int nidxmax, int lcnum, int ncnum,
295
int fbpsiz);
153
296
static void est_idx_increment(ESTIDX *idx);
154
297
static int est_idx_dnum(ESTIDX *idx);
155
static int est_idx_add(ESTIDX *idx, const char *word, int wsiz, const char *vbuf, int vsiz);
298
static int est_idx_add(ESTIDX *idx, const char *word, int wsiz,
299
const char *vbuf, int vsiz, int smode);
156
300
static int est_idx_put_one(ESTIDX *idx, int inum, const char *word, int wsiz,
157
301
const char *vbuf, int vsiz);
158
302
static int est_idx_out(ESTIDX *idx, const char *word, int wsiz);
159
static char *est_idx_get(ESTIDX *idx, const char *word, int wsiz, int *sp);
160
static char *est_idx_get_one(ESTIDX *idx, int inum, const char *word, int wsiz, int *sp);
303
static char *est_idx_scan(ESTIDX *idx, const char *word, int wsiz, int *sp, int smode);
304
static const char *est_idx_get_one(ESTIDX *idx, int inum, const char *word, int wsiz, int *sp);
161
305
static int est_idx_vsiz(ESTIDX *idx, const char *word, int wsiz);
162
306
static int est_idx_num(ESTIDX *idx);
163
307
static double est_idx_size(ESTIDX *idx);
164
308
static int est_idx_size_current(ESTIDX *idx);
309
static int est_idx_memflush(ESTIDX *idx);
165
310
static int est_idx_sync(ESTIDX *idx);
166
311
static int est_idx_optimize(ESTIDX *idx);
167
312
static void est_idx_set_current(ESTIDX *idx);
313
static int est_crput(CURIA *curia, int zmode, int id, const char *vbuf, int vsiz, int dmode);
314
static int est_crout(CURIA *curia, int id);
315
static char *est_crget(CURIA *curia, int flags, int id, int *sp);
316
static int est_aidx_seq_put(DEPOT *db, int id, const char *vbuf, int vsiz);
317
static int est_aidx_seq_out(DEPOT *db, int id);
318
static char *est_aidx_seq_get(DEPOT *db, int id, int *sp);
319
static int est_aidx_seq_narrow(DEPOT *db, const CBLIST *pdocs, const char *cop, int sign,
320
const char *oval, int osiz, const char *sval, int ssiz,
321
const void *regex, int onum, ESTSCORE *scores, int snum,
322
int limit, int *restp);
323
static int est_aidx_numcmp(const char *aptr, int asiz, const char *bptr, int bsiz);
324
static int est_aidx_attr_put(VILLA *db, int id, const char *vbuf, int vsiz);
325
static int est_aidx_attr_out(VILLA *db, int id, const char *vbuf, int vsiz);
326
static int est_aidx_attr_narrow(VILLA *db, const CBLIST *pdocs, const char *cop, int sign,
327
const char *oval, int osiz, const char *sval, int ssiz,
328
const void *regex, int onum, ESTSCORE *scores, int snum);
329
static int est_int_compare(const void *ap, const void *bp);
330
static int est_short_compare(const void *ap, const void *bp);
331
static void est_inodes_delete(void *arg);
332
static void est_inodes_delete_informer(const char *msg, void *opaque);
168
333
static int est_db_write_meta(ESTDB *db);
169
334
static void est_db_inform(ESTDB *db, const char *info);
170
335
static void est_db_prepare_meta(ESTDB *db);
336
static int est_db_score_doc(ESTDB *db, ESTDOC *doc, ESTCOND *cond, int *scp);
337
static int est_pidx_uri_to_id(ESTDB *db, const char *uri);
171
338
static CBLIST *est_phrase_terms(const char *phrase);
172
static int est_score_compare_by_id(const void *ap, const void *bp);
173
static int est_score_compare_by_score(const void *ap, const void *bp);
339
static int est_score_compare_by_id_asc(const void *ap, const void *bp);
340
static int est_score_compare_by_id_desc(const void *ap, const void *bp);
341
static int est_score_compare_by_score_asc(const void *ap, const void *bp);
342
static int est_score_compare_by_score_desc(const void *ap, const void *bp);
174
343
static int est_score_compare_by_str_asc(const void *ap, const void *bp);
175
344
static int est_score_compare_by_str_desc(const void *ap, const void *bp);
176
345
static int est_score_compare_by_num_asc(const void *ap, const void *bp);
177
346
static int est_score_compare_by_num_desc(const void *ap, const void *bp);
347
static int est_metascore_compare_by_id_asc(const void *ap, const void *bp);
348
static int est_metascore_compare_by_id_desc(const void *ap, const void *bp);
349
static int est_metascore_compare_by_score_asc(const void *ap, const void *bp);
350
static int est_metascore_compare_by_score_desc(const void *ap, const void *bp);
351
static int est_metascore_compare_by_str_asc(const void *ap, const void *bp);
352
static int est_metascore_compare_by_str_desc(const void *ap, const void *bp);
353
static int est_metascore_compare_by_num_asc(const void *ap, const void *bp);
354
static int est_metascore_compare_by_num_desc(const void *ap, const void *bp);
178
355
static ESTSCORE *est_search_uvset(ESTDB *db, int *nump, CBMAP *hints, int add);
179
356
static void est_expand_word_bw(ESTDB *db, const char *word, CBLIST *list);
180
357
static void est_expand_word_ew(ESTDB *db, const char *word, CBLIST *list);
181
358
static void est_expand_word_rx(ESTDB *db, const char *word, CBLIST *list);
359
static void est_expand_keyword_bw(ESTDB *db, const char *word, CBLIST *list);
360
static void est_expand_keyword_ew(ESTDB *db, const char *word, CBLIST *list);
361
static void est_expand_keyword_rx(ESTDB *db, const char *word, CBLIST *list);
182
362
static ESTSCORE *est_search_union(ESTDB *db, const char *term, int gstep,
183
int *nump, CBMAP *hints, int add);
363
void (*xpn)(const char *, CBLIST *),
364
int *nump, CBMAP *hints, int add, int auxmin, CBMAP *auxwords);
184
365
static const ESTSCORE *est_rescc_get(ESTDB *db, const char *word, int size, int *nump);
185
366
static void est_rescc_put(ESTDB *db, const char *word, int size, ESTSCORE *scores, int num);
186
static int est_narrow_scores(ESTDB *db, const CBLIST *attrs, const char *order,
187
ESTSCORE *scores, int snum, int limit, int *restp);
367
static ESTSCORE *est_search_keywords(ESTDB *db, const char *word, int min, int *nump);
368
static void est_weight_keywords(ESTDB *db, const char *word, ESTSCORE *scores, int snum);
369
static ESTSCORE *est_search_rank(ESTDB *db, const char *name, int top, int *nump);
370
static ESTSCORE *est_search_aidx_attr(ESTDB *db, const char *expr, int *nump);
371
static ESTSCORE *est_search_pidxs(ESTDB *db, ESTCOND *cond, ESTSCORE *scores, int *nump,
372
CBMAP *ordattrs);
373
static int est_narrow_scores(ESTDB *db, const CBLIST *attrs, int ign,
374
const char *order, const char *distinct, ESTSCORE *scores, int snum,
375
int limit, int *restp, CBMAP *ordattrs);
376
static ESTCATTR *est_make_cattr_list(const CBLIST *attrs, int *nump);
377
static void est_free_cattr_list(ESTCATTR *list, int anum);
188
378
static int est_eclipse_scores(ESTDB *db, ESTSCORE *scores, int snum, int num,
189
379
int vnum, int tfidf, double limit, CBMAP *shadows);
190
380
static int est_match_attr(const char *tval, int tsiz, const char *cop, int sign,
192
382
const void *regex, int onum);
193
383
static int est_check_strand(const char *tval, const char *oval);
194
384
static int est_check_stror(const char *tval, const char *oval);
385
static int est_check_stroreq(const char *tval, const char *oval);
195
386
static int est_check_numbt(const char *tval, const char *oval);
196
387
static int est_keysc_compare(const void *ap, const void *bp);
197
388
static ESTSCORE *est_search_similar(ESTDB *db, CBMAP *svmap, int *nump,
198
int knum, int unum, int tfidf, double nmin);
389
int knum, int unum, int mnum, int tfidf,
390
double nmin, int auxmin, CBMAP *auxwords);
199
391
static CBMAP *est_phrase_vector(const char *phrase);
200
392
static CBMAP *est_get_tvmap(ESTDB *db, int id, int vnum, int tfidf);
393
static int est_url_sameness(const char *aurl, const char *burl);
201
394
static void est_random_fclose(void);
202
395
static int est_signal_dispatch(int signum);
203
396
225
418
doc->id = -1;
226
419
doc->attrs = NULL;
227
420
doc->dtexts = NULL;
421
doc->kwords = NULL;
228
422
return doc;
229
423
}
230
424
234
428
ESTDOC *doc;
235
429
CBLIST *lines;
236
430
const char *line;
237
char *pv;
431
char *pv, *rp, *ep;
238
432
int i;
239
433
assert(draft);
240
434
doc = est_doc_new();
241
435
lines = cbsplit(draft, -1, "\n");
242
436
for(i = 0; i < CB_LISTNUM(lines); i++){
243
line = CB_LISTVAL(lines, i, NULL);
437
line = CB_LISTVAL(lines, i);
244
438
while(*line > '\0' && *line <= ' '){
245
439
line++;
246
440
}
248
442
i++;
249
443
break;
250
444
}
251
if(*line == '%') continue;
252
if((pv = strchr(line, '=')) != NULL){
445
if(*line == '%'){
446
if(cbstrfwmatch(line, ESTDCNTLVECTOR)){
447
if(!doc->kwords) doc->kwords = cbmapopenex(ESTMINIBNUM);
448
if((rp = strchr(line, '\t')) != NULL) rp++;
449
while(rp && (pv = strchr(rp, '\t')) != NULL){
450
pv++;
451
if((ep = strchr(pv, '\t')) != NULL){
452
*ep = '\0';
453
ep++;
454
}
455
if(rp[0] != '\0' && pv[0] != '\0') cbmapput(doc->kwords, rp, pv - rp - 1, pv, -1, TRUE);
456
rp = ep;
457
}
458
} else if(cbstrfwmatch(line, ESTDCNTLSCORE)){
459
if((rp = strchr(line, '\t')) != NULL) est_doc_set_score(doc, atoi(rp + 1));
460
}
461
} else if((pv = strchr(line, '=')) != NULL){
253
462
*(pv++) = '\0';
254
463
est_doc_add_attr(doc, line, pv);
255
464
}
256
465
}
257
466
for(; i < CB_LISTNUM(lines); i++){
258
line = CB_LISTVAL(lines, i, NULL);
467
line = CB_LISTVAL(lines, i);
259
468
if(*line == '\t'){
260
469
est_doc_add_hidden_text(doc, line + 1);
261
470
} else {
262
471
est_doc_add_text(doc, line);
263
472
}
264
473
}
265
cblistclose(lines);
474
CB_LISTCLOSE(lines);
266
475
return doc;
267
476
}
268
477
270
479
/* Destroy a document object. */
271
480
void est_doc_delete(ESTDOC *doc){
272
481
assert(doc);
273
if(doc->dtexts) cblistclose(doc->dtexts);
482
if(doc->kwords) cbmapclose(doc->kwords);
483
if(doc->dtexts) CB_LISTCLOSE(doc->dtexts);
274
484
if(doc->attrs) cbmapclose(doc->attrs);
275
485
free(doc);
276
486
}
279
489
/* Add an attribute to a document object. */
280
490
void est_doc_add_attr(ESTDOC *doc, const char *name, const char *value){
281
491
char *rbuf, *wp;
492
int len;
282
493
assert(doc && name);
283
494
if(name[0] == '\0' || name[0] == '%') return;
284
495
if(!doc->attrs) doc->attrs = cbmapopenex(ESTMINIBNUM);
288
499
if(*wp > 0 && *wp < ' ') *wp = ' ';
289
500
}
290
501
cbstrsqzspc(rbuf);
291
cbmapputvbuf(doc->attrs, name, strlen(name), rbuf, strlen(rbuf));
502
if((len = strlen(name)) > 0) cbmapput(doc->attrs, name, len, rbuf, -1, TRUE);
503
free(rbuf);
292
504
} else {
293
505
cbmapout(doc->attrs, name, -1);
294
506
}
305
517
text++;
306
518
}
307
519
if(text[0] == '\0') return;
308
if(!doc->dtexts) doc->dtexts = cblistopen();
520
if(!doc->dtexts) CB_LISTOPEN(doc->dtexts);
309
521
utext = (unsigned char *)est_uconv_in(text, strlen(text), &size);
310
522
est_normalize_text(utext, size, &size);
311
523
rtext = est_uconv_out((char *)utext, size, NULL);
314
526
}
315
527
cbstrsqzspc(rtext);
316
528
if(rtext[0] != '\0'){
317
cblistpushbuf(doc->dtexts, rtext, strlen(rtext));
529
CB_LISTPUSHBUF(doc->dtexts, rtext, strlen(rtext));
318
530
} else {
319
531
free(rtext);
320
532
}
349
561
}
350
562
351
563
564
/* Attach keywords to a document object. */
565
void est_doc_set_keywords(ESTDOC *doc, CBMAP *kwords){
566
assert(doc && kwords);
567
if(doc->kwords) cbmapclose(doc->kwords);
568
doc->kwords = cbmapdup(kwords);
569
}
570
571
572
/* Set the substitute score of a document object. */
573
void est_doc_set_score(ESTDOC *doc, int score){
574
char numbuf[ESTNUMBUFSIZ];
575
assert(doc);
576
if(!doc->attrs) doc->attrs = cbmapopenex(ESTMINIBNUM);
577
if(score >= 0){
578
sprintf(numbuf, "%d", score);
579
cbmapput(doc->attrs, "\t", 1, numbuf, -1, TRUE);
580
} else {
581
cbmapout(doc->attrs, "\t", 1);
582
}
583
}
584
585
352
586
/* Get the ID number of a document object. */
353
587
int est_doc_id(ESTDOC *doc){
354
588
assert(doc);
362
596
const char *kbuf;
363
597
int ksiz;
364
598
assert(doc);
365
if(!doc->attrs) return cblistopen();
366
names = cblistopen();
599
if(!doc->attrs){
600
CB_LISTOPEN(names);
601
return names;
602
}
603
CB_LISTOPEN(names);
367
604
cbmapiterinit(doc->attrs);
368
605
while((kbuf = cbmapiternext(doc->attrs, &ksiz)) != NULL){
369
if(ksiz > 0) cblistpush(names, kbuf, ksiz);
606
if(ksiz > 0 && kbuf[0] != '\t') CB_LISTPUSH(names, kbuf, ksiz);
370
607
}
371
608
cblistsort(names);
372
609
return names;
384
621
/* Get a list of sentences of the text of a document object. */
385
622
const CBLIST *est_doc_texts(ESTDOC *doc){
386
623
assert(doc);
387
if(!doc->dtexts) doc->dtexts = cblistopen();
624
if(!doc->dtexts) CB_LISTOPEN(doc->dtexts);
388
625
return doc->dtexts;
389
626
}
390
627
395
632
const char *elem;
396
633
int i, size;
397
634
if(!doc->dtexts) return cbmemdup("", 0);
398
datum = cbdatumopen("", 0);
635
CB_DATUMOPEN(datum);
399
636
for(i = 0; i < CB_LISTNUM(doc->dtexts); i++){
400
elem = CB_LISTVAL2(doc->dtexts, i, &size);
401
if(i > 0) cbdatumcat(datum, " ", 1);
402
cbdatumcat(datum, elem, size);
637
elem = CB_LISTVAL2(doc->dtexts, i, size);
638
if(i > 0) CB_DATUMCAT(datum, " ", 1);
639
CB_DATUMCAT(datum, elem, size);
403
640
}
404
641
return cbdatumtomalloc(datum, NULL);
405
642
}
406
643
407
644
645
/* Get attached keywords of a document object. */
646
CBMAP *est_doc_keywords(ESTDOC *doc){
647
assert(doc);
648
return doc->kwords;
649
}
650
651
652
/* Get the substitute score of a document object. */
653
int est_doc_score(ESTDOC *doc){
654
const char *vbuf;
655
assert(doc);
656
if(doc->attrs && (vbuf = cbmapget(doc->attrs, "\t", 1, NULL)) != NULL) return atoi(vbuf);
657
return -1;
658
}
659
660
408
661
/* Dump draft data of a document object. */
409
662
char *est_doc_dump_draft(ESTDOC *doc){
410
663
CBLIST *list;
412
665
const char *kbuf, *vbuf;
413
666
int i, ksiz, vsiz;
414
667
assert(doc);
415
datum = cbdatumopen("", 0);
668
CB_DATUMOPEN(datum);
416
669
if(doc->attrs){
417
670
list = est_doc_attr_names(doc);
418
671
for(i = 0; i < CB_LISTNUM(list); i++){
419
kbuf = CB_LISTVAL2(list, i, &ksiz);
672
kbuf = CB_LISTVAL2(list, i, ksiz);
420
673
vbuf = cbmapget(doc->attrs, kbuf, ksiz, &vsiz);
421
cbdatumcat(datum, kbuf, ksiz);
422
cbdatumcat(datum, "=", 1);
423
cbdatumcat(datum, vbuf, vsiz);
424
cbdatumcat(datum, "\n", 1);
425
}
426
cblistclose(list);
427
}
428
cbdatumcat(datum, "\n", 1);
674
CB_DATUMCAT(datum, kbuf, ksiz);
675
CB_DATUMCAT(datum, "=", 1);
676
CB_DATUMCAT(datum, vbuf, vsiz);
677
CB_DATUMCAT(datum, "\n", 1);
678
}
679
CB_LISTCLOSE(list);
680
}
681
if(doc->kwords && cbmaprnum(doc->kwords) > 0){
682
CB_DATUMCAT(datum, ESTDCNTLVECTOR, strlen(ESTDCNTLVECTOR));
683
cbmapiterinit(doc->kwords);
684
while((kbuf = cbmapiternext(doc->kwords, &ksiz)) != NULL){
685
CB_MAPITERVAL(vbuf, kbuf, vsiz);
686
CB_DATUMCAT(datum, "\t", 1);
687
CB_DATUMCAT(datum, kbuf, ksiz);
688
CB_DATUMCAT(datum, "\t", 1);
689
CB_DATUMCAT(datum, vbuf, vsiz);
690
}
691
CB_DATUMCAT(datum, "\n", 1);
692
}
693
if(doc->attrs && (vbuf = cbmapget(doc->attrs, "\t", 1, &vsiz)) != NULL){
694
CB_DATUMCAT(datum, ESTDCNTLSCORE, strlen(ESTDCNTLSCORE));
695
CB_DATUMCAT(datum, "\t", 1);
696
CB_DATUMCAT(datum, vbuf, vsiz);
697
CB_DATUMCAT(datum, "\n", 1);
698
}
699
CB_DATUMCAT(datum, "\n", 1);
429
700
if(doc->dtexts){
430
701
for(i = 0; i < CB_LISTNUM(doc->dtexts); i++){
431
kbuf = CB_LISTVAL2(doc->dtexts, i, &ksiz);
432
cbdatumcat(datum, kbuf, ksiz);
433
cbdatumcat(datum, "\n", 1);
702
kbuf = CB_LISTVAL2(doc->dtexts, i, ksiz);
703
CB_DATUMCAT(datum, kbuf, ksiz);
704
CB_DATUMCAT(datum, "\n", 1);
434
705
}
435
706
}
436
707
if(doc->attrs && (vbuf = cbmapget(doc->attrs, "", 0, &vsiz)) != NULL){
437
cbdatumcat(datum, "\t", 1);
438
cbdatumcat(datum, vbuf, vsiz);
439
cbdatumcat(datum, "\n", 1);
708
CB_DATUMCAT(datum, "\t", 1);
709
CB_DATUMCAT(datum, vbuf, vsiz);
710
CB_DATUMCAT(datum, "\n", 1);
440
711
}
441
712
return cbdatumtomalloc(datum, NULL);
442
713
}
444
715
445
716
/* Make a snippet of the body text of a document object. */
446
717
char *est_doc_make_snippet(ESTDOC *doc, const CBLIST *words, int wwidth, int hwidth, int awidth){
447
CBDATUM *res, *sbuf;
448
CBMAP *counts;
449
CBLIST *rwords;
450
const char *text, *word, *cval;
451
const unsigned char *rword;
452
unsigned char *rtext, *ctext;
453
int i, j, k, bi, size, wsiz, rwsiz, mywidth, awsiz, csiz;
718
CBDATUM *sbuf;
719
const char *text;
720
char *snippet;
721
int i, size;
454
722
assert(doc && words && wwidth >= 0 && hwidth >= 0 && awidth >= 0);
455
if(!doc->dtexts) doc->dtexts = cblistopen();
456
res = cbdatumopen("", 0);
457
rwords = cblistopen();
458
for(i = 0; i < CB_LISTNUM(words); i++){
459
word = CB_LISTVAL2(words, i, &wsiz);
460
if(wsiz < 1 || !strcmp(word, ESTOPUVSET)) continue;
461
rtext = (unsigned char *)est_uconv_in(word, wsiz, &size);
462
est_canonicalize_text(rtext, size, TRUE);
463
cblistpushbuf(rwords, (char *)rtext, size);
464
}
465
sbuf = cbdatumopen("", 0);
723
if(!doc->dtexts) CB_LISTOPEN(doc->dtexts);
724
CB_DATUMOPEN(sbuf);
466
725
for(i = 0; i < CB_LISTNUM(doc->dtexts); i++){
467
text = CB_LISTVAL2(doc->dtexts, i, &size);
468
if(i > 0) cbdatumcat(sbuf, " ", 1);
469
cbdatumcat(sbuf, text, size);
470
}
471
rtext = (unsigned char *)est_uconv_in(CB_DATUMPTR(sbuf), CB_DATUMSIZE(sbuf), &size);
472
ctext = (unsigned char *)cbmemdup((char *)rtext, size);
473
est_canonicalize_text(ctext, size, FALSE);
474
mywidth = hwidth;
475
if(CB_LISTNUM(rwords) < 1) mywidth *= 3;
476
if(mywidth > wwidth) mywidth = wwidth;
477
for(i = 0; i < size && mywidth > 0; i += 2){
478
mywidth -= est_char_category(rtext[i] * 0x100 + rtext[i+1]) == ESTEASTALPH ? 2 : 1;
479
}
480
awsiz = size - i;
481
if(awsiz > ESTWORDMAXLEN) awsiz = ESTWORDMAXLEN;
482
est_snippet_add_text(rtext, ctext, i, awsiz, res, rwords);
483
wwidth -= hwidth;
484
bi = i + 2;
485
cbdatumcat(res, "\n", 1);
486
if(awidth > 0){
487
counts = cbmapopenex(ESTMINIBNUM);
488
for(i = bi; i < size && wwidth >= 0; i += 2){
489
for(j = 0; j < CB_LISTNUM(rwords); j++){
490
rword = (unsigned char *)CB_LISTVAL2(rwords, j, &rwsiz);
491
if(est_str_fwmatch_wide(ctext + i, size - i, rword, rwsiz) > 0 &&
492
(!(cval = cbmapget(counts, (char *)rword, rwsiz, &csiz)) ||
493
csiz < (wwidth > awidth * 1.2 ? 2 : 1))){
494
cbmapputcat(counts, (char *)rword, rwsiz, "*", 1);
495
if(cbmaprnum(counts) >= CB_LISTNUM(rwords)){
496
cbmapclose(counts);
497
counts = cbmapopenex(ESTMINIBNUM);
498
}
499
mywidth = awidth / 2 + 1;
500
for(k = i - 2; k >= bi && mywidth >= 0; k -= 2){
501
mywidth -= est_char_category(rtext[k] * 0x100 + rtext[k+1]) == ESTEASTALPH ? 2 : 1;
502
}
503
bi = k;
504
mywidth = awidth / 2 + 1;
505
for(k = i + rwsiz + 2; k < size && mywidth >= 0; k += 2){
506
mywidth -= est_char_category(rtext[k] * 0x100 + rtext[k+1]) == ESTEASTALPH ? 2 : 1;
507
}
508
if(k > size) k = size;
509
est_snippet_add_text(rtext + bi, ctext + bi, k - bi, 0, res, rwords);
510
wwidth -= awidth + rwsiz / 2;
511
bi = k + 2;
512
i = bi - 2;
513
cbdatumcat(res, "\n", 1);
514
break;
515
}
516
}
517
}
518
cbmapclose(counts);
519
}
520
free(ctext);
521
free(rtext);
522
cbdatumclose(sbuf);
523
cblistclose(rwords);
524
return cbdatumtomalloc(res, NULL);
726
text = CB_LISTVAL2(doc->dtexts, i, size);
727
if(i > 0) CB_DATUMCAT(sbuf, " ", 1);
728
CB_DATUMCAT(sbuf, text, size);
729
}
730
snippet = est_make_snippet(CB_DATUMPTR(sbuf), CB_DATUMSIZE(sbuf),
731
words, wwidth, hwidth, awidth);
732
CB_DATUMCLOSE(sbuf);
733
return snippet;
525
734
}
526
735
527
736
538
747
cond->phrase = NULL;
539
748
cond->gstep = 2;
540
749
cond->tfidf = TRUE;
541
cond->simple = FALSE;
750
cond->pmode = ESTPMUSUAL;
751
cond->cbxpn = NULL;
542
752
cond->attrs = NULL;
543
753
cond->order = NULL;
544
754
cond->max = -1;
755
cond->skip = 0;
756
cond->auxmin = ESTAUXMIN;
757
cond->auxwords = NULL;
545
758
cond->scfb = FALSE;
546
759
cond->scores = NULL;
547
760
cond->snum = 0;
548
761
cond->opts = 0;
549
762
cond->ecllim = -1.0;
550
763
cond->shadows = NULL;
764
cond->distinct = NULL;
765
cond->mask = 0;
551
766
return cond;
552
767
}
553
768
556
771
void est_cond_delete(ESTCOND *cond){
557
772
assert(cond);
558
773
if(cond->shadows) cbmapclose(cond->shadows);
774
if(cond->auxwords) cbmapclose(cond->auxwords);
559
775
if(cond->scores) free(cond->scores);
560
776
if(cond->order) free(cond->order);
561
if(cond->attrs) cblistclose(cond->attrs);
777
if(cond->attrs) CB_LISTCLOSE(cond->attrs);
562
778
if(cond->phrase) free(cond->phrase);
779
if(cond->distinct) free(cond->distinct);
563
780
free(cond);
564
781
}
565
782
582
799
expr++;
583
800
}
584
801
if(*expr == '\0') return;
585
if(!cond->attrs) cond->attrs = cblistopen();
586
cblistpush(cond->attrs, expr, -1);
802
if(!cond->attrs) CB_LISTOPEN(cond->attrs);
803
CB_LISTPUSH(cond->attrs, expr, strlen(expr));
587
804
}
588
805
589
806
606
823
}
607
824
608
825
826
/* Set the number of skipped documents of a condition object. */
827
void est_cond_set_skip(ESTCOND *cond, int skip){
828
assert(cond && skip >= 0);
829
cond->skip = skip;
830
}
831
832
609
833
/* Set options of retrieval of a condition object. */
610
834
void est_cond_set_options(ESTCOND *cond, int options){
611
835
assert(cond);
614
838
if(options & ESTCONDFAST) cond->gstep = 3;
615
839
if(options & ESTCONDAGITO) cond->gstep = 4;
616
840
if(options & ESTCONDNOIDF) cond->tfidf = FALSE;
617
if(options & ESTCONDSIMPLE) cond->simple = TRUE;
841
if(options & ESTCONDSIMPLE) cond->pmode = ESTPMSIMPLE;
842
if(options & ESTCONDROUGH) cond->pmode = ESTPMROUGH;
843
if(options & ESTCONDUNION) cond->pmode = ESTPMUNION;
844
if(options & ESTCONDISECT) cond->pmode = ESTPMISECT;
618
845
if(options & ESTCONDSCFB) cond->scfb = TRUE;
619
846
cond->opts |= options;
620
847
}
621
848
622
849
850
/* Set permission to adopt result of the auxiliary index. */
851
void est_cond_set_auxiliary(ESTCOND *cond, int min){
852
assert(cond);
853
cond->auxmin = min;
854
}
855
856
623
857
/* Set the upper limit of similarity for document eclipse. */
624
858
void est_cond_set_eclipse(ESTCOND *cond, double limit){
625
859
assert(cond);
626
if(limit > 0.0 && limit < 1.0) cond->ecllim = limit;
860
if(limit > 0.0) cond->ecllim = limit;
861
}
862
863
864
/* Set the attribute distinction filter. */
865
void est_cond_set_distinct(ESTCOND *cond, const char *name){
866
assert(cond && name);
867
while(*name > '\0' && *name <= ' '){
868
name++;
869
}
870
if(*name == '\0') return;
871
if(cond->distinct) free(cond->distinct);
872
cond->distinct = cbmemdup(name, -1);
873
}
874
875
876
/* Set the mask of targets of meta search. */
877
void est_cond_set_mask(ESTCOND *cond, int mask){
878
assert(cond);
879
cond->mask = mask & 0x7fffffff;
627
880
}
628
881
629
882
633
886
*************************************************************************************************/
634
887
635
888
889
/* Inode map for duplication check. */
890
CBMAP *est_inodes = NULL;
891
892
636
893
/* Get the string of an error code. */
637
894
const char *est_err_msg(int ecode){
638
895
switch(ecode){
655
912
DEPOT *metadb;
656
913
ESTIDX *idxdb;
657
914
CURIA *attrdb, *textdb, *kwddb;
658
VILLA *fwmdb, *listdb;
659
char path[ESTPATHBUFSIZ], vbuf[ESTNUMBUFSIZ];
660
int domode, comode, vomode, idxnum, dseq, dnum, amode, vsiz;
915
VILLA *fwmdb, *auxdb, *xfmdb, *listdb;
916
CBMAP *aidxs;
917
CBLIST *list;
918
ESTATTRIDX attridx;
919
void *aidxdb;
920
const char *elem;
921
char path[ESTPATHBUFSIZ], vbuf[ESTNUMBUFSIZ], *dec;
922
int i, inode, domode, comode, vomode, flags, idxnum, dseq, dnum;
923
int amode, zmode, smode, vsiz, type, crdnum;
924
double bdiam, ddiam;
661
925
assert(name && ecp);
926
if(!est_inodes){
927
est_inodes = cbmapopenex(ESTMINIBNUM);
928
cbglobalgc(est_inodes, est_inodes_delete);
929
}
662
930
*ecp = ESTENOERR;
663
931
if((omode & ESTDBWRITER) && (omode & ESTDBCREAT) && !est_mkdir(name)){
664
932
switch(errno){
672
940
return NULL;
673
941
}
674
942
}
943
if((inode = est_inode(name)) < 1){
944
*ecp = ESTEIO;
945
return NULL;
946
}
947
if(cbmapget(est_inodes, (char *)&inode, sizeof(int), NULL)){
948
*ecp = ESTEACCES;
949
return NULL;
950
}
675
951
domode = DP_OREADER;
676
952
comode = CR_OREADER;
677
953
vomode = VL_OREADER;
678
954
if(omode & ESTDBWRITER){
679
955
domode = DP_OWRITER;
680
956
comode = CR_OWRITER;
681
vomode = VL_OWRITER | (ESTUSEZLIB ? VL_OZCOMP : 0);
957
vomode = VL_OWRITER;
958
if(ESTUSEBZIP){
959
vomode |= VL_OXCOMP;
960
} else if(ESTUSELZO){
961
vomode |= VL_OYCOMP;
962
} else if(ESTUSEZLIB){
963
vomode |= VL_OZCOMP;
964
}
682
965
if(omode & ESTDBCREAT){
683
966
domode |= DP_OCREAT;
684
967
comode |= CR_OCREAT;
700
983
comode |= CR_OLCKNB;
701
984
vomode |= VL_OLCKNB;
702
985
}
986
flags = 0;
703
987
idxnum = 0;
704
988
dseq = 0;
705
989
dnum = 0;
706
amode = ESTAMNORMAL;
990
amode = 0;
991
zmode = 0;
992
smode = 0;
993
if(omode & ESTDBSMALL){
994
bdiam = ESTDBSBRAT;
995
ddiam = ESTDBSDRAT;
996
} else if(omode & ESTDBLARGE){
997
bdiam = ESTDBLBRAT;
998
ddiam = ESTDBLDRAT;
999
} else if(omode & ESTDBHUGE){
1000
bdiam = ESTDBHBRAT;
1001
ddiam = ESTDBHDRAT;
1002
} else if(omode & ESTDBHUGE2){
1003
bdiam = ESTDBHBRAT * ESTDBH2RAT;
1004
ddiam = ESTDBHDRAT * ESTDBH2RAT;
1005
} else if(omode & ESTDBHUGE3){
1006
bdiam = ESTDBHBRAT * ESTDBH3RAT;
1007
ddiam = ESTDBHDRAT * ESTDBH3RAT;
1008
} else {
1009
bdiam = 1.0;
1010
ddiam = 1.0;
1011
}
707
1012
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTMETADBNAME);
708
1013
if((metadb = dpopen(path, domode, ESTMINIBNUM)) != NULL){
1014
flags = dpgetflags(metadb);
1015
if(dprnum(metadb) < 1){
1016
if(omode & ESTDBPERFNG){
1017
flags |= ESTDFPERFNG;
1018
} else if(omode & ESTDBCHRCAT){
1019
flags |= ESTDFCHRCAT;
1020
}
1021
if(ESTUSEBZIP){
1022
flags |= ESTDFBZIP;
1023
} else if(ESTUSELZO){
1024
flags |= ESTDFLZO;
1025
} else if(ESTUSEZLIB){
1026
flags |= ESTDFZLIB;
1027
}
1028
if(omode & ESTDBSCVOID){
1029
flags |= ESTDFSCVOID;
1030
} else if(omode & ESTDBSCINT){
1031
flags |= ESTDFSCINT;
1032
} else if(omode & ESTDBSCASIS){
1033
flags |= ESTDFSCASIS;
1034
}
1035
dpsetflags(metadb, flags);
1036
}
709
1037
if((vsiz = dpgetwb(metadb, ESTKEYIDXNUM, -1, 0, ESTNUMBUFSIZ - 1, vbuf)) > 0){
710
1038
vbuf[vsiz] = '\0';
711
1039
idxnum = atoi(vbuf);
718
1046
vbuf[vsiz] = '\0';
719
1047
dnum = atoi(vbuf);
720
1048
}
721
if((vsiz = dpgetwb(metadb, ESTKEYAMODE, -1, 0, ESTNUMBUFSIZ - 1, vbuf)) > 0){
722
vbuf[vsiz] = '\0';
723
amode = atoi(vbuf);
724
} else if(omode & ESTDBPERFNG){
725
amode = ESTAMPERFNG;
726
}
727
}
728
if(!metadb){
1049
if(flags & ESTDFPERFNG){
1050
amode = ESTDFPERFNG;
1051
} else if(flags & ESTDFCHRCAT){
1052
amode = ESTDFCHRCAT;
1053
}
1054
if(flags & ESTDFZLIB){
1055
zmode = ESTDFZLIB;
1056
} else if(flags & ESTDFLZO){
1057
zmode = ESTDFLZO;
1058
} else if(flags & ESTDFBZIP){
1059
zmode = ESTDFBZIP;
1060
}
1061
if(flags & ESTDFSCVOID){
1062
smode = ESTDFSCVOID;
1063
} else if(flags & ESTDFSCINT){
1064
smode = ESTDFSCINT;
1065
} else if(flags & ESTDFSCASIS){
1066
smode = ESTDFSCASIS;
1067
}
1068
} else {
729
1069
*ecp = dpecode == DP_ELOCK ? ESTELOCK : ESTEDB;
730
1070
return NULL;
731
1071
}
732
1072
if(idxnum < 1) idxnum = 1;
733
1073
if(dseq < 0) dseq = 0;
734
1074
if(dnum < 0) dnum = 0;
1075
crdnum = vlcrdnum;
735
1076
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTIDXDBNAME);
736
1077
idxdb = est_idx_open(path, vomode, idxnum);
737
1078
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTFWMDBNAME);
1079
vlcrdnum = ESTVLCRDNUM;
738
1080
fwmdb = vlopen(path, vomode, VL_CMPLEX);
1081
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTAUXDBNAME);
1082
vlcrdnum = ESTVLCRDNAUX;
1083
auxdb = vlopen(path, vomode, VL_CMPLEX);
1084
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTXFMDBNAME);
1085
vlcrdnum = ESTVLCRDNUM;
1086
xfmdb = vlopen(path, vomode, VL_CMPLEX);
739
1087
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTATTRDBNAME);
740
attrdb = cropen(path, comode, ESTATTRDBBNUM, ESTATTRDBDNUM);
1088
attrdb = cropen(path, comode, ESTATTRDBBNUM * bdiam, ESTATTRDBDNUM * ddiam);
741
1089
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTTEXTDBNAME);
742
textdb = cropen(path, comode, ESTTEXTDBBNUM, ESTTEXTDBDNUM);
1090
textdb = cropen(path, comode, ESTTEXTDBBNUM * bdiam, ESTTEXTDBDNUM * ddiam);
743
1091
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTKWDDBNAME);
744
kwddb = cropen(path, comode, ESTKWDDBBNUM, ESTKWDDBDNUM);
1092
kwddb = cropen(path, comode, ESTKWDDBBNUM * bdiam, ESTKWDDBDNUM * ddiam);
745
1093
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTLISTDBNAME);
1094
vlcrdnum = ESTVLCRDNUM;
746
1095
listdb = vlopen(path, vomode, VL_CMPLEX);
747
if(!metadb || !idxdb || !fwmdb || !attrdb ||!textdb || !listdb){
1096
vlcrdnum = crdnum;
1097
if(!idxdb || !fwmdb || !auxdb || !xfmdb || !attrdb ||!textdb || !kwddb || !listdb){
748
1098
if(listdb) vlclose(listdb);
749
1099
if(kwddb) crclose(kwddb);
750
1100
if(textdb) crclose(textdb);
751
1101
if(attrdb) crclose(attrdb);
1102
if(xfmdb) vlclose(xfmdb);
1103
if(auxdb) vlclose(auxdb);
752
1104
if(fwmdb) vlclose(fwmdb);
753
1105
if(idxdb) est_idx_close(idxdb);
754
1106
dpclose(metadb);
756
1108
return NULL;
757
1109
}
758
1110
if(omode & ESTDBWRITER){
1111
est_idx_set_tuning(idxdb, amode == ESTDFPERFNG ? ESTIDXDBLRMA : ESTIDXDBLRM, ESTIDXDBNIM,
1112
ESTIDXDBLCN, ESTIDXDBNCN, ESTIDXDBFBP);
1113
est_idx_set_current(idxdb);
1114
vlsettuning(fwmdb, ESTFWMDBLRM, ESTFWMDBNIM, ESTFWMDBLCN, ESTFWMDBNCN);
1115
vlsetfbpsiz(fwmdb, ESTFWMDBFBP);
1116
vlsettuning(auxdb, ESTAUXDBLRM, ESTAUXDBNIM, ESTAUXDBLCN, ESTAUXDBNCN);
1117
vlsetfbpsiz(auxdb, ESTAUXDBFBP);
1118
vlsettuning(xfmdb, ESTXFMDBLRM, ESTXFMDBNIM, ESTXFMDBLCN, ESTXFMDBNCN);
1119
vlsetfbpsiz(xfmdb, ESTXFMDBFBP);
759
1120
crsetalign(attrdb, ESTATTRDBALN);
1121
crsetfbpsiz(attrdb, ESTATTRDBFBP);
760
1122
crsetalign(textdb, ESTTEXTDBALN);
1123
crsetfbpsiz(textdb, ESTTEXTDBFBP);
761
1124
crsetalign(kwddb, ESTKWDDBALN);
762
est_idx_set_tuning(idxdb, ESTIDXDBLRM, ESTIDXDBNIM, ESTIDXDBLCN, ESTIDXDBNCN);
763
est_idx_set_current(idxdb);
764
vlsettuning(fwmdb, ESTFWMDBLRM, ESTFWMDBNIM, ESTFWMDBLCN, ESTFWMDBNCN);
1125
crsetfbpsiz(kwddb, ESTKWDDBFBP);
765
1126
vlsettuning(listdb, ESTLISTDBLRM, ESTLISTDBNIM, ESTLISTDBLCN, ESTLISTDBNCN);
1127
vlsetfbpsiz(listdb, ESTLISTDBFBP);
766
1128
} else {
767
est_idx_set_tuning(idxdb, -1, -1, ESTIDXDBRLCN, ESTIDXDBRNCN);
1129
est_idx_set_tuning(idxdb, -1, -1,
1130
amode == ESTDFPERFNG ? ESTIDXDBRLCNA : ESTIDXDBRLCN, ESTIDXDBRNCN, -1);
768
1131
vlsettuning(fwmdb, -1, -1, ESTFWMDBLCN, ESTFWMDBNCN);
1132
vlsettuning(auxdb, -1, -1, ESTAUXDBRLCN, ESTAUXDBRNCN);
1133
vlsettuning(xfmdb, -1, -1, ESTXFMDBLCN, ESTXFMDBNCN);
769
1134
vlsettuning(listdb, -1, -1, ESTLISTDBLCN, ESTLISTDBNCN);
770
1135
}
1136
if((omode & ESTDBWRITER) && (omode & ESTDBTRUNC) && (list = cbdirlist(name)) != NULL){
1137
for(i = 0; i < CB_LISTNUM(list); i++){
1138
elem = CB_LISTVAL(list, i);
1139
if(cbstrfwmatch(elem, ESTAISEQPREF) || cbstrfwmatch(elem, ESTAISTRPREF) ||
1140
cbstrfwmatch(elem, ESTAINUMPREF)){
1141
sprintf(path, "%s%c%s", name, ESTPATHCHR, elem);
1142
if(unlink(path) == -1) est_rmdir_rec(path);
1143
}
1144
}
1145
CB_LISTCLOSE(list);
1146
}
1147
aidxs = cbmapopenex(ESTMINIBNUM);
1148
if((list = cbdirlist(name)) != NULL){
1149
for(i = 0; i < CB_LISTNUM(list); i++){
1150
elem = CB_LISTVAL(list, i);
1151
dec = NULL;
1152
type = -1;
1153
if(cbstrfwmatch(elem, ESTAISEQPREF)){
1154
dec = est_hex_decode(elem + strlen(ESTAISEQPREF));
1155
type = ESTIDXATTRSEQ;
1156
} else if(cbstrfwmatch(elem, ESTAISTRPREF)){
1157
dec = est_hex_decode(elem + strlen(ESTAISTRPREF));
1158
type = ESTIDXATTRSTR;
1159
} else if(cbstrfwmatch(elem, ESTAINUMPREF)){
1160
dec = est_hex_decode(elem + strlen(ESTAINUMPREF));
1161
type = ESTIDXATTRNUM;
1162
}
1163
if(dec){
1164
sprintf(path, "%s%c%s", name, ESTPATHCHR, elem);
1165
switch(type){
1166
case ESTIDXATTRSTR:
1167
if((aidxdb = vlopen(path, vomode, VL_CMPLEX)) != NULL){
1168
vlsettuning(aidxdb, ESTAIDXLRM, ESTAIDXNIM, ESTAIDXLCN, ESTAIDXNCN);
1169
vlsetfbpsiz(aidxdb, ESTAIDXVLFBP);
1170
attridx.db = aidxdb;
1171
attridx.type = type;
1172
cbmapput(aidxs, dec, -1, (char *)&attridx, sizeof(ESTATTRIDX), FALSE);
1173
}
1174
break;
1175
case ESTIDXATTRNUM:
1176
if((aidxdb = vlopen(path, vomode, est_aidx_numcmp)) != NULL){
1177
vlsettuning(aidxdb, ESTAIDXLRM, ESTAIDXNIM, ESTAIDXLCN, ESTAIDXNCN);
1178
vlsetfbpsiz(aidxdb, ESTAIDXVLFBP);
1179
attridx.db = aidxdb;
1180
attridx.type = type;
1181
cbmapput(aidxs, dec, -1, (char *)&attridx, sizeof(ESTATTRIDX), FALSE);
1182
}
1183
break;
1184
default:
1185
if((aidxdb = dpopen(path, domode, crbnum(attrdb) / ESTAIBDIAM)) != NULL){
1186
dpsetfbpsiz(aidxdb, ESTAIDXDPFBP);
1187
attridx.db = aidxdb;
1188
attridx.type = type;
1189
cbmapput(aidxs, dec, -1, (char *)&attridx, sizeof(ESTATTRIDX), FALSE);
1190
}
1191
break;
1192
}
1193
free(dec);
1194
}
1195
}
1196
CB_LISTCLOSE(list);
1197
}
771
1198
CB_MALLOC(db, sizeof(ESTDB));
772
1199
db->name = cbmemdup(name, -1);
1200
db->inode = inode;
773
1201
db->metadb = metadb;
774
1202
db->idxdb = idxdb;
775
1203
db->fwmdb = fwmdb;
1204
db->auxdb = auxdb;
1205
db->xfmdb = xfmdb;
776
1206
db->attrdb = attrdb;
777
1207
db->textdb = textdb;
778
1208
db->kwddb = kwddb;
779
1209
db->listdb = listdb;
1210
db->aidxs = aidxs;
1211
CB_LISTOPEN(db->pdocs);
1212
db->puris = NULL;
780
1213
db->ecode = ESTENOERR;
781
1214
db->fatal = FALSE;
782
1215
db->dseq = dseq;
783
1216
db->dnum = dnum;
784
1217
db->amode = amode;
1218
db->zmode = zmode;
1219
db->smode = smode;
785
1220
if(omode & ESTDBWRITER){
786
1221
db->idxcc = cbmapopenex(ESTIDXCCBNUM);
1222
db->auxcc = cbmapopenex(ESTAUXCCBNUM);
787
1223
db->icsiz = 0;
788
1224
db->icmax = ESTIDXCCMAX;
789
1225
db->outcc = cbmapopenex(ESTOUTCCBNUM);
790
1226
} else {
791
1227
db->idxcc = cbmapopenex(1);
1228
db->auxcc = cbmapopenex(1);
792
1229
db->icsiz = 0;
793
1230
db->icmax = 0;
794
1231
db->outcc = cbmapopenex(1);
806
1243
db->spacc = NULL;
807
1244
db->scmnum = 0;
808
1245
db->scname = NULL;
809
db->cbinfo = NULL;
1246
db->infocb = NULL;
1247
db->infoop = NULL;
810
1248
db->dfdb = NULL;
811
1249
db->metacc = NULL;
1250
db->wildmax = ESTWILDMAX;
1251
db->flsflag = FALSE;
812
1252
db->intflag = FALSE;
1253
cbmapput(est_inodes, (char *)&inode, sizeof(int), (char *)&db, sizeof(ESTDB *), FALSE);
813
1254
return db;
814
1255
}
815
1256
816
1257
817
1258
/* Close a database. */
818
1259
int est_db_close(ESTDB *db, int *ecp){
1260
ESTATTRIDX *attridx;
1261
const char *kbuf;
819
1262
int err;
820
1263
assert(db && ecp);
821
1264
*ecp = ESTENOERR;
822
1265
err = FALSE;
1266
cbmapout(est_inodes, (char *)&(db->inode), sizeof(int));
823
1267
if(dpwritable(db->metadb)){
824
if(!est_db_flush(db, -1) || !est_db_write_meta(db)) err = TRUE;
1268
if(!est_db_flush(db, -1)) err = TRUE;
1269
if(!est_db_write_meta(db)) err = TRUE;
825
1270
}
826
1271
est_db_inform(db, "closing");
827
1272
if(db->metacc) cbmapclose(db->metacc);
835
1280
cbmapclose(db->attrcc);
836
1281
cbmapclose(db->keycc);
837
1282
cbmapclose(db->outcc);
1283
cbmapclose(db->auxcc);
838
1284
cbmapclose(db->idxcc);
1285
if(db->puris) cbmapclose(db->puris);
1286
CB_LISTCLOSE(db->pdocs);
1287
cbmapiterinit(db->aidxs);
1288
while((kbuf = cbmapiternext(db->aidxs, NULL)) != NULL){
1289
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
1290
switch(attridx->type){
1291
case ESTIDXATTRSTR:
1292
case ESTIDXATTRNUM:
1293
if(!vlclose(attridx->db)) err = TRUE;
1294
break;
1295
default:
1296
if(!dpclose(attridx->db)) err = TRUE;
1297
break;
1298
}
1299
}
1300
cbmapclose(db->aidxs);
839
1301
if(!vlclose(db->listdb)) err = TRUE;
840
1302
if(!crclose(db->kwddb)) err = TRUE;
841
1303
if(!crclose(db->textdb)) err = TRUE;
842
1304
if(!crclose(db->attrdb)) err = TRUE;
1305
if(!vlclose(db->xfmdb)) err = TRUE;
1306
if(!vlclose(db->auxdb)) err = TRUE;
843
1307
if(!vlclose(db->fwmdb)) err = TRUE;
844
1308
if(!est_idx_close(db->idxdb)) err = TRUE;
845
1309
if(!dpclose(db->metadb)) err = TRUE;
869
1333
}
870
1334
871
1335
1336
/* Add an index for narrowing or sorting with document attributes. */
1337
int est_db_add_attr_index(ESTDB *db, const char *name, int type){
1338
ESTATTRIDX attridx;
1339
ESTSCORE *scores;
1340
void *aidxdb;
1341
char path[ESTPATHBUFSIZ], *enc, *vbuf;
1342
int i, domode, vomode, crdnum, err, snum;
1343
assert(db && name);
1344
if(!dpwritable(db->metadb)){
1345
db->ecode = ESTEACCES;
1346
return FALSE;
1347
}
1348
if(cbmapget(db->aidxs, name, -1, NULL)){
1349
db->ecode = ESTEMISC;
1350
return FALSE;
1351
}
1352
enc = est_hex_encode(name);
1353
switch(type){
1354
case ESTIDXATTRSEQ:
1355
sprintf(path, "%s%c%s%s", db->name, ESTPATHCHR, ESTAISEQPREF, enc);
1356
break;
1357
case ESTIDXATTRSTR:
1358
sprintf(path, "%s%c%s%s", db->name, ESTPATHCHR, ESTAISTRPREF, enc);
1359
break;
1360
case ESTIDXATTRNUM:
1361
sprintf(path, "%s%c%s%s", db->name, ESTPATHCHR, ESTAINUMPREF, enc);
1362
break;
1363
default:
1364
free(enc);
1365
db->ecode = ESTEINVAL;
1366
return FALSE;
1367
}
1368
free(enc);
1369
domode = DP_OWRITER | DP_OCREAT | DP_OTRUNC;
1370
vomode = VL_OWRITER | VL_OCREAT | VL_OTRUNC;
1371
if(ESTUSEBZIP){
1372
vomode |= VL_OXCOMP;
1373
} else if(ESTUSELZO){
1374
vomode |= VL_OYCOMP;
1375
} else if(ESTUSEZLIB){
1376
vomode |= VL_OZCOMP;
1377
}
1378
err = FALSE;
1379
crdnum = vlcrdnum;
1380
switch(type){
1381
case ESTIDXATTRSTR:
1382
vlcrdnum = ESTVLCRDNUM;
1383
if(!(aidxdb = vlopen(path, vomode, VL_CMPLEX))){
1384
db->ecode = ESTEDB;
1385
vlcrdnum = crdnum;
1386
return FALSE;
1387
}
1388
vlsettuning(aidxdb, ESTAIDXLRM, ESTAIDXNIM, ESTAIDXLCN, ESTAIDXNCN);
1389
vlsetfbpsiz(aidxdb, ESTAIDXVLFBP);
1390
if(est_db_doc_num(db) > 0){
1391
scores = est_search_uvset(db, &snum, NULL, TRUE);
1392
for(i = 0; i < snum; i++){
1393
if((vbuf = est_db_get_doc_attr(db, scores[i].id, name)) != NULL){
1394
if(!est_aidx_attr_put(aidxdb, scores[i].id, vbuf, strlen(vbuf))){
1395
db->ecode = ESTEDB;
1396
db->fatal = TRUE;
1397
err = TRUE;
1398
}
1399
free(vbuf);
1400
}
1401
if(i % (ESTCCCBFREQ / 10) == 0) est_db_inform(db, "entering existing attributes");
1402
}
1403
free(scores);
1404
}
1405
break;
1406
case ESTIDXATTRNUM:
1407
vlcrdnum = ESTVLCRDNUM;
1408
if(!(aidxdb = vlopen(path, vomode, est_aidx_numcmp))){
1409
db->ecode = ESTEDB;
1410
vlcrdnum = crdnum;
1411
return FALSE;
1412
}
1413
vlsettuning(aidxdb, ESTAIDXLRM, ESTAIDXNIM, ESTAIDXLCN, ESTAIDXNCN);
1414
vlsetfbpsiz(aidxdb, ESTAIDXVLFBP);
1415
if(est_db_doc_num(db) > 0){
1416
scores = est_search_uvset(db, &snum, NULL, TRUE);
1417
for(i = 0; i < snum; i++){
1418
if((vbuf = est_db_get_doc_attr(db, scores[i].id, name)) != NULL){
1419
if(!est_aidx_attr_put(aidxdb, scores[i].id, vbuf, strlen(vbuf))){
1420
db->ecode = ESTEDB;
1421
db->fatal = TRUE;
1422
err = TRUE;
1423
}
1424
free(vbuf);
1425
}
1426
if(i % (ESTCCCBFREQ / 10) == 0) est_db_inform(db, "entering existing attributes");
1427
}
1428
free(scores);
1429
}
1430
break;
1431
default:
1432
if(!(aidxdb = dpopen(path, domode, crbnum(db->attrdb) * ESTAIBDIAM))){
1433
db->ecode = ESTEDB;
1434
vlcrdnum = crdnum;
1435
return FALSE;
1436
}
1437
dpsetfbpsiz(aidxdb, ESTAIDXDPFBP);
1438
if(est_db_doc_num(db) > 0){
1439
scores = est_search_uvset(db, &snum, NULL, TRUE);
1440
for(i = 0; i < snum; i++){
1441
if((vbuf = est_db_get_doc_attr(db, scores[i].id, name)) != NULL){
1442
if(!est_aidx_seq_put(aidxdb, scores[i].id, vbuf, strlen(vbuf))){
1443
db->ecode = ESTEDB;
1444
db->fatal = TRUE;
1445
err = TRUE;
1446
}
1447
free(vbuf);
1448
}
1449
if(i % (ESTCCCBFREQ / 10) == 0) est_db_inform(db, "entering existing attributes");
1450
}
1451
free(scores);
1452
}
1453
break;
1454
}
1455
vlcrdnum = crdnum;
1456
attridx.db = aidxdb;
1457
attridx.type = type;
1458
cbmapput(db->aidxs, name, -1, (char *)&attridx, sizeof(ESTATTRIDX), FALSE);
1459
return err ? FALSE : TRUE;
1460
}
1461
1462
872
1463
/* Flush index words in the cache of a database. */
873
1464
int est_db_flush(ESTDB *db, int max){
1465
ESTATTRIDX *attridx;
874
1466
CBMAP *ids;
875
1467
CBLIST *keys;
876
1468
CBDATUM *nval;
877
const char *kbuf, *vbuf, *rp, *pv;
878
char *tbuf;
879
int i, j, inc, err, ksiz, vsiz, rnum, id, dnum, tsiz;
1469
const char *kbuf, *vbuf, *rp, *pv, *ep;
1470
char *tbuf, *wp, numbuf[ESTNUMBUFSIZ];
1471
int i, j, inc, err, ksiz, vsiz, rnum, len, id, sum, cid, vnum, lid, dnum, tsiz, vstep;
880
1472
assert(db);
881
1473
if(!dpwritable(db->metadb)){
882
1474
db->ecode = ESTEACCES;
883
1475
return FALSE;
884
1476
}
885
if(cbmaprnum(db->idxcc) < 1 && cbmaprnum(db->outcc) < 1) return TRUE;
1477
if(max < 1 || max >= INT_MAX){
1478
if(!est_db_write_meta(db)) err = TRUE;
1479
if(!dpmemflush(db->metadb)) err = TRUE;
1480
if(!crmemflush(db->attrdb)) err = TRUE;
1481
if(!crmemflush(db->textdb)) err = TRUE;
1482
if(!crmemflush(db->kwddb)) err = TRUE;
1483
if(!vlmemflush(db->listdb)) err = TRUE;
1484
cbmapiterinit(db->aidxs);
1485
while((kbuf = cbmapiternext(db->aidxs, NULL)) != NULL){
1486
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
1487
switch(attridx->type){
1488
case ESTIDXATTRSTR:
1489
case ESTIDXATTRNUM:
1490
if(!vlmemflush(attridx->db)) err = TRUE;
1491
break;
1492
default:
1493
if(!dpmemflush(attridx->db)) err = TRUE;
1494
break;
1495
}
1496
}
1497
}
1498
if(cbmaprnum(db->idxcc) < 1 && cbmaprnum(db->auxcc) < 1 && cbmaprnum(db->outcc) < 1)
1499
return TRUE;
1500
db->flsflag = TRUE;
886
1501
db->intflag = FALSE;
887
1502
inc = est_db_used_cache_size(db) > db->icmax;
888
1503
err = FALSE;
889
keys = cblistopen();
1504
CB_LISTOPEN(keys);
890
1505
cbmapiterinit(db->idxcc);
891
1506
while((kbuf = cbmapiternext(db->idxcc, &ksiz)) != NULL){
892
cblistpush(keys, kbuf, ksiz);
1507
CB_LISTPUSH(keys, kbuf, ksiz);
893
1508
}
894
1509
rnum = CB_LISTNUM(keys);
895
1510
cblistsort(keys);
896
1511
if(max > 0){
897
1512
while(CB_LISTNUM(keys) > max){
898
free(cblistpop(keys, NULL));
1513
CB_LISTDROP(keys);
899
1514
}
900
1515
}
901
1516
for(i = 0; i < CB_LISTNUM(keys); i++){
902
kbuf = CB_LISTVAL2(keys, i, &ksiz);
1517
kbuf = CB_LISTVAL2(keys, i, ksiz);
903
1518
vbuf = cbmapget(db->idxcc, kbuf, ksiz, &vsiz);
904
if(!est_idx_add(db->idxdb, kbuf, ksiz, vbuf, vsiz) ||
1519
if(!est_idx_add(db->idxdb, kbuf, ksiz, vbuf, vsiz, db->smode) ||
905
1520
(!vlput(db->fwmdb, kbuf, ksiz, "", 0, VL_DKEEP) && dpecode != DP_EKEEP)){
906
1521
err = TRUE;
907
1522
break;
918
1533
}
919
1534
if(max > 0 && db->intflag && i > 0 && i % ESTCCIRSLOT == 0) break;
920
1535
}
921
cblistclose(keys);
1536
CB_LISTCLOSE(keys);
922
1537
if(cbmaprnum(db->idxcc) < 1){
923
1538
cbmapclose(db->idxcc);
924
1539
db->idxcc = cbmapopenex(rnum > ESTIDXCCBNUM ? rnum * 1.5 : ESTIDXCCBNUM);
1540
if(cbmaprnum(db->auxcc) > 0){
1541
CB_LISTOPEN(keys);
1542
cbmapiterinit(db->auxcc);
1543
while((kbuf = cbmapiternext(db->auxcc, &ksiz)) != NULL){
1544
CB_LISTPUSH(keys, kbuf, ksiz);
1545
}
1546
cblistsort(keys);
1547
for(i = 0; i < CB_LISTNUM(keys); i++){
1548
kbuf = CB_LISTVAL2(keys, i, ksiz);
1549
vbuf = cbmapget(db->auxcc, kbuf, ksiz, &vsiz);
1550
if(!vlput(db->auxdb, kbuf, ksiz, vbuf, vsiz, VL_DCAT)){
1551
err = TRUE;
1552
break;
1553
}
1554
len = sprintf(numbuf, "%d", vlvsiz(db->auxdb, kbuf, ksiz) / (int)(sizeof(int) * 2));
1555
if(!vlput(db->xfmdb, kbuf, ksiz, numbuf, len, VL_DOVER)){
1556
err = TRUE;
1557
break;
1558
}
1559
cbmapout(db->auxcc, kbuf, ksiz);
1560
db->icsiz -= vsiz;
1561
if(i % ESTCCCBFREQ == 0) est_db_inform(db, "flushing auxiliary keywords");
1562
if(max > 0 && db->intflag && i > 0 && i % ESTCCIRSLOT == 0) break;
1563
}
1564
CB_LISTCLOSE(keys);
1565
if(cbmaprnum(db->auxcc) < 1){
1566
cbmapclose(db->auxcc);
1567
db->auxcc = cbmapopenex(ESTAUXCCBNUM);
1568
}
1569
}
925
1570
}
926
1571
if(max < 1 && cbmaprnum(db->outcc) > 0){
927
1572
ids = cbmapopen();
928
keys = cblistopen();
1573
CB_LISTOPEN(keys);
929
1574
cbmapiterinit(db->outcc);
930
1575
while((kbuf = cbmapiternext(db->outcc, &ksiz)) != NULL){
931
1576
if(*kbuf == '\t'){
932
1577
id = atoi(kbuf + 1);
933
1578
cbmapput(ids, (char *)&id, sizeof(int), "", 0, FALSE);
934
1579
} else {
935
cblistpush(keys, kbuf, ksiz);
1580
CB_LISTPUSH(keys, kbuf, ksiz);
936
1581
}
937
1582
}
938
1583
cblistsort(keys);
939
1584
dnum = est_idx_dnum(db->idxdb);
940
1585
for(i = 0; i < CB_LISTNUM(keys); i++){
941
kbuf = CB_LISTVAL2(keys, i, &ksiz);
942
for(j = 0; j < dnum; j++){
943
if((tbuf = est_idx_get_one(db->idxdb, j, kbuf, ksiz, &tsiz)) != NULL){
944
nval = cbdatumopen("", 0);
1586
kbuf = CB_LISTVAL2(keys, i, ksiz);
1587
if(kbuf[0] == ' '){
1588
if((tbuf = vlget(db->auxdb, kbuf + 1, ksiz - 1, &tsiz)) != NULL){
945
1589
rp = tbuf;
946
while(rp < tbuf + tsiz){
947
pv = rp;
948
rp += 5;
949
while(*rp != 0x0){
950
rp += 2;
1590
wp = tbuf;
1591
ep = tbuf + tsiz;
1592
while(rp < ep){
1593
if(!cbmapget(ids, rp, sizeof(int), NULL)){
1594
memmove(wp, rp, sizeof(int) * 2);
1595
wp += sizeof(int) * 2;
951
1596
}
952
rp++;
953
if(!cbmapget(ids, pv, sizeof(int), NULL)) cbdatumcat(nval, pv, rp - pv);
954
}
955
if(!est_idx_put_one(db->idxdb, j, kbuf, ksiz, CB_DATUMPTR(nval), CB_DATUMSIZE(nval)))
956
err = TRUE;
957
cbdatumclose(nval);
1597
rp += sizeof(int) * 2;
1598
}
1599
if(wp > tbuf){
1600
if(!vlput(db->auxdb, kbuf + 1, ksiz - 1, tbuf, wp - tbuf, VL_DOVER)) err = TRUE;
1601
len = sprintf(numbuf, "%d", (int)((wp - tbuf) / (sizeof(int) * 2)));
1602
if(!vlput(db->xfmdb, kbuf + 1, ksiz - 1, numbuf, len, VL_DOVER)) err = TRUE;
1603
} else {
1604
if(!vlout(db->auxdb, kbuf + 1, ksiz - 1)) err = TRUE;
1605
if(!vlout(db->xfmdb, kbuf + 1, ksiz - 1) && dpecode != DP_ENOITEM) err = TRUE;
1606
}
958
1607
free(tbuf);
959
1608
}
1609
} else {
1610
sum = 0;
1611
for(j = 0; j < dnum; j++){
1612
if((vbuf = est_idx_get_one(db->idxdb, j, kbuf, ksiz, &tsiz)) != NULL){
1613
CB_DATUMOPEN(nval);
1614
rp = vbuf;
1615
ep = vbuf + tsiz;
1616
lid = 0;
1617
cid = 0;
1618
while(rp < ep){
1619
EST_READ_VNUMBUF(rp, vnum, vstep);
1620
cid += vnum + 1;
1621
rp += vstep;
1622
pv = rp;
1623
switch(db->smode){
1624
case ESTDFSCVOID:
1625
break;
1626
default:
1627
rp++;
1628
break;
1629
case ESTDFSCINT:
1630
case ESTDFSCASIS:
1631
rp += sizeof(int);
1632
break;
1633
}
1634
while(*rp != 0x0){
1635
rp += 2;
1636
}
1637
rp++;
1638
if(!cbmapget(ids, (char *)&cid, sizeof(int), NULL)){
1639
EST_SET_VNUMBUF(vstep, numbuf, cid - lid - 1);
1640
CB_DATUMCAT(nval, numbuf, vstep);
1641
CB_DATUMCAT(nval, pv, rp - pv);
1642
lid = cid;
1643
}
1644
}
1645
if(!est_idx_put_one(db->idxdb, j, kbuf, ksiz, CB_DATUMPTR(nval), CB_DATUMSIZE(nval)))
1646
err = TRUE;
1647
sum += CB_DATUMSIZE(nval);
1648
CB_DATUMCLOSE(nval);
1649
}
1650
}
1651
if(sum < 1 && !vlout(db->fwmdb, kbuf, ksiz) && dpecode != DP_ENOITEM) err = TRUE;
960
1652
}
961
1653
cbmapout(db->outcc, kbuf, ksiz);
962
1654
if(i % ESTCCCBFREQ == 0) est_db_inform(db, "cleaning dispensable keys");
963
1655
if(max > 0 && db->intflag && i > 0 && i % ESTCCIRSLOT == 0) break;
964
1656
}
965
cblistclose(keys);
966
cbmapclose(ids);
967
if(!(max > 0 && db->intflag)){
1657
if(cbmaprnum(db->outcc) <= cbmaprnum(ids)){
968
1658
cbmapclose(db->outcc);
969
1659
db->outcc = cbmapopenex(ESTOUTCCBNUM);
970
1660
}
1661
CB_LISTCLOSE(keys);
1662
cbmapclose(ids);
971
1663
}
972
1664
cbmapclose(db->keycc);
973
1665
db->keycc = cbmapopenex(ESTKEYCCMNUM + 1);
976
1668
est_db_inform(db, "adding a new database file");
977
1669
est_idx_increment(db->idxdb);
978
1670
}
1671
if(max < 1 || max >= INT_MAX){
1672
if(!vlmemflush(db->auxdb)) err = TRUE;
1673
if(!est_idx_memflush(db->idxdb)) err = TRUE;
1674
}
979
1675
if(max > 0 && db->intflag) est_db_inform(db, "flushing interrupted");
1676
db->flsflag = FALSE;
980
1677
db->intflag = FALSE;
981
1678
if(err){
982
1679
db->ecode = ESTEDB;
989
1686
990
1687
/* Synchronize updating contents of a database. */
991
1688
int est_db_sync(ESTDB *db){
1689
ESTATTRIDX *attridx;
1690
const char *kbuf;
992
1691
int err;
993
1692
assert(db);
994
1693
if(!dpwritable(db->metadb)){
1003
1702
if(!est_idx_sync(db->idxdb)) err = TRUE;
1004
1703
est_db_inform(db, "synchronizing the database for forward matching");
1005
1704
if(!vlsync(db->fwmdb)) err = TRUE;
1006
est_db_inform(db, "synchronizing the database for attrutes");
1705
est_db_inform(db, "synchronizing the database for attributes");
1007
1706
if(!crsync(db->attrdb)) err = TRUE;
1008
1707
est_db_inform(db, "synchronizing the database for texts");
1009
1708
if(!crsync(db->textdb)) err = TRUE;
1011
1710
if(!crsync(db->kwddb)) err = TRUE;
1012
1711
est_db_inform(db, "synchronizing the database for document list");
1013
1712
if(!vlsync(db->listdb)) err = TRUE;
1713
if(cbmaprnum(db->aidxs) > 0){
1714
est_db_inform(db, "synchronizing the databases for attribute narrowing");
1715
cbmapiterinit(db->aidxs);
1716
while((kbuf = cbmapiternext(db->aidxs, NULL)) != NULL){
1717
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
1718
switch(attridx->type){
1719
case ESTIDXATTRSTR:
1720
case ESTIDXATTRNUM:
1721
if(!vlsync(attridx->db)) err = TRUE;
1722
break;
1723
default:
1724
if(!dpsync(attridx->db)) err = TRUE;
1725
break;
1726
}
1727
}
1728
}
1014
1729
if(err){
1015
1730
db->ecode = ESTEDB;
1016
1731
db->fatal = TRUE;
1024
1739
CBMAP *dmap;
1025
1740
CBLIST *words;
1026
1741
CBDATUM *nval;
1027
const char *word, *rp, *pv;
1028
char *kbuf, *vbuf;
1029
int i, err, id, ksiz, vsiz, wsiz;
1742
ESTATTRIDX *attridx;
1743
const char *word, *rp, *pv, *ep;
1744
char *kbuf, *vbuf, *wp, numbuf[ESTNUMBUFSIZ];
1745
int i, err, id, ksiz, vsiz, wsiz, len, vstep;
1030
1746
assert(db);
1031
1747
if(!dpwritable(db->metadb)){
1032
1748
db->ecode = ESTEACCES;
1043
1759
free(vbuf);
1044
1760
vlcurnext(db->listdb);
1045
1761
}
1046
words = cblistopen();
1762
CB_LISTOPEN(words);
1047
1763
vlcurfirst(db->fwmdb);
1048
1764
while((kbuf = vlcurkey(db->fwmdb, &ksiz)) != NULL){
1049
cblistpushbuf(words, kbuf, ksiz);
1765
CB_LISTPUSHBUF(words, kbuf, ksiz);
1050
1766
vlcurnext(db->fwmdb);
1051
1767
}
1052
1768
for(i = 0; i < CB_LISTNUM(words); i++){
1053
1769
if(i % (ESTIDXDBLRM * 4) == 0) est_idx_set_current(db->idxdb);
1054
word = CB_LISTVAL2(words, i, &wsiz);
1055
if((vbuf = est_idx_get(db->idxdb, word, wsiz, &vsiz)) != NULL){
1056
nval = cbdatumopen("", 0);
1057
rp = vbuf;
1058
while(rp < vbuf + vsiz){
1059
pv = rp;
1060
rp += 5;
1061
while(*rp != 0x0){
1062
rp += 2;
1063
}
1770
word = CB_LISTVAL2(words, i, wsiz);
1771
vbuf = est_idx_scan(db->idxdb, word, wsiz, &vsiz, db->smode);
1772
CB_DATUMOPEN(nval);
1773
rp = vbuf;
1774
ep = vbuf + vsiz;
1775
while(rp < ep){
1776
pv = rp;
1777
EST_READ_VNUMBUF(rp, id, vstep);
1778
rp += vstep;
1779
switch(db->smode){
1780
case ESTDFSCVOID:
1781
break;
1782
default:
1064
1783
rp++;
1065
if(cbmapget(dmap, pv, sizeof(int), NULL)) cbdatumcat(nval, pv, rp - pv);
1066
}
1067
if(!est_idx_out(db->idxdb, word, wsiz)) err = TRUE;
1068
if(CB_DATUMSIZE(nval) > 0){
1069
if(!est_idx_add(db->idxdb, word, wsiz, CB_DATUMPTR(nval), CB_DATUMSIZE(nval)))
1070
err = TRUE;
1071
} else {
1072
if(!vlout(db->fwmdb, word, wsiz)) err = TRUE;
1073
}
1074
cbdatumclose(nval);
1075
free(vbuf);
1784
break;
1785
case ESTDFSCINT:
1786
case ESTDFSCASIS:
1787
rp += sizeof(int);
1788
break;
1789
}
1790
while(*rp != 0x00){
1791
rp += 2;
1792
}
1793
rp++;
1794
if(cbmapget(dmap, (char *)&id, sizeof(int), NULL)) CB_DATUMCAT(nval, pv, rp - pv);
1795
}
1796
if(!est_idx_out(db->idxdb, word, wsiz)) err = TRUE;
1797
if(CB_DATUMSIZE(nval) > 0){
1798
if(!est_idx_add(db->idxdb, word, wsiz, CB_DATUMPTR(nval), CB_DATUMSIZE(nval), db->smode))
1799
err = TRUE;
1076
1800
} else {
1077
err = TRUE;
1801
if(!vlout(db->fwmdb, word, wsiz)) err = TRUE;
1078
1802
}
1803
CB_DATUMCLOSE(nval);
1804
free(vbuf);
1079
1805
free(kbuf);
1080
1806
if(i % ESTCCCBFREQ == 0) est_db_inform(db, "cleaning dispensable keys");
1081
1807
}
1082
cblistclose(words);
1808
CB_LISTCLOSE(words);
1809
CB_LISTOPEN(words);
1810
vlcurfirst(db->auxdb);
1811
while((kbuf = vlcurkey(db->auxdb, &ksiz)) != NULL){
1812
CB_LISTPUSHBUF(words, kbuf, ksiz);
1813
vlcurnext(db->auxdb);
1814
}
1815
for(i = 0; i < CB_LISTNUM(words); i++){
1816
word = CB_LISTVAL2(words, i, wsiz);
1817
if(!(vbuf = vlget(db->auxdb, word, wsiz, &vsiz))) continue;
1818
rp = vbuf;
1819
wp = vbuf;
1820
ep = vbuf + vsiz;
1821
while(rp < ep){
1822
if(cbmapget(dmap, rp, sizeof(int), NULL)){
1823
memmove(wp, rp, sizeof(int) * 2);
1824
wp += sizeof(int) * 2;
1825
}
1826
rp += sizeof(int) * 2;
1827
}
1828
if(wp > vbuf){
1829
if(!vlput(db->auxdb, word, wsiz, vbuf, wp - vbuf, VL_DOVER)) err = TRUE;
1830
len = sprintf(numbuf, "%d", (int)((wp - vbuf) / (sizeof(int) * 2)));
1831
if(!vlput(db->xfmdb, word, wsiz, numbuf, len, VL_DOVER)) err = TRUE;
1832
} else {
1833
if(!vlout(db->auxdb, word, wsiz)) err = TRUE;
1834
if(!vlout(db->xfmdb, word, wsiz) && dpecode != DP_ENOITEM) err = TRUE;
1835
}
1836
free(vbuf);
1837
if(i % ESTCCCBFREQ == 0) est_db_inform(db, "cleaning dispensable auxiliary keys");
1838
}
1839
CB_LISTCLOSE(words);
1083
1840
cbmapclose(dmap);
1084
1841
}
1085
1842
if(!(options & ESTOPTNODBOPT)){
1087
1844
if(!est_idx_optimize(db->idxdb)) err = TRUE;
1088
1845
est_db_inform(db, "optimizing the database for forward matching");
1089
1846
if(!vloptimize(db->fwmdb)) err = TRUE;
1090
est_db_inform(db, "optimizing the database for attrutes");
1847
est_db_inform(db, "optimizing the auxiliary index");
1848
if(!vloptimize(db->auxdb)) err = TRUE;
1849
est_db_inform(db, "optimizing the database for auxiliary forward matching");
1850
if(!vloptimize(db->xfmdb)) err = TRUE;
1851
est_db_inform(db, "optimizing the database for attributes");
1091
1852
if(!croptimize(db->attrdb, -1)) err = TRUE;
1092
1853
est_db_inform(db, "optimizing the database for texts");
1093
1854
if(!croptimize(db->textdb, -1)) err = TRUE;
1095
1856
if(!croptimize(db->kwddb, -1)) err = TRUE;
1096
1857
est_db_inform(db, "optimizing the database for document list");
1097
1858
if(!vloptimize(db->listdb)) err = TRUE;
1859
if(cbmaprnum(db->aidxs) > 0){
1860
est_db_inform(db, "optimizing the databases for attribute narrowing");
1861
cbmapiterinit(db->aidxs);
1862
while((rp = cbmapiternext(db->aidxs, NULL)) != NULL){
1863
attridx = (ESTATTRIDX *)cbmapiterval(rp, NULL);
1864
switch(attridx->type){
1865
case ESTIDXATTRSTR:
1866
case ESTIDXATTRNUM:
1867
if(!vloptimize(attridx->db)) err = TRUE;
1868
break;
1869
default:
1870
if(!dpoptimize(attridx->db, -1)) err = TRUE;
1871
break;
1872
}
1873
}
1874
}
1098
1875
}
1099
1876
cbmapclose(db->rescc);
1100
1877
db->rescc = cbmapopenex(db->rcmnum * 2 + 1);
1106
1883
}
1107
1884
1108
1885
1886
/* Merge another database. */
1887
int est_db_merge(ESTDB *db, const char *name, int options){
1888
ESTDB *tgdb;
1889
ESTATTRIDX *attridx;
1890
CBMAP *idmap, *seqmap, *attrs;
1891
CBLIST *words;
1892
CBDATUM *rbuf;
1893
const char *kbuf, *vbuf, *rp, *ep, *sp;
1894
char *tbuf, numbuf[ESTNUMBUFSIZ];
1895
int i, j, ecode, err, ksiz, vsiz, tsiz, oid, nid, len, vstep, anum, *ary;
1896
assert(db && name);
1897
if(!dpwritable(db->metadb)){
1898
db->ecode = ESTEACCES;
1899
return FALSE;
1900
}
1901
est_db_inform(db, "opening the target database");
1902
if(!(tgdb = est_db_open(name, ESTDBREADER, &ecode))){
1903
db->ecode = ecode;
1904
return FALSE;
1905
}
1906
if(dpgetflags(db->metadb) != dpgetflags(tgdb->metadb)){
1907
est_db_close(tgdb, &ecode);
1908
db->ecode = ESTEMISC;
1909
return FALSE;
1910
}
1911
err = FALSE;
1912
idmap = cbmapopenex(est_db_doc_num(tgdb) + 1);
1913
vlcurfirst(tgdb->listdb);
1914
for(i = 0; (kbuf = vlcurkeycache(tgdb->listdb, &ksiz)) != NULL; i++){
1915
if((vbuf = vlgetcache(db->listdb, kbuf, ksiz, NULL)) != NULL &&
1916
!est_db_out_doc(db, atoi(vbuf), options & ESTMGCLEAN ? ESTODCLEAN : 0)) err = TRUE;
1917
oid = atoi(vlcurvalcache(tgdb->listdb, NULL));
1918
db->dseq++;
1919
db->dnum++;
1920
cbmapput(idmap, (char *)&oid, sizeof(int), (char *)&(db->dseq), sizeof(int), FALSE);
1921
vlcurnext(tgdb->listdb);
1922
if(i % (ESTCCCBFREQ / 10) == 0) est_db_inform(db, "calculating ID mapping");
1923
}
1924
if(!est_db_flush(db, -1)){
1925
cbmapclose(idmap);
1926
est_db_close(tgdb, &ecode);
1927
return FALSE;
1928
}
1929
cbmapiterinit(idmap);
1930
for(i = 0; (kbuf = cbmapiternext(idmap, &ksiz)) != NULL; i++){
1931
CB_MAPITERVAL(vbuf, kbuf, vsiz);
1932
oid = *(int *)kbuf;
1933
nid = *(int *)vbuf;
1934
if((tbuf = est_crget(tgdb->attrdb, tgdb->zmode, oid, &tsiz)) != NULL){
1935
attrs = cbmapload(tbuf, tsiz);
1936
len = sprintf(numbuf, "%d", nid);
1937
cbmapput(attrs, ESTDATTRID, -1, numbuf, len, TRUE);
1938
free(tbuf);
1939
tbuf = cbmapdump(attrs, &tsiz);
1940
if((vbuf = cbmapget(attrs, ESTDATTRURI, -1, &vsiz)) != NULL){
1941
if(!vlput(db->listdb, vbuf, vsiz, numbuf, len, VL_DKEEP)){
1942
db->ecode = ESTEDB;
1943
db->fatal = TRUE;
1944
err = TRUE;
1945
}
1946
} else {
1947
db->ecode = ESTEDB;
1948
db->fatal = TRUE;
1949
err = TRUE;
1950
}
1951
if(!est_crput(db->attrdb, db->zmode, nid, tbuf, tsiz, CR_DKEEP)){
1952
db->ecode = ESTEDB;
1953
db->fatal = TRUE;
1954
err = TRUE;
1955
}
1956
if(cbmaprnum(db->aidxs) > 0){
1957
cbmapiterinit(db->aidxs);
1958
while((kbuf = cbmapiternext(db->aidxs, &ksiz)) != NULL){
1959
if(!(vbuf = cbmapget(attrs, kbuf, ksiz, &vsiz))) continue;
1960
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
1961
switch(attridx->type){
1962
case ESTIDXATTRSTR:
1963
case ESTIDXATTRNUM:
1964
if(!est_aidx_attr_put(attridx->db, nid, vbuf, vsiz)){
1965
db->ecode = ESTEDB;
1966
db->fatal = TRUE;
1967
err = TRUE;
1968
}
1969
break;
1970
default:
1971
if(!est_aidx_seq_put(attridx->db, nid, vbuf, vsiz)){
1972
db->ecode = ESTEDB;
1973
db->fatal = TRUE;
1974
err = TRUE;
1975
}
1976
break;
1977
}
1978
}
1979
}
1980
cbmapclose(attrs);
1981
free(tbuf);
1982
} else {
1983
db->ecode = ESTEDB;
1984
err = TRUE;
1985
}
1986
if((tbuf = est_crget(tgdb->textdb, tgdb->zmode, oid, &tsiz)) != NULL){
1987
if(!est_crput(db->textdb, db->zmode, nid, tbuf, tsiz, CR_DKEEP)){
1988
db->ecode = ESTEDB;
1989
db->fatal = TRUE;
1990
err = TRUE;
1991
}
1992
free(tbuf);
1993
} else {
1994
db->ecode = ESTEDB;
1995
err = TRUE;
1996
}
1997
if((tbuf = est_crget(tgdb->kwddb, tgdb->zmode, oid, &tsiz)) != NULL){
1998
if(!est_crput(db->kwddb, db->zmode, nid, tbuf, tsiz, CR_DKEEP)){
1999
db->ecode = ESTEDB;
2000
db->fatal = TRUE;
2001
err = TRUE;
2002
}
2003
free(tbuf);
2004
} else if(dpecode != DP_ENOITEM){
2005
db->ecode = ESTEDB;
2006
db->fatal = TRUE;
2007
err = TRUE;
2008
}
2009
if(i % (ESTCCCBFREQ / 10) == 0) est_db_inform(db, "importing documents");
2010
}
2011
CB_LISTOPEN(words);
2012
vlcurfirst(tgdb->fwmdb);
2013
while((kbuf = vlcurkeycache(tgdb->fwmdb, &ksiz)) != NULL){
2014
CB_LISTPUSH(words, kbuf, ksiz);
2015
vlcurnext(tgdb->fwmdb);
2016
}
2017
for(i = 0; i < CB_LISTNUM(words); i++){
2018
kbuf = CB_LISTVAL2(words, i, ksiz);
2019
seqmap = cbmapopenex(tsiz / sizeof(int) + 1);
2020
tbuf = est_idx_scan(tgdb->idxdb, kbuf, ksiz, &tsiz, tgdb->smode);
2021
rp = tbuf;
2022
ep = tbuf + tsiz;
2023
while(rp < ep){
2024
EST_READ_VNUMBUF(rp, oid, vstep);
2025
rp += vstep;
2026
vbuf = cbmapget(idmap, (char *)&oid, sizeof(int), NULL);
2027
nid = vbuf ? *(int *)vbuf : -1;
2028
sp = rp;
2029
switch(tgdb->smode){
2030
case ESTDFSCVOID:
2031
break;
2032
default:
2033
rp++;
2034
break;
2035
case ESTDFSCINT:
2036
case ESTDFSCASIS:
2037
rp += sizeof(int);
2038
break;
2039
}
2040
while(*rp != 0x00){
2041
rp += 2;
2042
}
2043
rp++;
2044
if(nid > 0) cbmapputcat(seqmap, (char *)&nid, sizeof(int), sp, rp - sp);
2045
}
2046
anum = cbmaprnum(seqmap);
2047
CB_MALLOC(ary, anum * sizeof(int) + 1);
2048
cbmapiterinit(seqmap);
2049
for(j = 0; (rp = cbmapiternext(seqmap, NULL)) != NULL; j++){
2050
ary[j] = *(int *)rp;
2051
}
2052
qsort(ary, anum, sizeof(int), est_int_compare);
2053
CB_DATUMOPEN(rbuf);
2054
for(j = 0; j < anum; j++){
2055
EST_SET_VNUMBUF(vstep, numbuf, ary[j]);
2056
CB_DATUMCAT(rbuf, numbuf, vstep);
2057
vbuf = cbmapget(seqmap, (char *)(ary + j), sizeof(int), &vsiz);
2058
CB_DATUMCAT(rbuf, vbuf, vsiz);
2059
}
2060
if(!est_idx_add(db->idxdb, kbuf, ksiz, CB_DATUMPTR(rbuf), CB_DATUMSIZE(rbuf), db->smode)){
2061
db->ecode = ESTEDB;
2062
db->fatal = TRUE;
2063
err = TRUE;
2064
}
2065
CB_DATUMCLOSE(rbuf);
2066
free(ary);
2067
cbmapclose(seqmap);
2068
free(tbuf);
2069
vlput(db->fwmdb, kbuf, ksiz, "", 0, VL_DKEEP);
2070
if(i % ESTCCCBFREQ == 0){
2071
est_db_inform(db, "importing words");
2072
if(est_idx_size_current(db->idxdb) >= ESTIDXDBMAX){
2073
est_db_inform(db, "adding a new database file");
2074
est_idx_increment(db->idxdb);
2075
}
2076
}
2077
}
2078
CB_LISTCLOSE(words);
2079
CB_LISTOPEN(words);
2080
vlcurfirst(tgdb->auxdb);
2081
while((kbuf = vlcurkeycache(tgdb->auxdb, &ksiz)) != NULL){
2082
CB_LISTPUSH(words, kbuf, ksiz);
2083
vlcurnext(tgdb->auxdb);
2084
}
2085
for(i = 0; i < CB_LISTNUM(words); i++){
2086
kbuf = CB_LISTVAL2(words, i, ksiz);
2087
vbuf = vlgetcache(tgdb->auxdb, kbuf, ksiz, &vsiz);
2088
CB_DATUMOPEN(rbuf);
2089
rp = vbuf;
2090
ep = vbuf + vsiz;
2091
while(rp < ep){
2092
oid = *(int *)rp;
2093
vbuf = cbmapget(idmap, rp, sizeof(int), NULL);
2094
nid = vbuf ? *(int *)vbuf : -1;
2095
if(nid > 0){
2096
CB_DATUMCAT(rbuf, (char *)&nid, sizeof(int));
2097
CB_DATUMCAT(rbuf, rp + sizeof(int), sizeof(int));
2098
}
2099
rp += sizeof(int) * 2;
2100
}
2101
if(!vlput(db->auxdb, kbuf, ksiz, CB_DATUMPTR(rbuf), CB_DATUMSIZE(rbuf), VL_DCAT)){
2102
db->ecode = ESTEDB;
2103
db->fatal = TRUE;
2104
err = TRUE;
2105
}
2106
CB_DATUMCLOSE(rbuf);
2107
anum = 0;
2108
if((vbuf = vlgetcache(tgdb->xfmdb, kbuf, ksiz, NULL)) != NULL) anum += atoi(vbuf);
2109
len = sprintf(numbuf, "%d", anum);
2110
vlput(db->xfmdb, kbuf, ksiz, numbuf, len, VL_DOVER);
2111
if(i % ESTCCCBFREQ == 0) est_db_inform(db, "importing auxiliary words");
2112
}
2113
CB_LISTCLOSE(words);
2114
cbmapclose(idmap);
2115
est_db_inform(db, "closing the target database");
2116
if(!est_db_close(tgdb, &ecode)){
2117
db->ecode = ecode;
2118
return FALSE;
2119
}
2120
if(!est_db_flush(db, -1)) err = TRUE;
2121
return err ? FALSE : TRUE;
2122
}
2123
2124
1109
2125
/* Add a document to a database. */
1110
2126
int est_db_put_doc(ESTDB *db, ESTDOC *doc, int options){
1111
2127
CBMAP *ocmap, *fmap, *qmap;
1112
2128
CBLIST *words;
1113
2129
CBDATUM *ocbuf;
2130
ESTATTRIDX *attridx;
1114
2131
md5_state_t ms;
1115
2132
const char *uri, *ndig, *text, *word, *fnext, *snext, *kbuf, *vbuf;
1116
2133
unsigned char junc[2], c;
1117
char dobuf[32], dsbuf[64], *wp, *odig, wbuf[ESTWORDMAXLEN+3], *sbuf, *zbuf, nbuf[ESTNUMBUFSIZ];
1118
int i, j, id, err, wnum, wsiz, fnsiz, snsiz, *np, num, ksiz, vsiz, ssiz, zsiz;
1119
double tune;
2134
char dobuf[32], dsbuf[64], *wp, *odig, wbuf[ESTWORDMAXLEN+3], *sbuf, nbuf[ESTNUMBUFSIZ];
2135
int i, j, id, err, wnum, wsiz, fnsiz, snsiz, *np, score, num, ksiz, vsiz, ssiz;
2136
double tune, weight;
1120
2137
assert(db && doc);
1121
2138
if(!dpwritable(db->metadb)){
1122
2139
db->ecode = ESTEACCES;
1126
2143
db->ecode = ESTEINVAL;
1127
2144
return FALSE;
1128
2145
}
1129
if(!doc->dtexts) doc->dtexts = cblistopen();
2146
if(!doc->dtexts) CB_LISTOPEN(doc->dtexts);
1130
2147
if(!(ndig = cbmapget(doc->attrs, ESTDATTRDIGEST, -1, NULL))){
1131
2148
md5_init(&ms);
1132
for(i = 0; i < cblistnum(doc->dtexts); i++){
1133
vbuf = cblistval(doc->dtexts, i, &vsiz);
2149
for(i = 0; i < CB_LISTNUM(doc->dtexts); i++){
2150
vbuf = CB_LISTVAL2(doc->dtexts, i, vsiz);
1134
2151
md5_append(&ms, (md5_byte_t *)vbuf, vsiz);
1135
2152
md5_append(&ms, (md5_byte_t *)"\n", 1);
1136
2153
}
1171
2188
if(i < 0){
1172
2189
if(!(text = cbmapget(doc->attrs, "", 0, NULL))) continue;
1173
2190
} else {
1174
text = CB_LISTVAL(doc->dtexts, i, NULL);
2191
text = CB_LISTVAL(doc->dtexts, i);
1175
2192
}
1176
words = cblistopen();
2193
CB_LISTOPEN(words);
1177
2194
switch(db->amode){
1178
case ESTAMPERFNG:
2195
case ESTDFPERFNG:
1179
2196
est_break_text_perfng(text, words, FALSE, TRUE);
1180
2197
break;
2198
case ESTDFCHRCAT:
2199
est_break_text_chrcat(text, words, FALSE);
2200
break;
1181
2201
default:
1182
2202
est_break_text(text, words, FALSE, TRUE);
1183
2203
break;
1184
2204
}
1185
2205
wnum += CB_LISTNUM(words);
1186
2206
for(j = 0; j < CB_LISTNUM(words); j++){
1187
word = CB_LISTVAL2(words, j, &wsiz);
2207
word = CB_LISTVAL2(words, j, wsiz);
1188
2208
if(wsiz > ESTWORDMAXLEN) continue;
1189
2209
fnext = cblistval(words, j + 1, &fnsiz);
1190
2210
snext = cblistval(words, j + 2, &snsiz);
1198
2218
num += ESTOCPOINT;
1199
2219
cbmapput(fmap, word, wsiz, (char *)&num, sizeof(int), TRUE);
1200
2220
if(cbmapput(qmap, wbuf, wsiz + 3, "", 0, FALSE))
1201
cbmapputcat(ocmap, word, wsiz, (char *)junc, 2);
2221
cbmapputcat(ocmap, word, wsiz, (char *)junc, fnext ? 2 : 0);
1202
2222
}
1203
cblistclose(words);
1204
}
1205
tune = log(wnum + 32);
1206
tune = (tune * tune) / 12.0;
2223
CB_LISTCLOSE(words);
2224
}
2225
score = (vbuf = cbmapget(doc->attrs, "\t", 1, NULL)) ? atoi(vbuf) : -1;
2226
weight = 1.0;
2227
if(score < 0 && (options & ESTPDWEIGHT) &&
2228
(vbuf = cbmapget(doc->attrs, ESTDATTRWEIGHT, -1, NULL)) != NULL){
2229
weight = strtod(vbuf, NULL);
2230
weight = weight >= 0.01 ? weight : 0.01;
2231
}
2232
tune = sqrt(wnum + 128) / 16.0 / weight;
1207
2233
cbmapiterinit(ocmap);
1208
2234
while((kbuf = cbmapiternext(ocmap, &ksiz)) != NULL){
1209
vbuf = cbmapget(ocmap, kbuf, ksiz, &vsiz);
1210
ocbuf = cbdatumopen("", 0);
1211
cbdatumcat(ocbuf, (char *)&(doc->id), sizeof(int));
1212
num = *(int *)cbmapget(fmap, kbuf, ksiz, NULL) / tune;
1213
if(num >= 0x80) num += (0x80 - num) * 0.75;
1214
if(num >= 0xc0) num += (0xc0 - num) * 0.75;
1215
c = num < 0xff ? num : 0xff;
1216
cbdatumcat(ocbuf, (char *)&c, 1);
1217
cbdatumcat(ocbuf, vbuf, vsiz);
2235
CB_MAPITERVAL(vbuf, kbuf, vsiz);
2236
if(vsiz > 2) qsort((void *)vbuf, vsiz / 2, 2, est_short_compare);
2237
CB_DATUMOPEN(ocbuf);
2238
EST_SET_VNUMBUF(wsiz, wbuf, doc->id);
2239
CB_DATUMCAT(ocbuf, wbuf, wsiz);
2240
switch(db->smode){
2241
case ESTDFSCVOID:
2242
break;
2243
default:
2244
num = score < 0 ? *(int *)cbmapget(fmap, kbuf, ksiz, NULL) / tune : score;
2245
if(num >= 0x80) num += (0x80 - num) * 0.75;
2246
if(num >= 0xc0) num += (0xc0 - num) * 0.75;
2247
c = num < 0xff ? num : 0xff;
2248
CB_DATUMCAT(ocbuf, (char *)&c, 1);
2249
break;
2250
case ESTDFSCINT:
2251
case ESTDFSCASIS:
2252
num = score < 0 ? *(int *)cbmapget(fmap, kbuf, ksiz, NULL) * 10 / tune : score;
2253
CB_DATUMCAT(ocbuf, (char *)&num, sizeof(int));
2254
break;
2255
}
2256
CB_DATUMCAT(ocbuf, vbuf, vsiz);
1218
2257
c = 0x00;
1219
cbdatumcat(ocbuf, (char *)&c, 1);
2258
CB_DATUMCAT(ocbuf, (char *)&c, 1);
1220
2259
cbmapputcat(db->idxcc, kbuf, ksiz, CB_DATUMPTR(ocbuf), CB_DATUMSIZE(ocbuf));
1221
2260
db->icsiz += CB_DATUMSIZE(ocbuf);
1222
cbdatumclose(ocbuf);
2261
CB_DATUMCLOSE(ocbuf);
1223
2262
}
1224
2263
cbmapclose(qmap);
1225
2264
cbmapclose(fmap);
1226
2265
cbmapclose(ocmap);
1227
2266
err = FALSE;
1228
2267
sbuf = cbmapdump(doc->attrs, &ssiz);
1229
if(!crput(db->attrdb, (char *)&(doc->id), sizeof(int), sbuf, ssiz, CR_DKEEP)){
2268
if(!est_crput(db->attrdb, db->zmode, doc->id, sbuf, ssiz, CR_DKEEP)){
1230
2269
db->ecode = ESTEDB;
1231
2270
db->fatal = TRUE;
1232
2271
err = TRUE;
1233
2272
}
1234
2273
free(sbuf);
1235
2274
sbuf = cblistdump(doc->dtexts, &ssiz);
1236
if(!(zbuf = est_deflate(sbuf, ssiz, &zsiz))){
1237
CB_MALLOC(zbuf, 1);
1238
zsiz = 0;
1239
db->ecode = ESTEMISC;
1240
db->fatal = TRUE;
1241
err = TRUE;
1242
}
1243
if(!crput(db->textdb, (char *)&(doc->id), sizeof(int), zbuf, zsiz, CR_DKEEP)){
2275
if(!est_crput(db->textdb, db->zmode, doc->id, sbuf, ssiz, CR_DKEEP)){
1244
2276
db->ecode = ESTEDB;
1245
2277
db->fatal = TRUE;
1246
2278
err = TRUE;
1247
2279
}
1248
free(zbuf);
1249
2280
free(sbuf);
2281
if(doc->kwords && !est_db_put_keywords(db, doc->id, doc->kwords, weight)) err = TRUE;
1250
2282
sprintf(nbuf, "%d", doc->id);
1251
2283
if(!vlput(db->listdb, uri, -1, nbuf, -1, VL_DKEEP)){
1252
2284
db->ecode = ESTEDB;
1253
2285
db->fatal = TRUE;
1254
2286
err = TRUE;
1255
2287
}
2288
if(cbmaprnum(db->aidxs) > 0){
2289
cbmapiterinit(db->aidxs);
2290
while((kbuf = cbmapiternext(db->aidxs, &ksiz)) != NULL){
2291
if(!(vbuf = cbmapget(doc->attrs, kbuf, ksiz, &vsiz))) continue;
2292
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
2293
switch(attridx->type){
2294
case ESTIDXATTRSTR:
2295
case ESTIDXATTRNUM:
2296
if(!est_aidx_attr_put(attridx->db, doc->id, vbuf, vsiz)){
2297
db->ecode = ESTEDB;
2298
db->fatal = TRUE;
2299
err = TRUE;
2300
}
2301
break;
2302
default:
2303
if(!est_aidx_seq_put(attridx->db, doc->id, vbuf, vsiz)){
2304
db->ecode = ESTEDB;
2305
db->fatal = TRUE;
2306
err = TRUE;
2307
}
2308
break;
2309
}
2310
}
2311
}
1256
2312
db->dnum++;
1257
2313
if(est_db_used_cache_size(db) > db->icmax && !est_db_flush(db, INT_MAX)) err = TRUE;
1258
2314
return err ? FALSE : TRUE;
1263
2319
int est_db_out_doc(ESTDB *db, int id, int options){
1264
2320
ESTDOC *doc;
1265
2321
CBLIST *words;
1266
const char *uri, *text, *word;
2322
ESTATTRIDX *attridx;
2323
const char *uri, *kbuf, *vbuf, *text, *word;
1267
2324
char numbuf[ESTNUMBUFSIZ];
1268
int i, j, len, wsiz;
2325
int i, j, ksiz, vsiz, len, wsiz;
1269
2326
assert(db && id > 0);
1270
2327
if(!dpwritable(db->metadb)){
1271
2328
db->ecode = ESTEACCES;
1272
2329
return FALSE;
1273
2330
}
1274
if(!(doc = est_db_get_doc(db, id, 0))) return FALSE;
2331
if(id >= ESTPDOCIDMIN){
2332
db->ecode = ESTEINVAL;
2333
return FALSE;
2334
}
2335
if(!(doc = est_db_get_doc(db, id, ESTGDNOKWD))) return FALSE;
1275
2336
if(!doc->attrs || !(uri = cbmapget(doc->attrs, ESTDATTRURI, -1, NULL))){
1276
2337
est_doc_delete(doc);
1277
2338
db->ecode = ESTEDB;
1278
2339
db->fatal = TRUE;
1279
2340
return FALSE;
1280
2341
}
1281
if(!crout(db->attrdb, (char *)&id, sizeof(int)) ||
1282
!crout(db->textdb, (char *)&id, sizeof(int)) ||
1283
(!crout(db->kwddb, (char *)&id, sizeof(int)) && dpecode != DP_ENOITEM) ||
1284
!vlout(db->listdb, uri, -1)){
2342
if(!est_crout(db->attrdb, id) || !est_crout(db->textdb, id) || !vlout(db->listdb, uri, -1)){
1285
2343
est_doc_delete(doc);
1286
2344
db->ecode = ESTEDB;
1287
2345
db->fatal = TRUE;
1289
2347
}
1290
2348
cbmapout(db->attrcc, (char *)&id, sizeof(int));
1291
2349
cbmapout(db->textcc, (char *)&id, sizeof(int));
1292
cbmapout(db->veccc, (char *)&id, sizeof(int));
1293
2350
if(db->spacc) cbmapout(db->spacc, (char *)&id, sizeof(int));
1294
if((options & ESTODCLEAN) && doc->dtexts){
2351
if(cbmaprnum(db->aidxs) > 0){
2352
cbmapiterinit(db->aidxs);
2353
while((kbuf = cbmapiternext(db->aidxs, &ksiz)) != NULL){
2354
if(!(vbuf = cbmapget(doc->attrs, kbuf, ksiz, &vsiz))) continue;
2355
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
2356
switch(attridx->type){
2357
case ESTIDXATTRSTR:
2358
case ESTIDXATTRNUM:
2359
if(!est_aidx_attr_out(attridx->db, doc->id, vbuf, vsiz)){
2360
est_doc_delete(doc);
2361
db->ecode = ESTEDB;
2362
db->fatal = TRUE;
2363
return FALSE;
2364
}
2365
break;
2366
default:
2367
if(!est_aidx_seq_out(attridx->db, doc->id)){
2368
est_doc_delete(doc);
2369
db->ecode = ESTEDB;
2370
db->fatal = TRUE;
2371
return FALSE;
2372
}
2373
break;
2374
}
2375
}
2376
}
2377
if(options & ESTODCLEAN){
1295
2378
len = sprintf(numbuf, "\t%d", doc->id);
1296
2379
cbmapput(db->outcc, numbuf, len, "", 0, FALSE);
1297
2380
for(i = -1; i < CB_LISTNUM(doc->dtexts); i++){
1298
2381
if(i < 0){
1299
2382
if(!(text = cbmapget(doc->attrs, "", 0, NULL))) continue;
1300
2383
} else {
1301
text = CB_LISTVAL(doc->dtexts, i, NULL);
2384
text = CB_LISTVAL(doc->dtexts, i);
1302
2385
}
1303
words = cblistopen();
2386
CB_LISTOPEN(words);
1304
2387
switch(db->amode){
1305
case ESTAMPERFNG:
2388
case ESTDFPERFNG:
1306
2389
est_break_text_perfng(text, words, FALSE, TRUE);
1307
2390
break;
2391
case ESTDFCHRCAT:
2392
est_break_text_chrcat(text, words, FALSE);
2393
break;
1308
2394
default:
1309
2395
est_break_text(text, words, FALSE, TRUE);
1310
2396
break;
1311
2397
}
1312
2398
for(j = 0; j < CB_LISTNUM(words); j++){
1313
word = CB_LISTVAL2(words, j, &wsiz);
2399
word = CB_LISTVAL2(words, j, wsiz);
1314
2400
cbmapput(db->outcc, word, wsiz, "", 0, FALSE);
1315
2401
}
1316
cblistclose(words);
1317
}
2402
CB_LISTCLOSE(words);
2403
}
2404
if(!est_db_out_keywords(db, id) && db->ecode != ESTENOITEM){
2405
est_doc_delete(doc);
2406
db->ecode = ESTEDB;
2407
db->fatal = TRUE;
2408
return FALSE;
2409
}
2410
} else {
2411
if(!est_crout(db->kwddb, id) && dpecode != DP_ENOITEM){
2412
est_doc_delete(doc);
2413
db->ecode = ESTEDB;
2414
db->fatal = TRUE;
2415
return FALSE;
2416
}
2417
cbmapout(db->veccc, (char *)&id, sizeof(int));
1318
2418
}
1319
2419
est_doc_delete(doc);
1320
2420
if(!est_db_set_doc_entity(db, id, NULL, -1) && db->ecode != ESTENOITEM) return FALSE;
1325
2425
1326
2426
/* Edit attributes of a document object in a database. */
1327
2427
int est_db_edit_doc(ESTDB *db, ESTDOC *doc){
1328
const char *uri, *tmp;
2428
ESTDOC *odoc;
2429
ESTATTRIDX *attridx;
2430
const char *uri, *tmp, *kbuf, *vbuf;
1329
2431
char *ouri, numbuf[ESTNUMBUFSIZ], *text, *sbuf;
1330
int err, id, oid, ssiz;
2432
int err, id, oid, ksiz, vsiz, ssiz;
1331
2433
assert(db && doc);
1332
2434
if(!dpwritable(db->metadb)){
1333
2435
db->ecode = ESTEACCES;
1339
2441
if((tmp = cbmapget(doc->attrs, ESTDATTRID, -1, NULL)) != NULL) id = atoi(tmp);
1340
2442
if((tmp = cbmapget(doc->attrs, ESTDATTRURI, -1, NULL)) != NULL) uri = tmp;
1341
2443
}
1342
if(id < 1 || (doc->id > 0 && doc->id != id) || !uri || uri[0] == '\0'){
2444
if(id < 1 || id >= ESTPDOCIDMIN || (doc->id > 0 && doc->id != id) || !uri || uri[0] == '\0'){
1343
2445
db->ecode = ESTEINVAL;
1344
2446
return FALSE;
1345
2447
}
1357
2459
db->ecode = ESTEINVAL;
1358
2460
return FALSE;
1359
2461
}
2462
doc->id = id;
2463
if(cbmaprnum(db->aidxs) > 0 && (odoc = est_db_get_doc(db, id, ESTGDNOTEXT))){
2464
if(!odoc->attrs) odoc->attrs = cbmapopenex(ESTMINIBNUM);
2465
cbmapiterinit(db->aidxs);
2466
while((kbuf = cbmapiternext(db->aidxs, &ksiz)) != NULL){
2467
if(!(vbuf = cbmapget(odoc->attrs, kbuf, ksiz, &vsiz))) continue;
2468
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
2469
switch(attridx->type){
2470
case ESTIDXATTRSTR:
2471
case ESTIDXATTRNUM:
2472
if(!est_aidx_attr_out(attridx->db, id, vbuf, vsiz)){
2473
db->ecode = ESTEDB;
2474
db->fatal = TRUE;
2475
err = TRUE;
2476
}
2477
break;
2478
default:
2479
if(!est_aidx_seq_out(attridx->db, id)){
2480
db->ecode = ESTEDB;
2481
db->fatal = TRUE;
2482
err = TRUE;
2483
}
2484
break;
2485
}
2486
}
2487
cbmapiterinit(db->aidxs);
2488
while((kbuf = cbmapiternext(db->aidxs, &ksiz)) != NULL){
2489
if(!(vbuf = cbmapget(doc->attrs, kbuf, ksiz, &vsiz))) continue;
2490
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
2491
switch(attridx->type){
2492
case ESTIDXATTRSTR:
2493
case ESTIDXATTRNUM:
2494
if(!est_aidx_attr_put(attridx->db, id, vbuf, vsiz)){
2495
db->ecode = ESTEDB;
2496
db->fatal = TRUE;
2497
err = TRUE;
2498
}
2499
break;
2500
default:
2501
if(!est_aidx_seq_put(attridx->db, id, vbuf, vsiz)){
2502
db->ecode = ESTEDB;
2503
db->fatal = TRUE;
2504
err = TRUE;
2505
}
2506
break;
2507
}
2508
}
2509
est_doc_delete(odoc);
2510
}
1360
2511
if((text = est_db_get_doc_attr(db, id, "")) != NULL){
1361
2512
cbmapput(doc->attrs, "", 0, text, -1, TRUE);
1362
2513
free(text);
1363
2514
}
1364
2515
sbuf = cbmapdump(doc->attrs, &ssiz);
1365
if(!crput(db->attrdb, (char *)&id, sizeof(int), sbuf, ssiz, CR_DOVER)){
2516
if(!est_crput(db->attrdb, db->zmode, id, sbuf, ssiz, CR_DOVER)){
1366
2517
db->ecode = ESTEDB;
1367
2518
db->fatal = TRUE;
1368
2519
err = TRUE;
1378
2529
ESTDOC *est_db_get_doc(ESTDB *db, int id, int options){
1379
2530
ESTDOC *doc;
1380
2531
const char *cbuf;
1381
char *vbuf, *zbuf;
1382
int i, csiz, vsiz, zsiz, num;
2532
char *vbuf, numbuf[ESTNUMBUFSIZ];
2533
int i, csiz, vsiz, num;
1383
2534
assert(db && id > 0);
2535
if(id >= ESTPDOCIDMIN){
2536
if((num = id - ESTPDOCIDMIN) >= CB_LISTNUM(db->pdocs)){
2537
db->ecode = ESTENOITEM;
2538
return NULL;
2539
}
2540
if((vbuf = cbreadfile(CB_LISTVAL(db->pdocs, num), NULL)) != NULL){
2541
doc = est_doc_new_from_draft(vbuf);
2542
free(vbuf);
2543
} else {
2544
doc = est_doc_new();
2545
}
2546
doc->id = id;
2547
sprintf(numbuf, "%d", id);
2548
est_doc_add_attr(doc, ESTDATTRID, numbuf);
2549
if(!est_doc_attr(doc, ESTDATTRURI))
2550
est_doc_add_attr(doc, ESTDATTRURI, CB_LISTVAL(db->pdocs, num));
2551
return doc;
2552
}
1384
2553
cbuf = NULL;
1385
2554
if(options & ESTGDNOATTR){
1386
if(!crvsiz(db->attrdb, (char *)&id, sizeof(int))){
2555
if(crvsiz(db->attrdb, (char *)&id, sizeof(int)) == -1){
1387
2556
if(dpecode == DP_ENOITEM){
1388
2557
db->ecode = ESTENOITEM;
1389
2558
return NULL;
1397
2566
} else if((cbuf = cbmapget(db->attrcc, (char *)&id, sizeof(int), &csiz)) != NULL){
1398
2567
cbmapmove(db->attrcc, (char *)&id, sizeof(int), FALSE);
1399
2568
vbuf = NULL;
1400
} else if(!(vbuf = crget(db->attrdb, (char *)&id, sizeof(int), 0, -1, &vsiz))){
2569
} else if(!(vbuf = est_crget(db->attrdb, db->zmode, id, &vsiz))){
1401
2570
if(dpecode == DP_ENOITEM){
1402
2571
db->ecode = ESTENOITEM;
1403
2572
return NULL;
1413
2582
doc->attrs = cbmapload(cbuf, csiz);
1414
2583
} else if(vbuf){
1415
2584
doc->attrs = cbmapload(vbuf, vsiz);
1416
cbmapputvbuf(db->attrcc, (char *)&id, sizeof(int), vbuf, vsiz);
2585
if(db->acmnum > 0) cbmapput(db->attrcc, (char *)&id, sizeof(int), vbuf, vsiz, TRUE);
2586
free(vbuf);
1417
2587
if(cbmaprnum(db->attrcc) > db->acmnum){
1418
2588
num = cbmaprnum(db->attrcc) * 0.1 + 1;
1419
2589
cbmapiterinit(db->attrcc);
1429
2599
cbmapmove(db->textcc, (char *)&id, sizeof(int), FALSE);
1430
2600
doc->dtexts = cblistload(cbuf, csiz);
1431
2601
} else {
1432
if(!(zbuf = crget(db->textdb, (char *)&id, sizeof(int), 0, -1, &zsiz))){
1433
db->ecode = ESTEDB;
1434
db->fatal = TRUE;
1435
est_doc_delete(doc);
1436
return NULL;
1437
}
1438
if(!(vbuf = est_inflate(zbuf, zsiz, &vsiz))){
1439
db->ecode = ESTEDB;
1440
db->fatal = TRUE;
1441
free(zbuf);
2602
if(!(vbuf = est_crget(db->textdb, db->zmode, id, &vsiz))){
2603
db->ecode = ESTEDB;
2604
db->fatal = TRUE;
1442
2605
est_doc_delete(doc);
1443
2606
return NULL;
1444
2607
}
1445
2608
doc->dtexts = cblistload(vbuf, vsiz);
1446
cbmapputvbuf(db->textcc, (char *)&id, sizeof(int), vbuf, vsiz);
2609
if(db->tcmnum > 0) cbmapput(db->textcc, (char *)&id, sizeof(int), vbuf, vsiz, TRUE);
2610
free(vbuf);
1447
2611
if(cbmaprnum(db->textcc) > db->tcmnum){
1448
2612
num = cbmaprnum(db->textcc) * 0.1 + 1;
1449
2613
cbmapiterinit(db->textcc);
1451
2615
cbmapout(db->textcc, cbuf, sizeof(int));
1452
2616
}
1453
2617
}
1454
free(zbuf);
1455
2618
}
1456
2619
}
2620
if(!(options & ESTGDNOKWD)) doc->kwords = est_db_get_keywords(db, id);
1457
2621
return doc;
1458
2622
}
1459
2623
1460
2624
1461
2625
/* Retrieve the value of an attribute of a document in a database. */
1462
2626
char *est_db_get_doc_attr(ESTDB *db, int id, const char *name){
2627
ESTATTRIDX *attridx;
2628
ESTDOC *doc;
1463
2629
const char *cbuf;
1464
2630
char *mbuf, *vbuf;
1465
2631
int cb, csiz, msiz, vsiz;
1466
2632
assert(db && id > 0 && name);
2633
if(id >= ESTPDOCIDMIN){
2634
if(!(doc = est_db_get_doc(db, id, 0))){
2635
db->ecode = ESTENOITEM;
2636
return NULL;
2637
}
2638
if(!(cbuf = est_doc_attr(doc, name))){
2639
db->ecode = ESTENOITEM;
2640
est_doc_delete(doc);
2641
return NULL;
2642
}
2643
vbuf = cbmemdup(cbuf, -1);
2644
est_doc_delete(doc);
2645
return vbuf;
2646
}
1467
2647
cb = db->spacc && !strcmp(name, db->scname);
1468
2648
if(cb && (cbuf = cbmapget(db->spacc, (char *)&id, sizeof(int), &csiz)) != NULL){
1469
2649
cbmapmove(db->spacc, (char *)&id, sizeof(int), FALSE);
1470
2650
return cbmemdup(cbuf, csiz);
1471
2651
}
1472
if(!(mbuf = crget(db->attrdb, (char *)&id, sizeof(int), 0, -1, &msiz))){
2652
if((attridx = (ESTATTRIDX *)cbmapget(db->aidxs, name, -1, NULL)) != NULL &&
2653
attridx->type == ESTIDXATTRSEQ){
2654
if(!(vbuf = est_aidx_seq_get(attridx->db, id, &vsiz))){
2655
db->ecode = ESTENOITEM;
2656
return NULL;
2657
}
2658
if(cb) cbmapput(db->spacc, (char *)&id, sizeof(int), vbuf, vsiz, FALSE);
2659
return vbuf;
2660
}
2661
if(!(mbuf = est_crget(db->attrdb, db->zmode, id, &msiz))){
1473
2662
db->ecode = dpecode == DP_ENOITEM ? ESTENOITEM : ESTEDB;
1474
2663
return NULL;
1475
2664
}
1486
2675
1487
2676
/* Get the ID of a document spacified by URI. */
1488
2677
int est_db_uri_to_id(ESTDB *db, const char *uri){
1489
char *vbuf;
2678
const char *vbuf;
1490
2679
int id;
1491
2680
assert(db && uri);
1492
if(!(vbuf = vlget(db->listdb, uri, -1, NULL))){
2681
if(!(vbuf = vlgetcache(db->listdb, uri, -1, NULL))){
2682
if(CB_LISTNUM(db->pdocs) > 0 && (id = est_pidx_uri_to_id(db, uri)) > 0) return id;
1493
2683
db->ecode = ESTENOITEM;
1494
2684
return -1;
1495
2685
}
1496
id = atoi(vbuf);
1497
free(vbuf);
1498
return id;
2686
return atoi(vbuf);
1499
2687
}
1500
2688
1501
2689
1515
2703
1516
2704
/* Get the number of words in a database. */
1517
2705
int est_db_word_num(ESTDB *db){
2706
int wnum;
1518
2707
assert(db);
1519
return vlrnum(db->fwmdb);
2708
wnum = vlrnum(db->fwmdb);
2709
return wnum > 0 ? wnum : 0;
1520
2710
}
1521
2711
1522
2712
1523
2713
/* Get the size of a database. */
1524
2714
double est_db_size(ESTDB *db){
2715
ESTATTRIDX *attridx;
2716
const char *kbuf;
2717
double size;
1525
2718
assert(db);
1526
return (double)dpfsiz(db->metadb) + est_idx_size(db->idxdb) + (double)vlfsiz(db->fwmdb) +
1527
crfsizd(db->attrdb) + crfsizd(db->textdb) + crfsizd(db->kwddb) + (double)vlfsiz(db->listdb);
2719
size = (double)dpfsiz(db->metadb) + est_idx_size(db->idxdb) + vlfsiz(db->fwmdb) +
2720
vlfsiz(db->auxdb) + vlfsiz(db->xfmdb) + crfsizd(db->attrdb) + crfsizd(db->textdb) +
2721
crfsizd(db->kwddb) + vlfsiz(db->listdb);
2722
if(cbmaprnum(db->aidxs) > 0){
2723
cbmapiterinit(db->aidxs);
2724
while((kbuf = cbmapiternext(db->aidxs, NULL)) != NULL){
2725
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
2726
switch(attridx->type){
2727
case ESTIDXATTRSTR:
2728
case ESTIDXATTRNUM:
2729
size += vlfsiz(attridx->db);
2730
break;
2731
default:
2732
size += dpfsiz(attridx->db);
2733
break;
2734
}
2735
}
2736
}
2737
return size;
1528
2738
}
1529
2739
1530
2740
1531
2741
/* Search documents corresponding a condition for a database. */
1532
2742
int *est_db_search(ESTDB *db, ESTCOND *cond, int *nump, CBMAP *hints){
1533
2743
ESTSCORE *scores, *tscores;
1534
CBMAP *svmap;
2744
CBMAP *svmap, *ordattrs;
1535
2745
CBLIST *terms;
1536
2746
const char *term, *rp;
1537
2747
char *tmp, numbuf[ESTNUMBUFSIZ];
1538
int i, j, snum, pcnum, ncnum, tsnum, add, nnum, id, score, hnum, len, rest, *rval;
2748
int i, j, snum, ign, unum, knum, mnum, top, pcnum, ncnum, tsnum, add;
2749
int nnum, id, score, hnum, len, rest, *rval, rnum;
1539
2750
double tune;
1540
2751
assert(db && cond && nump);
2752
if(cond->auxwords) cbmapclose(cond->auxwords);
2753
cond->auxwords = cbmapopenex(ESTMINIBNUM);
1541
2754
scores = NULL;
1542
2755
snum = 0;
2756
ign = -1;
2757
ordattrs = cbmapopenex(cond->order ? (CB_LISTNUM(db->pdocs) + ESTMINIBNUM) : 1);
1543
2758
if(cond->phrase){
1544
2759
if(cbstrfwmatch(cond->phrase, ESTOPID)){
1545
2760
if((id = atoi(cond->phrase + strlen(ESTOPID))) > 0){
1546
2761
CB_MALLOC(scores, sizeof(ESTSCORE));
1547
2762
scores[0].id = id;
1548
2763
scores[0].score = 0;
2764
scores[0].value = NULL;
1549
2765
snum = 1;
1550
2766
} else {
1551
2767
CB_MALLOC(scores, 1);
1560
2776
CB_MALLOC(scores, sizeof(ESTSCORE));
1561
2777
scores[0].id = id;
1562
2778
scores[0].score = 0;
2779
scores[0].value = NULL;
1563
2780
snum = 1;
1564
2781
} else {
1565
2782
CB_MALLOC(scores, 1);
1570
2787
while(*rp > '\0' && *rp <= ' '){
1571
2788
rp++;
1572
2789
}
2790
knum = -1;
2791
unum = -1;
2792
mnum = -1;
2793
if(*rp >= '0' && *rp <= '9'){
2794
knum = atoi(rp);
2795
while(*rp >= '0' && *rp <= '9'){
2796
rp++;
2797
}
2798
while(*rp > '\0' && *rp <= ' '){
2799
rp++;
2800
}
2801
if(*rp >= '0' && *rp <= '9'){
2802
unum = atoi(rp);
2803
while(*rp >= '0' && *rp <= '9'){
2804
rp++;
2805
}
2806
while(*rp > '\0' && *rp <= ' '){
2807
rp++;
2808
}
2809
if(*rp >= '0' && *rp <= '9'){
2810
mnum = atoi(rp);
2811
while(*rp >= '0' && *rp <= '9'){
2812
rp++;
2813
}
2814
while(*rp > '\0' && *rp <= ' '){
2815
rp++;
2816
}
2817
}
2818
}
2819
}
2820
if(knum < 1) knum = ESTSMLRKNUM;
2821
if(unum < 1) unum = ESTSMLRUNUM;
2822
if(mnum < 1) mnum = ESTSMLRMNUM;
1573
2823
svmap = est_phrase_vector(rp);
1574
scores = est_search_similar(db, svmap, &snum, ESTSMLRKNUM, ESTSMLRUNUM, cond->tfidf,
1575
cond->order ? ESTSMLRNMIN : 0.0);
2824
scores = est_search_similar(db, svmap, &snum, knum, unum, mnum, cond->tfidf,
2825
cond->order ? ESTSMLRNMIN : 0.0, cond->auxmin, cond->auxwords);
1576
2826
cbmapclose(svmap);
2827
} else if(cbstrfwmatch(cond->phrase, ESTOPRANK)){
2828
rp = cond->phrase + strlen(ESTOPRANK);
2829
while(*rp > '\0' && *rp <= ' '){
2830
rp++;
2831
}
2832
top = atoi(rp);
2833
while((*rp >= '0' && *rp <= '9') || *rp == '-'){
2834
rp++;
2835
}
2836
while(*rp > '\0' && *rp <= ' '){
2837
rp++;
2838
}
2839
scores = est_search_rank(db, rp, top, &snum);
1577
2840
} else {
1578
if(cond->simple){
1579
tmp = est_phrase_from_thumb(cond->phrase);
1580
terms = est_phrase_terms(tmp);
1581
free(tmp);
1582
} else {
2841
switch(cond->pmode){
2842
default:
1583
2843
terms = est_phrase_terms(cond->phrase);
2844
break;
2845
case ESTPMSIMPLE:
2846
tmp = est_phrase_from_simple(cond->phrase);
2847
terms = est_phrase_terms(tmp);
2848
free(tmp);
2849
break;
2850
case ESTPMROUGH:
2851
tmp = est_phrase_from_rough(cond->phrase);
2852
terms = est_phrase_terms(tmp);
2853
free(tmp);
2854
break;
2855
case ESTPMUNION:
2856
tmp = est_phrase_from_union(cond->phrase);
2857
terms = est_phrase_terms(tmp);
2858
free(tmp);
2859
break;
2860
case ESTPMISECT:
2861
tmp = est_phrase_from_isect(cond->phrase);
2862
terms = est_phrase_terms(tmp);
2863
free(tmp);
2864
break;
1584
2865
}
1585
2866
pcnum = 0;
1586
2867
ncnum = 0;
1587
2868
add = TRUE;
1588
2869
for(i = 0; i < CB_LISTNUM(terms); i++){
1589
term = CB_LISTVAL(terms, i, NULL);
2870
term = CB_LISTVAL(terms, i);
1590
2871
if(!strcmp(term, ESTOPISECT)){
1591
2872
add = TRUE;
1592
2873
} else if(!strcmp(term, ESTOPDIFF)){
1595
2876
if(!strcmp(term, ESTOPUVSET)){
1596
2877
tscores = est_search_uvset(db, &tsnum, hints, add);
1597
2878
} else {
1598
tscores = est_search_union(db, term, cond->gstep, &tsnum, hints, add);
2879
tscores = est_search_union(db, term, cond->gstep, cond->cbxpn, &tsnum, hints, add,
2880
add && !cond->order ? cond->auxmin : -1, cond->auxwords);
1599
2881
}
1600
2882
if(add){
1601
if(cond->tfidf){
1602
tune = log(tsnum + 3);
1603
tune = tune * tune * tune;
1604
if(tune < 8.0) tune = 8.0;
1605
for(j = 0; j < tsnum; j++){
1606
tscores[j].score *= 10000 / tune;
2883
if(db->smode != ESTDFSCASIS){
2884
if(cond->tfidf){
2885
tune = pow(tsnum + 64, 0.4);
2886
for(j = 0; j < tsnum; j++){
2887
tscores[j].score *= 100.0 / tune;
2888
}
2889
} else {
2890
for(j = 0; j < tsnum; j++){
2891
tscores[j].score *= 10;
2892
}
1607
2893
}
1608
2894
}
1609
2895
pcnum++;
1615
2901
for(j = 0; j < tsnum; j++){
1616
2902
scores[snum+j].id = tscores[j].id;
1617
2903
scores[snum+j].score = add ? tscores[j].score : -1;
2904
scores[snum+j].value = NULL;
1618
2905
}
1619
2906
snum += tsnum;
1620
2907
free(tscores);
1626
2913
}
1627
2914
if(scores){
1628
2915
if(pcnum > 1 || ncnum > 0){
1629
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id);
2916
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
1630
2917
nnum = 0;
1631
2918
for(i = 0; i < snum; i++){
1632
2919
id = scores[i].id;
1634
2921
hnum = score >= 0 ? 1 : 0;
1635
2922
for(j = i + 1; j < snum && scores[j].id == id; j++){
1636
2923
if(score >= 0 && scores[j].score >= 0){
1637
score += scores[j].score;
2924
if(db->smode != ESTDFSCASIS) score += scores[j].score;
1638
2925
hnum++;
1639
2926
} else {
1640
2927
score = -1;
1643
2930
if(score >= 0 && hnum >= pcnum){
1644
2931
scores[nnum].id = id;
1645
2932
scores[nnum].score = score;
2933
scores[nnum].value = NULL;
1646
2934
nnum++;
1647
2935
}
1648
2936
i = j - 1;
1653
2941
CB_MALLOC(scores, 1);
1654
2942
snum = 0;
1655
2943
}
1656
cblistclose(terms);
2944
CB_LISTCLOSE(terms);
1657
2945
}
1658
2946
} else if(cond->attrs){
1659
scores = est_search_uvset(db, &snum, hints, TRUE);
2947
scores = NULL;
2948
for(i = 0; i < CB_LISTNUM(cond->attrs); i++){
2949
if((scores = est_search_aidx_attr(db, CB_LISTVAL(cond->attrs, i), &snum)) != NULL){
2950
ign = i;
2951
break;
2952
}
2953
}
2954
if(!scores) scores = est_search_uvset(db, &snum, hints, TRUE);
1660
2955
} else {
1661
2956
CB_MALLOC(scores, 1);
1662
2957
snum = 0;
1663
2958
}
2959
if(CB_LISTNUM(db->pdocs) > 0) scores = est_search_pidxs(db, cond, scores, &snum, ordattrs);
1664
2960
if(cbmaprnum(db->outcc) > 0){
1665
2961
tsnum = 0;
1666
2962
for(i = 0; i < snum; i++){
1670
2966
}
1671
2967
snum = tsnum;
1672
2968
}
1673
if(cond->max > 0 && cond->max * ESTATTRALW + 1 < snum && cond->attrs && !cond->order){
1674
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score);
1675
nnum = est_narrow_scores(db, cond->attrs, cond->order, scores, snum,
1676
cond->max * ESTATTRALW + 1, &rest);
2969
if(cond->max > 0 && cond->max * ESTATTRALW + 1 < snum && cond->attrs &&
2970
!cond->order && !cond->distinct){
2971
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score_desc);
2972
nnum = est_narrow_scores(db, cond->attrs, ign, cond->order, cond->distinct, scores, snum,
2973
cond->max * ESTATTRALW + 1, &rest, ordattrs);
1677
2974
if(hints){
1678
2975
sprintf(numbuf, "%d",
1679
2976
rest > cond->max / 2 ? (int)(snum * (nnum / (double)(snum - rest))) : nnum);
1680
cbmapput(hints, "", 0, numbuf, -1, FALSE);
2977
cbmapput(hints, "", 0, numbuf, -1, TRUE);
1681
2978
}
1682
2979
snum = nnum;
1683
2980
} else {
1684
if(cond->attrs || cond->order)
1685
snum = est_narrow_scores(db, cond->attrs, cond->order, scores, snum, INT_MAX, &rest);
1686
if(!cond->order) qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score);
2981
if(cond->attrs || cond->order || cond->distinct)
2982
snum = est_narrow_scores(db, cond->attrs, ign, cond->order, cond->distinct, scores, snum,
2983
INT_MAX, &rest, ordattrs);
2984
if(!cond->order) qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score_desc);
1687
2985
if(hints){
1688
2986
sprintf(numbuf, "%d", snum);
1689
cbmapput(hints, "", 0, numbuf, -1, FALSE);
2987
cbmapput(hints, "", 0, numbuf, -1, TRUE);
1690
2988
}
1691
2989
}
1692
2990
if(cond->shadows) cbmapclose(cond->shadows);
1697
2995
} else {
1698
2996
cond->shadows = NULL;
1699
2997
}
1700
if(cond->max >= 0 && cond->max < snum) snum = cond->max;
1701
CB_MALLOC(rval, snum * sizeof(int) + 1);
1702
for(i = 0; i < snum; i++){
1703
rval[i] = scores[i].id;
2998
rnum = snum - cond->skip;
2999
if(rnum < 0) rnum = 0;
3000
if(cond->max >= 0 && cond->max < rnum) rnum = cond->max;
3001
CB_MALLOC(rval, rnum * sizeof(int) + 1);
3002
tscores = scores + cond->skip;
3003
for(i = 0; i < rnum; i++){
3004
rval[i] = tscores[i].id;
1704
3005
}
1705
3006
if(cond->scfb){
1706
CB_REALLOC(cond->scores, snum * sizeof(int) + 1);
1707
for(i = 0; i < snum; i++){
1708
cond->scores[i] = scores[i].score;
3007
CB_REALLOC(cond->scores, rnum * sizeof(int) + 1);
3008
for(i = 0; i < rnum; i++){
3009
cond->scores[i] = tscores[i].score;
1709
3010
}
1710
cond->snum = snum;
3011
cond->snum = rnum;
1711
3012
}
1712
*nump = snum;
3013
*nump = rnum;
1713
3014
if(*nump < 1) db->ecode = ESTENOITEM;
3015
cbmapclose(ordattrs);
1714
3016
free(scores);
1715
3017
return rval;
1716
3018
}
1717
3019
1718
3020
3021
/* Search documents of plural databases. */
3022
int *est_db_search_meta(ESTDB **dbs, int dbnum, ESTCOND *cond, int *nump, CBMAP *hints){
3023
ESTMETASCORE *scores, *tscores;
3024
ESTCOND *tcond;
3025
CBMAP *thints, *umap;
3026
const char *kbuf, *otype, *rp;
3027
char *distinct, numbuf[ESTNUMBUFSIZ], *oname, *wp, *vbuf;
3028
int i, j, max, skip, smax, snum, *res, rnum, ksiz, num;
3029
time_t tval;
3030
assert(dbs && dbnum >= 0 && cond && nump);
3031
max = cond->max;
3032
if(cond->distinct) cond->max = -1;
3033
skip = cond->skip;
3034
cond->skip = 0;
3035
distinct = cond->distinct;
3036
cond->distinct = NULL;
3037
smax = ESTALLOCUNIT;
3038
CB_MALLOC(scores, smax * sizeof(ESTMETASCORE));
3039
snum = 0;
3040
for(i = 0; i < dbnum; i++){
3041
if(cond->mask & (1 << i)) continue;
3042
tcond = est_cond_dup(cond);
3043
est_cond_set_options(tcond, ESTCONDSCFB);
3044
thints = cbmapopenex(ESTMINIBNUM);
3045
res = est_db_search(dbs[i], tcond, &rnum, thints);
3046
for(j = 0; j < rnum; j++){
3047
if(snum >= smax){
3048
smax *= 2;
3049
CB_REALLOC(scores, smax * sizeof(ESTMETASCORE));
3050
}
3051
scores[snum].db = i;
3052
scores[snum].id = res[j];
3053
scores[snum].score = est_cond_score(tcond, j);
3054
scores[snum].value = NULL;
3055
snum++;
3056
}
3057
if(hints){
3058
cbmapiterinit(thints);
3059
while((kbuf = cbmapiternext(thints, &ksiz)) != NULL){
3060
num = atoi(cbmapiterval(kbuf, NULL));
3061
if((rp = cbmapget(hints, kbuf, ksiz, NULL)) != NULL) num += atoi(rp);
3062
sprintf(numbuf, "%d", num);
3063
cbmapput(hints, kbuf, ksiz, numbuf, -1, TRUE);
3064
}
3065
}
3066
free(res);
3067
cbmapclose(thints);
3068
est_cond_delete(tcond);
3069
}
3070
oname = NULL;
3071
otype = NULL;
3072
if(cond->order){
3073
oname = cbmemdup(cond->order, -1);
3074
cbstrtrim(oname);
3075
otype = ESTORDSTRA;
3076
if((wp = strchr(oname, ' ')) != NULL){
3077
*wp = '\0';
3078
rp = wp + 1;
3079
while(*rp == ' '){
3080
rp++;
3081
}
3082
otype = rp;
3083
}
3084
}
3085
if(oname){
3086
if(!cbstricmp(oname, ESTORDIDA)){
3087
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_id_asc);
3088
} else if(!cbstricmp(oname, ESTORDIDD)){
3089
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_id_desc);
3090
} else if(!cbstricmp(oname, ESTORDSCA)){
3091
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_score_asc);
3092
} else if(!cbstricmp(oname, ESTORDSCD)){
3093
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_score_desc);
3094
} else {
3095
for(i = 0; i < snum; i++){
3096
scores[i].value = est_db_get_doc_attr(dbs[scores[i].db], scores[i].id, oname);
3097
if(!scores[i].value) scores[i].value = cbmemdup("", 0);
3098
}
3099
if(!cbstricmp(otype, ESTORDSTRA)){
3100
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_str_asc);
3101
} else if(!cbstricmp(otype, ESTORDSTRD)){
3102
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_str_desc);
3103
} else if(!cbstricmp(otype, ESTORDNUMA)){
3104
for(i = 0; i < snum; i++){
3105
tval = cbstrmktime(scores[i].value);
3106
free(scores[i].value);
3107
scores[i].value = (void *)tval;
3108
}
3109
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_num_asc);
3110
for(i = 0; i < snum; i++){
3111
scores[i].value = NULL;
3112
}
3113
} else if(!cbstricmp(otype, ESTORDNUMD)){
3114
for(i = 0; i < snum; i++){
3115
tval = cbstrmktime(scores[i].value);
3116
free(scores[i].value);
3117
scores[i].value = (void *)tval;
3118
}
3119
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_num_desc);
3120
for(i = 0; i < snum; i++){
3121
scores[i].value = NULL;
3122
}
3123
}
3124
for(i = 0; i < snum; i++){
3125
free(scores[i].value);
3126
}
3127
}
3128
free(oname);
3129
} else {
3130
qsort(scores, snum, sizeof(ESTMETASCORE), est_metascore_compare_by_score_desc);
3131
}
3132
if(distinct){
3133
umap = cbmapopenex(snum + 1);
3134
rnum = 0;
3135
for(i = 0; i < snum; i++){
3136
vbuf = est_db_get_doc_attr(dbs[scores[i].db], scores[i].id, distinct);
3137
if(!vbuf) vbuf = cbmemdup("", 0);
3138
if(cbmapput(umap, vbuf, -1, "", 0, FALSE)) scores[rnum++] = scores[i];
3139
free(vbuf);
3140
}
3141
snum = rnum;
3142
cbmapclose(umap);
3143
}
3144
rnum = snum - skip;
3145
if(rnum < 0) rnum = 0;
3146
if(cond->max >= 0 && cond->max < rnum) rnum = cond->max;
3147
CB_MALLOC(res, rnum * sizeof(int) * 2 + 1);
3148
tscores = scores + skip;
3149
for(i = 0; i < rnum; i++){
3150
res[i*2] = tscores[i].db;
3151
res[i*2+1] = tscores[i].id;
3152
}
3153
if(cond->scfb){
3154
CB_REALLOC(cond->scores, rnum * sizeof(int) + 1);
3155
for(i = 0; i < rnum; i++){
3156
cond->scores[i] = tscores[i].score;
3157
}
3158
cond->snum = rnum;
3159
}
3160
*nump = rnum * 2;
3161
free(scores);
3162
cond->max = max;
3163
cond->skip = skip;
3164
cond->distinct = distinct;
3165
return res;
3166
}
3167
3168
1719
3169
/* Check whether a document object matches the phrase of a search condition object definitely. */
1720
3170
int est_db_scan_doc(ESTDB *db, ESTDOC *doc, ESTCOND *cond){
1721
3171
struct { char *word; int num; } wsets[ESTSCANWNUM];
1725
3175
char *tmp;
1726
3176
int i, j, k, wsnum, add, rsiz, hit;
1727
3177
assert(db && doc && cond);
1728
if(!cond->phrase || cbstrfwmatch(cond->phrase, ESTOPSIMILAR)) return FALSE;
1729
if(!doc->dtexts) doc->dtexts = cblistopen();
1730
if(cond->simple){
1731
tmp = est_phrase_from_thumb(cond->phrase);
1732
terms = est_phrase_terms(tmp);
1733
free(tmp);
1734
} else {
3178
if(!cond->phrase || cbstrfwmatch(cond->phrase, ESTOPSIMILAR) ||
3179
cbstrfwmatch(cond->phrase, ESTOPID) || cbstrfwmatch(cond->phrase, ESTOPURI)) return FALSE;
3180
if(!doc->dtexts) CB_LISTOPEN(doc->dtexts);
3181
switch(cond->pmode){
3182
default:
1735
3183
terms = est_phrase_terms(cond->phrase);
3184
break;
3185
case ESTPMSIMPLE:
3186
tmp = est_phrase_from_simple(cond->phrase);
3187
terms = est_phrase_terms(tmp);
3188
free(tmp);
3189
break;
3190
case ESTPMROUGH:
3191
tmp = est_phrase_from_rough(cond->phrase);
3192
terms = est_phrase_terms(tmp);
3193
free(tmp);
3194
break;
3195
case ESTPMUNION:
3196
tmp = est_phrase_from_union(cond->phrase);
3197
terms = est_phrase_terms(tmp);
3198
free(tmp);
3199
break;
3200
case ESTPMISECT:
3201
tmp = est_phrase_from_isect(cond->phrase);
3202
terms = est_phrase_terms(tmp);
3203
free(tmp);
3204
break;
1736
3205
}
1737
3206
wsnum = 0;
1738
3207
add = TRUE;
1739
3208
for(i = 0; i < CB_LISTNUM(terms); i++){
1740
term = CB_LISTVAL(terms, i, NULL);
3209
term = CB_LISTVAL(terms, i);
1741
3210
if(!strcmp(term, ESTOPISECT)){
1742
3211
add = TRUE;
1743
3212
} else if(!strcmp(term, ESTOPDIFF)){
1757
3226
wsets[wsnum].num = i;
1758
3227
wsnum++;
1759
3228
}
1760
cblistclose(words);
3229
CB_LISTCLOSE(words);
1761
3230
}
1762
3231
}
1763
3232
for(i = -1; i < CB_LISTNUM(doc->dtexts); i++){
1764
3233
if(i < 0){
1765
3234
if(!doc->attrs || !(text = cbmapget(doc->attrs, "", 0, NULL))) continue;
1766
3235
} else {
1767
text = CB_LISTVAL(doc->dtexts, i, NULL);
3236
text = CB_LISTVAL(doc->dtexts, i);
1768
3237
}
1769
3238
rbuf = (unsigned char *)est_uconv_in(text, strlen(text), &rsiz);
1770
3239
est_canonicalize_text(rbuf, rsiz, FALSE);
1771
3240
tmp = est_uconv_out((char *)rbuf, rsiz, &rsiz);
1772
3241
for(j = 0; j < wsnum; j++){
1773
3242
if(!wsets[j].word) continue;
1774
if(strstr(tmp, wsets[j].word)){
3243
if(est_strstr_sparse(tmp, wsets[j].word)){
1775
3244
for(k = 0; k < wsnum; k++){
1776
3245
if(!wsets[k].word) continue;
1777
3246
if(wsets[k].num == wsets[j].num){
1790
3259
free(wsets[i].word);
1791
3260
hit = FALSE;
1792
3261
}
1793
cblistclose(terms);
3262
CB_LISTCLOSE(terms);
1794
3263
return hit;
1795
3264
}
1796
3265
1798
3267
/* Set the maximum size of the cache memory of a database. */
1799
3268
void est_db_set_cache_size(ESTDB *db, size_t size, int anum, int tnum, int rnum){
1800
3269
assert(db);
1801
if(dpwritable(db->metadb) && size > 0) db->icmax = size;
1802
if(anum > 0) db->acmnum = anum;
1803
if(tnum > 0) db->tcmnum = tnum;
1804
if(rnum > 0) db->rcmnum = rnum;
1805
db->vcmnum = db->acmnum / 2 + 1;
3270
if(dpwritable(db->metadb) && size >= 0) db->icmax = size;
3271
if(anum >= 0) db->acmnum = anum;
3272
if(tnum >= 0) db->tcmnum = tnum;
3273
if(rnum >= 0) db->rcmnum = rnum;
3274
db->vcmnum = db->acmnum / 2;
3275
}
3276
3277
3278
/* Add a pseudo index directory to a database. */
3279
int est_db_add_pseudo_index(ESTDB *db, const char *path){
3280
CBLIST *files;
3281
const char *file;
3282
char pbuf[ESTPATHBUFSIZ];
3283
int i, len;
3284
assert(db && path);
3285
if(!(files = cbdirlist(path))){
3286
db->ecode = ESTEINVAL;
3287
return FALSE;
3288
}
3289
cblistsort(files);
3290
for(i = 0; i < CB_LISTNUM(files); i++){
3291
file = CB_LISTVAL(files, i);
3292
if(!strcmp(file, ESTCDIRSTR) || !strcmp(file, ESTPDIRSTR)) continue;
3293
len = sprintf(pbuf, "%s%c%s", path, ESTPATHCHR, file);
3294
CB_LISTPUSH(db->pdocs, pbuf, len);
3295
}
3296
CB_LISTCLOSE(files);
3297
if(db->puris){
3298
cbmapclose(db->puris);
3299
db->puris = NULL;
3300
}
3301
return TRUE;
1806
3302
}
1807
3303
1808
3304
1827
3323
unsigned char *utext;
1828
3324
char *tmp;
1829
3325
int i, j, k, size, cc, wsiz, nsiz, tsiz;
1830
assert(text);
3326
assert(text && list);
1831
3327
utext = (unsigned char *)est_uconv_in(text, strlen(text), &size);
1832
3328
if(norm) est_normalize_text(utext, size, &size);
1833
3329
est_canonicalize_text(utext, size, FALSE);
1834
words = cblistopen();
3330
CB_LISTOPEN(words);
1835
3331
for(i = 0; i < size; i += 2){
1836
3332
cc = est_char_category(utext[i] * 0x100 + utext[i+1]);
1837
3333
for(j = i + 2; j < size; j += 2){
1840
3336
switch(cc){
1841
3337
case ESTDELIMCHR:
1842
3338
case ESTWESTALPH:
1843
cblistpush(words, (char *)(utext + i), j - i);
3339
CB_LISTPUSH(words, (char *)(utext + i), j - i);
1844
3340
break;
1845
3341
case ESTEASTALPH:
1846
3342
for(k = i; k < j; k += 2){
1847
3343
if(j - k >= 4){
1848
cblistpush(words, (char *)(utext + k), 4);
3344
CB_LISTPUSH(words, (char *)(utext + k), 4);
1849
3345
} else {
1850
cblistpush(words, (char *)(utext + k), 2);
3346
CB_LISTPUSH(words, (char *)(utext + k), 2);
1851
3347
}
1852
3348
}
1853
3349
break;
1857
3353
i = j - 2;
1858
3354
}
1859
3355
for(i = 0; i < CB_LISTNUM(words); i++){
1860
word = (unsigned char *)CB_LISTVAL2(words, i, &wsiz);
3356
word = (unsigned char *)CB_LISTVAL2(words, i, wsiz);
1861
3357
if(est_char_category(word[0] * 0x100 + word[1]) == ESTEASTALPH && wsiz == 2 &&
1862
3358
i < CB_LISTNUM(words) - 1){
1863
3359
next = (unsigned char *)cblistval(words, i + 1, &nsiz);
1871
3367
}
1872
3368
}
1873
3369
for(i = 0; i < CB_LISTNUM(words); i++){
1874
word = (unsigned char *)CB_LISTVAL2(words, i, &wsiz);
3370
word = (unsigned char *)CB_LISTVAL2(words, i, wsiz);
1875
3371
if(!tail && wsiz == 2 && i == CB_LISTNUM(words) - 1){
1876
3372
if(est_char_category(word[0] * 0x100 + word[1]) == ESTEASTALPH) continue;
1877
3373
}
1878
3374
tmp = est_uconv_out((char *)word, wsiz, &tsiz);
1879
cblistpushbuf(list, tmp, tsiz);
3375
CB_LISTPUSHBUF(list, tmp, tsiz);
1880
3376
}
1881
cblistclose(words);
3377
CB_LISTCLOSE(words);
1882
3378
free(utext);
1883
3379
}
1884
3380
1890
3386
unsigned char *utext;
1891
3387
char *tmp;
1892
3388
int i, j, k, size, cc, wsiz, nsiz, tsiz;
1893
assert(text);
3389
assert(text && list);
1894
3390
utext = (unsigned char *)est_uconv_in(text, strlen(text), &size);
1895
3391
if(norm) est_normalize_text(utext, size, &size);
1896
3392
est_canonicalize_text(utext, size, FALSE);
1897
words = cblistopen();
3393
CB_LISTOPEN(words);
1898
3394
for(i = 0; i < size; i += 2){
1899
3395
cc = est_char_category_perfng(utext[i] * 0x100 + utext[i+1]);
1900
3396
for(j = i + 2; j < size; j += 2){
1904
3400
case ESTEASTALPH:
1905
3401
for(k = i; k < j; k += 2){
1906
3402
if(j - k >= 4){
1907
cblistpush(words, (char *)(utext + k), 4);
3403
CB_LISTPUSH(words, (char *)(utext + k), 4);
1908
3404
} else {
1909
cblistpush(words, (char *)(utext + k), 2);
3405
CB_LISTPUSH(words, (char *)(utext + k), 2);
1910
3406
}
1911
3407
}
1912
3408
break;
1916
3412
i = j - 2;
1917
3413
}
1918
3414
for(i = 0; i < CB_LISTNUM(words); i++){
1919
word = (unsigned char *)CB_LISTVAL2(words, i, &wsiz);
3415
word = (unsigned char *)CB_LISTVAL2(words, i, wsiz);
1920
3416
if(est_char_category_perfng(word[0] * 0x100 + word[1]) == ESTEASTALPH && wsiz == 2 &&
1921
3417
i < CB_LISTNUM(words) - 1){
1922
3418
next = (unsigned char *)cblistval(words, i + 1, &nsiz);
1930
3426
}
1931
3427
}
1932
3428
for(i = 0; i < CB_LISTNUM(words); i++){
1933
word = (unsigned char *)CB_LISTVAL2(words, i, &wsiz);
3429
word = (unsigned char *)CB_LISTVAL2(words, i, wsiz);
1934
3430
if(!tail && wsiz == 2 && i == CB_LISTNUM(words) - 1){
1935
3431
if(est_char_category_perfng(word[0] * 0x100 + word[1]) == ESTEASTALPH) continue;
1936
3432
}
1937
3433
tmp = est_uconv_out((char *)word, wsiz, &tsiz);
1938
cblistpushbuf(list, tmp, tsiz);
1939
}
1940
cblistclose(words);
3434
CB_LISTPUSHBUF(list, tmp, tsiz);
3435
}
3436
CB_LISTCLOSE(words);
3437
free(utext);
3438
}
3439
3440
3441
/* Make a snippet of an arbitrary string. */
3442
char *est_str_make_snippet(const char *str, const CBLIST *words,
3443
int wwidth, int hwidth, int awidth){
3444
assert(str && words && wwidth >= 0 && hwidth >= 0 && awidth >= 0);
3445
return est_make_snippet(str, strlen(str), words, wwidth, hwidth, awidth);
3446
}
3447
3448
3449
/* Break a sentence of text and extract words, using character category analyzer.
3450
`text' specifies a sentence of text.
3451
`list' specifies a list object to which extract words are added.
3452
`norm' specifies whether to normalize the text. */
3453
void est_break_text_chrcat(const char *text, CBLIST *list, int norm){
3454
unsigned char *utext;
3455
char *tmp;
3456
int i, j, size, cc, tsiz;
3457
assert(text && list);
3458
utext = (unsigned char *)est_uconv_in(text, strlen(text), &size);
3459
if(norm) est_normalize_text(utext, size, &size);
3460
est_canonicalize_text(utext, size, FALSE);
3461
for(i = 0; i < size; i += 2){
3462
cc = est_char_category_chrcat(utext[i] * 0x100 + utext[i+1]);
3463
for(j = i + 2; j < size; j += 2){
3464
if(est_char_category_chrcat(utext[j] * 0x100 + utext[j+1]) != cc &&
3465
(cc != ESTWESTALPH || utext[j] != 0x00 || utext[j+1] != 0x2d) &&
3466
(cc != ESTHIRAGANA || utext[j] != 0x30 || utext[j+1] != 0xfc)) break;
3467
}
3468
if(cc != ESTSPACECHR){
3469
tmp = est_uconv_out((char *)(utext + i), j - i, &tsiz);
3470
CB_LISTPUSHBUF(list, tmp, tsiz);
3471
}
3472
i = j - 2;
3473
}
1941
3474
free(utext);
1942
3475
}
1943
3476
2158
3691
2159
3692
2160
3693
/* Compress a serial object with ZLIB. */
2161
char *est_deflate(const char *ptr, int size, int *sp){
3694
char *est_deflate(const char *ptr, int size, int *sp, int mode){
3695
#if ESTUSEZLIB
2162
3696
z_stream zs;
2163
3697
char *buf;
2164
3698
unsigned char obuf[ESTIOBUFSIZ];
2165
3699
int rv, asiz, bsiz, osiz;
2166
assert(ptr && sp);
2167
3700
if(size < 0) size = strlen(ptr);
2168
if(!ESTUSEZLIB){
2169
if(sp) *sp = size;
2170
return cbmemdup(ptr, size);
2171
}
2172
3701
zs.zalloc = Z_NULL;
2173
3702
zs.zfree = Z_NULL;
2174
3703
zs.opaque = Z_NULL;
2175
if(deflateInit2(&zs, 5, Z_DEFLATED, -15, 7, Z_DEFAULT_STRATEGY) != Z_OK) return NULL;
2176
asiz = ESTIOBUFSIZ;
3704
switch(mode){
3705
case -1:
3706
if(deflateInit2(&zs, 5, Z_DEFLATED, -15, 7, Z_DEFAULT_STRATEGY) != Z_OK)
3707
return NULL;
3708
break;
3709
case 1:
3710
if(deflateInit2(&zs, 6, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY) != Z_OK)
3711
return NULL;
3712
break;
3713
default:
3714
if(deflateInit2(&zs, 6, Z_DEFLATED, 15, 8, Z_DEFAULT_STRATEGY) != Z_OK)
3715
return NULL;
3716
break;
3717
}
3718
asiz = size + 16;
3719
if(asiz < ESTIOBUFSIZ) asiz = ESTIOBUFSIZ;
2177
3720
CB_MALLOC(buf, asiz);
2178
3721
bsiz = 0;
2179
3722
zs.next_in = (unsigned char *)ptr;
2203
3746
}
2204
3747
memcpy(buf + bsiz, obuf, osiz);
2205
3748
bsiz += osiz;
2206
buf[bsiz++] = '\0';
3749
buf[bsiz] = '\0';
3750
if(mode == -1) bsiz++;
2207
3751
*sp = bsiz;
2208
3752
deflateEnd(&zs);
2209
3753
return buf;
3754
#else
3755
if(size < 0) size = strlen(ptr);
3756
*sp = size;
3757
return cbmemdup(ptr, size);
3758
#endif
2210
3759
}
2211
3760
2212
3761
2213
3762
/* Decompress a serial object compressed with ZLIB. */
2214
char *est_inflate(const char *ptr, int size, int *sp){
3763
char *est_inflate(const char *ptr, int size, int *sp, int mode){
3764
#if ESTUSEZLIB
2215
3765
z_stream zs;
2216
3766
char *buf;
2217
3767
unsigned char obuf[ESTIOBUFSIZ];
2218
3768
int rv, asiz, bsiz, osiz;
2219
assert(ptr && size >= 0 && sp);
2220
if(!ESTUSEZLIB){
2221
*sp = size;
2222
return cbmemdup(ptr, size);
2223
}
2224
3769
zs.zalloc = Z_NULL;
2225
3770
zs.zfree = Z_NULL;
2226
3771
zs.opaque = Z_NULL;
2227
if(inflateInit2(&zs, -15) != Z_OK) return NULL;
2228
asiz = ESTIOBUFSIZ;
3772
switch(mode){
3773
case -1:
3774
if(inflateInit2(&zs, -15) != Z_OK) return NULL;
3775
break;
3776
case 1:
3777
if(inflateInit2(&zs, 15 + 16) != Z_OK) return NULL;
3778
break;
3779
default:
3780
if(inflateInit2(&zs, 15) != Z_OK) return NULL;
3781
break;
3782
}
3783
asiz = size * 2 + 16;
3784
if(asiz < ESTIOBUFSIZ) asiz = ESTIOBUFSIZ;
2229
3785
CB_MALLOC(buf, asiz);
2230
3786
bsiz = 0;
2231
3787
zs.next_in = (unsigned char *)ptr;
2259
3815
if(sp) *sp = bsiz;
2260
3816
inflateEnd(&zs);
2261
3817
return buf;
3818
#else
3819
if(sp) *sp = size;
3820
return cbmemdup(ptr, size);
3821
#endif
3822
}
3823
3824
3825
/* Compress a serial object with LZO. */
3826
char *est_lzoencode(const char *ptr, int size, int *sp){
3827
#if ESTUSELZO
3828
char wrkmem[LZO1X_1_MEM_COMPRESS];
3829
lzo_bytep buf;
3830
lzo_uint bsiz;
3831
if(size < 0) size = strlen(ptr);
3832
CB_MALLOC(buf, size + size / 16 + 80);
3833
if(lzo1x_1_compress((lzo_bytep)ptr, size, buf, &bsiz, wrkmem) != LZO_E_OK){
3834
free(buf);
3835
return NULL;
3836
}
3837
buf[bsiz] = '\0';
3838
*sp = bsiz;
3839
return (char *)buf;
3840
#else
3841
if(size < 0) size = strlen(ptr);
3842
*sp = size;
3843
return cbmemdup(ptr, size);
3844
#endif
3845
}
3846
3847
3848
/* Decompress a serial object compressed with LZO. */
3849
char *est_lzodecode(const char *ptr, int size, int *sp){
3850
#if ESTUSELZO
3851
lzo_bytep buf;
3852
lzo_uint bsiz;
3853
int rat, rv;
3854
rat = 4;
3855
while(TRUE){
3856
bsiz = (size + 256) * rat + 3;
3857
CB_MALLOC(buf, bsiz + 1);
3858
rv = lzo1x_decompress_safe((lzo_bytep)(ptr), size, buf, &bsiz, NULL);
3859
if(rv == LZO_E_OK){
3860
break;
3861
} else if(rv == LZO_E_OUTPUT_OVERRUN){
3862
free(buf);
3863
rat *= 2;
3864
} else {
3865
free(buf);
3866
return NULL;
3867
}
3868
}
3869
buf[bsiz] = '\0';
3870
if(sp) *sp = bsiz;
3871
return (char *)buf;
3872
#else
3873
if(sp) *sp = size;
3874
return cbmemdup(ptr, size);
3875
#endif
3876
}
3877
3878
3879
/* Compress a serial object with BZIP2. */
3880
char *est_bzencode(const char *ptr, int size, int *sp){
3881
#if ESTUSEBZIP
3882
bz_stream zs;
3883
char *buf, obuf[ESTIOBUFSIZ];
3884
int rv, asiz, bsiz, osiz;
3885
if(size < 0) size = strlen(ptr);
3886
zs.bzalloc = NULL;
3887
zs.bzfree = NULL;
3888
zs.opaque = NULL;
3889
if(BZ2_bzCompressInit(&zs, 9, 0, 30) != BZ_OK) return NULL;
3890
asiz = size + 16;
3891
if(asiz < ESTIOBUFSIZ) asiz = ESTIOBUFSIZ;
3892
CB_MALLOC(buf, asiz);
3893
bsiz = 0;
3894
zs.next_in = (char *)ptr;
3895
zs.avail_in = size;
3896
zs.next_out = obuf;
3897
zs.avail_out = ESTIOBUFSIZ;
3898
while((rv = BZ2_bzCompress(&zs, BZ_FINISH)) == BZ_FINISH_OK){
3899
osiz = ESTIOBUFSIZ - zs.avail_out;
3900
if(bsiz + osiz > asiz){
3901
asiz = asiz * 2 + osiz;
3902
CB_REALLOC(buf, asiz);
3903
}
3904
memcpy(buf + bsiz, obuf, osiz);
3905
bsiz += osiz;
3906
zs.next_out = obuf;
3907
zs.avail_out = ESTIOBUFSIZ;
3908
}
3909
if(rv != BZ_STREAM_END){
3910
free(buf);
3911
BZ2_bzCompressEnd(&zs);
3912
return NULL;
3913
}
3914
osiz = ESTIOBUFSIZ - zs.avail_out;
3915
if(bsiz + osiz + 1 > asiz){
3916
asiz = asiz * 2 + osiz;
3917
CB_REALLOC(buf, asiz);
3918
}
3919
memcpy(buf + bsiz, obuf, osiz);
3920
bsiz += osiz;
3921
buf[bsiz] = '\0';
3922
*sp = bsiz;
3923
BZ2_bzCompressEnd(&zs);
3924
return buf;
3925
#else
3926
if(size < 0) size = strlen(ptr);
3927
*sp = size;
3928
return cbmemdup(ptr, size);
3929
#endif
3930
}
3931
3932
3933
/* Decompress a serial object compressed with BZIP2. */
3934
char *est_bzdecode(const char *ptr, int size, int *sp){
3935
#if ESTUSEBZIP
3936
bz_stream zs;
3937
char *buf, obuf[ESTIOBUFSIZ];
3938
int rv, asiz, bsiz, osiz;
3939
zs.bzalloc = NULL;
3940
zs.bzfree = NULL;
3941
zs.opaque = NULL;
3942
if(BZ2_bzDecompressInit(&zs, 0, 0) != BZ_OK) return NULL;
3943
asiz = size * 2 + 16;
3944
if(asiz < ESTIOBUFSIZ) asiz = ESTIOBUFSIZ;
3945
CB_MALLOC(buf, asiz);
3946
bsiz = 0;
3947
zs.next_in = (char *)ptr;
3948
zs.avail_in = size;
3949
zs.next_out = obuf;
3950
zs.avail_out = ESTIOBUFSIZ;
3951
while((rv = BZ2_bzDecompress(&zs)) == BZ_OK){
3952
osiz = ESTIOBUFSIZ - zs.avail_out;
3953
if(bsiz + osiz >= asiz){
3954
asiz = asiz * 2 + osiz;
3955
CB_REALLOC(buf, asiz);
3956
}
3957
memcpy(buf + bsiz, obuf, osiz);
3958
bsiz += osiz;
3959
zs.next_out = obuf;
3960
zs.avail_out = ESTIOBUFSIZ;
3961
}
3962
if(rv != BZ_STREAM_END){
3963
free(buf);
3964
BZ2_bzDecompressEnd(&zs);
3965
return NULL;
3966
}
3967
osiz = ESTIOBUFSIZ - zs.avail_out;
3968
if(bsiz + osiz >= asiz){
3969
asiz = asiz * 2 + osiz;
3970
CB_REALLOC(buf, asiz);
3971
}
3972
memcpy(buf + bsiz, obuf, osiz);
3973
bsiz += osiz;
3974
buf[bsiz] = '\0';
3975
if(sp) *sp = bsiz;
3976
BZ2_bzDecompressEnd(&zs);
3977
return buf;
3978
#else
3979
if(sp) *sp = size;
3980
return cbmemdup(ptr, size);
3981
#endif
2262
3982
}
2263
3983
2264
3984
2340
4060
/* Create a regular expression object. */
2341
4061
void *est_regex_new(const char *str){
2342
4062
regex_t regex;
4063
int options;
2343
4064
assert(str);
2344
if(regcomp(®ex, str, REG_EXTENDED | REG_NOSUB) != 0) return NULL;
4065
options = REG_EXTENDED | REG_NOSUB;
4066
if(str[0] == '*' && str[1] == 'I' && str[2] == ':'){
4067
options |= REG_ICASE;
4068
str += 3;
4069
}
4070
if(regcomp(®ex, str, options) != 0) return NULL;
2345
4071
return cbmemdup((char *)®ex, sizeof(regex_t));
2346
4072
}
2347
4073
2361
4087
}
2362
4088
2363
4089
4090
/* Check whether a regular expression matches a string. */
4091
int est_regex_match_str(const char *rstr, const char *tstr){
4092
void *regex;
4093
int rv;
4094
assert(rstr && tstr);
4095
if(!(regex = est_regex_new(rstr))) return FALSE;
4096
rv = est_regex_match(regex, tstr);
4097
est_regex_delete(regex);
4098
return rv;
4099
}
4100
4101
2364
4102
/* Replace each substring matching a regular expression string. */
2365
4103
char *est_regex_replace(const char *str, const char *bef, const char *aft){
2366
4104
regex_t regex;
2367
4105
regmatch_t subs[256];
2368
4106
CBDATUM *datum;
2369
const char *sp;
2370
int first;
4107
const char *sp, *rp;
4108
int options, first, num;
2371
4109
assert(str && bef && aft);
2372
if(bef[0] == '\0' || regcomp(®ex, bef, REG_EXTENDED) != 0) return cbmemdup(str, -1);
4110
options = REG_EXTENDED;
4111
if(bef[0] == '*' && bef[1] == 'I' && bef[2] == ':'){
4112
options |= REG_ICASE;
4113
bef += 3;
4114
}
4115
if(bef[0] == '\0' || regcomp(®ex, bef, options) != 0) return cbmemdup(str, -1);
2373
4116
if(regexec(®ex, str, ESTREGSUBMAX, subs, 0) != 0){
2374
4117
regfree(®ex);
2375
4118
return cbmemdup(str, -1);
2376
4119
}
2377
4120
sp = str;
2378
datum = cbdatumopen("", 0);
4121
CB_DATUMOPEN(datum);
2379
4122
first = TRUE;
2380
while(sp[0] != '\0' && regexec(®ex, sp, 1, subs, first ? 0 : REG_NOTBOL) == 0){
4123
while(sp[0] != '\0' && regexec(®ex, sp, 10, subs, first ? 0 : REG_NOTBOL) == 0){
2381
4124
first = FALSE;
2382
4125
if(subs[0].rm_so == -1) break;
2383
cbdatumcat(datum, sp, subs[0].rm_so);
2384
cbdatumcat(datum, aft, -1);
4126
CB_DATUMCAT(datum, sp, subs[0].rm_so);
4127
for(rp = aft; *rp != '\0'; rp++){
4128
if(*rp == '\\'){
4129
if(rp[1] >= '0' && rp[1] <= '9'){
4130
num = rp[1] - '0';
4131
if(subs[num].rm_so != -1 && subs[num].rm_eo != -1)
4132
CB_DATUMCAT(datum, sp + subs[num].rm_so, subs[num].rm_eo - subs[num].rm_so);
4133
++rp;
4134
} else if(rp[1] != '\0'){
4135
CB_DATUMCAT(datum, ++rp, 1);
4136
}
4137
} else if(*rp == '&'){
4138
CB_DATUMCAT(datum, sp + subs[0].rm_so, subs[0].rm_eo - subs[0].rm_so);
4139
} else {
4140
CB_DATUMCAT(datum, rp, 1);
4141
}
4142
}
2385
4143
sp += subs[0].rm_eo;
2386
4144
if(subs[0].rm_eo < 1) break;
2387
4145
}
2388
cbdatumcat(datum, sp, -1);
4146
CB_DATUMCAT(datum, sp, strlen(sp));
2389
4147
regfree(®ex);
2390
4148
return cbdatumtomalloc(datum, NULL);
2391
4149
}
2392
4150
2393
4151
4152
/* Duplicate a document object. */
4153
ESTDOC *est_doc_dup(ESTDOC *doc){
4154
ESTDOC *ndoc;
4155
assert(doc);
4156
CB_MALLOC(ndoc, sizeof(ESTDOC));
4157
ndoc->id = doc->id;
4158
ndoc->attrs = doc->attrs ? cbmapdup(doc->attrs) : NULL;
4159
ndoc->dtexts = doc->dtexts ? cblistdup(doc->dtexts) : NULL;
4160
ndoc->kwords = doc->kwords ? cbmapdup(doc->kwords) : NULL;
4161
return ndoc;
4162
}
4163
4164
2394
4165
/* Set the ID number of a document object. */
2395
4166
void est_doc_set_id(ESTDOC *doc, int id){
2396
4167
assert(doc);
2407
4178
}
2408
4179
2409
4180
4181
/* Reduce the texts to fit to the specified size. */
4182
void est_doc_slim(ESTDOC *doc, int len){
4183
const char *vbuf;
4184
unsigned char *tbuf;
4185
int i, vsiz, tsiz;
4186
assert(doc && len >= 0);
4187
if(!doc->dtexts) return;
4188
if(doc->attrs && cbmapget(doc->attrs, "", 0, &vsiz)) len -= vsiz;
4189
for(i = 0; i < CB_LISTNUM(doc->dtexts); i++){
4190
vbuf = CB_LISTVAL2(doc->dtexts, i, vsiz);
4191
len -= vsiz;
4192
if(len < 0){
4193
tbuf = (unsigned char *)cbmemdup(vbuf, vsiz);
4194
tsiz = vsiz > -len ? vsiz + len : 0;
4195
if(tsiz > 0){
4196
while(tsiz < vsiz){
4197
if(tbuf[tsiz] <= ' ' || (tbuf[tsiz] & 0xf0) == 0xe0) break;
4198
tsiz++;
4199
}
4200
}
4201
while(CB_LISTNUM(doc->dtexts) > i){
4202
CB_LISTDROP(doc->dtexts);
4203
}
4204
CB_LISTPUSHBUF(doc->dtexts, (char *)tbuf, tsiz);
4205
break;
4206
}
4207
}
4208
}
4209
4210
2410
4211
/* Check whether a docuemnt object is empty. */
2411
4212
int est_doc_is_empty(ESTDOC *doc){
2412
4213
assert(doc);
2413
if((!doc->dtexts || cblistnum(doc->dtexts) < 1) &&
4214
if((!doc->dtexts || CB_LISTNUM(doc->dtexts) < 1) &&
2414
4215
(!doc->attrs || !cbmapget(doc->attrs, "", 0, NULL))) return TRUE;
2415
4216
return FALSE;
2416
4217
}
2417
4218
2418
4219
4220
/* Duplicate a condition object. */
4221
ESTCOND *est_cond_dup(ESTCOND *cond){
4222
ESTCOND *ncond;
4223
assert(cond);
4224
CB_MALLOC(ncond, sizeof(ESTCOND));
4225
ncond->phrase = cond->phrase ? cbmemdup(cond->phrase, -1) : NULL;
4226
ncond->gstep = cond->gstep;
4227
ncond->tfidf = cond->tfidf;
4228
ncond->pmode = cond->pmode;
4229
ncond->cbxpn = cond->cbxpn;
4230
ncond->attrs = cond->attrs ? cblistdup(cond->attrs) : NULL;
4231
ncond->order = cond->order ? cbmemdup(cond->order, -1) : NULL;
4232
ncond->max = cond->max;
4233
ncond->skip = cond->skip;
4234
ncond->auxmin = cond->auxmin;
4235
ncond->auxwords = cond->auxwords ? cbmapdup(cond->auxwords) : NULL;
4236
ncond->scfb = cond->scfb;
4237
ncond->scores = cond->scores ?
4238
(int *)cbmemdup((char *)cond->scores, cond->snum * sizeof(int)) : NULL;
4239
ncond->snum = cond->snum;
4240
ncond->opts = cond->opts;
4241
ncond->ecllim = cond->ecllim;
4242
ncond->shadows = cond->shadows ? cbmapdup(cond->shadows) : NULL;
4243
ncond->distinct = cond->distinct ? cbmemdup(cond->distinct, -1) : NULL;
4244
ncond->mask = cond->mask;
4245
return ncond;
4246
}
4247
4248
2419
4249
/* Get the phrase of a condition object. */
2420
4250
const char *est_cond_phrase(ESTCOND *cond){
2421
4251
assert(cond);
2444
4274
}
2445
4275
2446
4276
4277
/* Get the number of skipped documents of a condition object. */
4278
int est_cond_skip(ESTCOND *cond){
4279
assert(cond);
4280
return cond->skip;
4281
}
4282
4283
2447
4284
/* Get the options of a condition object. */
2448
4285
int est_cond_options(ESTCOND *cond){
2449
4286
assert(cond);
2451
4288
}
2452
4289
2453
4290
4291
/* Get permission to adopt result of the auxiliary index. */
4292
int est_cond_auxiliary(ESTCOND *cond){
4293
assert(cond);
4294
return cond->auxmin;
4295
}
4296
4297
4298
/* Get the attribute distinction filter. */
4299
const char *est_cond_distinct(ESTCOND *cond){
4300
assert(cond);
4301
return cond->distinct;
4302
}
4303
4304
4305
/* Get the mask of targets of meta search. */
4306
int est_cond_mask(ESTCOND *cond){
4307
assert(cond);
4308
return cond->mask;
4309
}
4310
4311
2454
4312
/* Get the score of a document corresponding to a condition object. */
2455
4313
int est_cond_score(ESTCOND *cond, int index){
2456
4314
assert(cond);
2459
4317
}
2460
4318
2461
4319
4320
/* Check whether a condition object has used the auxiliary index. */
4321
int est_cond_auxiliary_word(ESTCOND *cond, const char *word){
4322
assert(cond && word);
4323
if(!cond->auxwords) return FALSE;
4324
if(word[0] != '\0') return cbmapget(cond->auxwords, word, -1, NULL) != NULL;
4325
return cbmaprnum(cond->auxwords) > 0;
4326
}
4327
4328
2462
4329
/* Get an array of ID numbers of eclipsed docuemnts of a document in a condition object. */
2463
4330
const int *est_cond_shadows(ESTCOND *cond, int id, int *np){
2464
4331
const char *vbuf;
2473
4340
}
2474
4341
2475
4342
4343
/* Set the callback function for query expansion. */
4344
void est_cond_set_expander(ESTCOND *cond, void (*func)(const char *, CBLIST *)){
4345
assert(cond && func);
4346
cond->cbxpn = func;
4347
}
4348
4349
2476
4350
/* Set the error code of a database. */
2477
4351
void est_db_set_ecode(ESTDB *db, int ecode){
2478
4352
assert(db);
2480
4354
}
2481
4355
2482
4356
4357
/* Check whether an option is set. */
4358
int est_db_check_option(ESTDB *db, int option){
4359
assert(db);
4360
switch(option){
4361
case ESTDBREADER:
4362
return !dpwritable(db->metadb);
4363
case ESTDBWRITER:
4364
return dpwritable(db->metadb);
4365
case ESTDBCREAT:
4366
return -1;
4367
case ESTDBTRUNC:
4368
return -1;
4369
case ESTDBNOLCK:
4370
return -1;
4371
case ESTDBLCKNB:
4372
return -1;
4373
case ESTDBPERFNG:
4374
return db->amode == ESTDFPERFNG;
4375
case ESTDBCHRCAT:
4376
return db->amode == ESTDFCHRCAT;
4377
case ESTDBSMALL:
4378
return -1;
4379
case ESTDBLARGE:
4380
return -1;
4381
case ESTDBHUGE:
4382
return -1;
4383
case ESTDBHUGE2:
4384
return -1;
4385
case ESTDBHUGE3:
4386
return -1;
4387
case ESTDBSCVOID:
4388
return db->smode == ESTDFSCVOID;
4389
case ESTDBSCINT:
4390
return db->smode == ESTDFSCINT;
4391
case ESTDBSCASIS:
4392
return db->smode == ESTDFSCASIS;
4393
default:
4394
break;
4395
}
4396
return -1;
4397
}
4398
4399
4400
/* Get the inode number of a database. */
4401
int est_db_inode(ESTDB *db){
4402
assert(db);
4403
return db->inode;
4404
}
4405
4406
2483
4407
/* Set the entity data of a document in a database. */
2484
4408
int est_db_set_doc_entity(ESTDB *db, int id, const char *ptr, int size){
2485
4409
int err;
2504
4428
}
2505
4429
2506
4430
4431
/* Set the maximum number of expansion of wild cards. */
4432
void est_db_set_wildmax(ESTDB *db, int num){
4433
assert(db && num >= 0);
4434
db->wildmax = num;
4435
}
4436
4437
2507
4438
/* Get the entity data of a document in a database. */
2508
4439
char *est_db_get_doc_entity(ESTDB *db, int id, int *sp){
2509
4440
char *ptr;
2562
4493
int i, wsiz, num, smax, snum, vsiz;
2563
4494
assert(doc && max >= 0);
2564
4495
if(!doc->dtexts) return cbmapopenex(1);
2565
keys = cbmapopenex(max * 1.5);
2566
words = cblistopen();
4496
keys = cbmapopenex(max * 2 + 1);
4497
CB_LISTOPEN(words);
2567
4498
for(i = -1; i < CB_LISTNUM(doc->dtexts); i++){
2568
4499
if(i < 0){
2569
4500
if(!doc->attrs || !(text = cbmapget(doc->attrs, "", 0, NULL))) continue;
2570
4501
} else {
2571
text = CB_LISTVAL(doc->dtexts, i, NULL);
4502
text = CB_LISTVAL(doc->dtexts, i);
2572
4503
}
2573
4504
if(db){
2574
4505
switch(db->amode){
2575
case ESTAMPERFNG:
2576
est_break_text_perfng(text, words, FALSE, TRUE);
4506
case ESTDFPERFNG:
4507
est_break_text_perfng(text, words, FALSE, FALSE);
4508
break;
4509
case ESTDFCHRCAT:
4510
est_break_text_chrcat(text, words, FALSE);
2577
4511
break;
2578
4512
default:
2579
est_break_text(text, words, FALSE, TRUE);
4513
est_break_text(text, words, FALSE, FALSE);
2580
4514
break;
2581
4515
}
2582
4516
} else {
2583
est_break_text(text, words, FALSE, TRUE);
4517
est_break_text(text, words, FALSE, FALSE);
2584
4518
}
2585
4519
}
2586
4520
umap = cbmapopenex(CB_LISTNUM(words) + 1);
2587
4521
for(i = 0; i < CB_LISTNUM(words); i++){
2588
word = CB_LISTVAL2(words, i, &wsiz);
4522
word = CB_LISTVAL2(words, i, wsiz);
2589
4523
if(wsiz > ESTWORDMAXLEN) continue;
2590
4524
num = (vbuf = cbmapget(umap, word, wsiz, NULL)) ? *(int *)vbuf + 1 : 1;
2591
4525
cbmapput(umap, word, wsiz, (char *)&num, sizeof(int), TRUE);
2596
4530
while((uword = (unsigned char *)cbmapiternext(umap, &wsiz)) != NULL){
2597
4531
scores[snum].word = (char *)uword;
2598
4532
scores[snum].wsiz = wsiz;
2599
scores[snum].pt = (vbuf = cbmapget(umap, (char *)uword, wsiz, NULL)) ? *(int *)vbuf : 0;
4533
scores[snum].pt = (vbuf = cbmapiterval((char *)uword, NULL)) ? *(int *)vbuf : 0;
2600
4534
if(uword[0] >= 0xe3){
2601
4535
if(wsiz <= 3){
2602
4536
scores[snum].pt /= 2;
4537
if((uword[0] == 0xe3 && (uword[1] == 0x80 || uword[1] == 0x81 ||
4538
(uword[1] == 0x82 && uword[2] <= 0x9f))) ||
4539
(uword[0] == 0xef && uword[1] >= 0xbc)) scores[snum].pt /= 2;
2603
4540
} else {
2604
if(uword[0] == 0xe3 && (uword[1] == 0x80 || uword[1] == 0x81 ||
2605
(uword[1] == 0x82 && uword[2] <= 0x9f))) scores[snum].pt /= 2;
2606
if(uword[3] == 0xe3 && (uword[4] == 0x80 || uword[4] == 0x81 ||
2607
(uword[4] == 0x82 && uword[5] <= 0x9f))) scores[snum].pt /= 2;
2608
}
2609
} else if(wsiz <= 1){
2610
scores[snum].pt /= 3;
2611
} else if(wsiz <= 2){
2612
scores[snum].pt /= 2;
4541
if((uword[0] == 0xe3 && (uword[1] == 0x80 || uword[1] == 0x81 ||
4542
(uword[1] == 0x82 && uword[2] <= 0x9f))) ||
4543
(uword[0] == 0xef && uword[1] >= 0xbc)) scores[snum].pt /= 2;
4544
if((uword[3] == 0xe3 && (uword[4] == 0x80 || uword[4] == 0x81 ||
4545
(uword[4] == 0x82 && uword[5] <= 0x9f))) ||
4546
(uword[3] == 0xef && uword[4] >= 0xbc)) scores[snum].pt /= 2;
4547
}
4548
} else if((uword[0] > '\0' && uword[0] <= '/') || (uword[0] >= ':' && uword[0] <= '@') ||
4549
(uword[0] >= '[' && uword[0] <= '`') || (uword[0] >= '{' && uword[0] <= '~')){
4550
scores[snum].pt /= 25;
4551
if(wsiz <= 1) scores[snum].pt /= 2;
4552
} else {
4553
switch(wsiz){
4554
case 1:
4555
scores[snum].pt /= 9;
4556
break;
4557
case 2:
4558
scores[snum].pt /= 5;
4559
break;
4560
case 3:
4561
scores[snum].pt /= 3;
4562
break;
4563
case 4:
4564
scores[snum].pt /= 2;
4565
break;
4566
case 5:
4567
scores[snum].pt /= 1.5;
4568
break;
4569
case 6:
4570
scores[snum].pt /= 1.25;
4571
break;
4572
}
2613
4573
}
2614
4574
snum++;
2615
4575
}
2620
4580
for(i = 0; i < snum; i++){
2621
4581
if((vbuf = cbmapget(db->keycc, scores[i].word, scores[i].wsiz, NULL)) != NULL){
2622
4582
cbmapmove(db->keycc, scores[i].word, scores[i].wsiz, FALSE);
2623
vsiz = *(int*)vbuf;
4583
vsiz = *(int *)vbuf;
2624
4584
} else {
2625
4585
if(db->dfdb){
2626
4586
if((vsiz = dpgetwb(db->dfdb, scores[i].word, scores[i].wsiz,
2636
4596
}
2637
4597
cbmapput(db->keycc, scores[i].word, scores[i].wsiz, (char *)&vsiz, sizeof(int), FALSE);
2638
4598
}
2639
scores[i].pt *= 400000.0 / (vsiz + 64);
4599
scores[i].pt *= 100000.0 / pow(vsiz + 64, 0.6);
2640
4600
}
2641
4601
if(db->kcmnum >= 0 && cbmaprnum(db->keycc) > db->kcmnum){
2642
4602
num = db->kcmnum * 0.1 + 1;
2647
4607
}
2648
4608
qsort(scores, snum, sizeof(ESTKEYSC), est_keysc_compare);
2649
4609
}
2650
for(i = 0; i < snum && i < max; i++){
2651
vsiz = sprintf(numbuf, "%d", scores[i].pt);
4610
for(i = 0; i < snum && i < max; i++){
4611
vsiz = sprintf(numbuf, "%d", scores[i].pt > 0 ? scores[i].pt : 1);
2652
4612
cbmapput(keys, scores[i].word, scores[i].wsiz, numbuf, vsiz, FALSE);
2653
4613
}
2654
4614
free(scores);
2655
4615
cbmapclose(umap);
2656
cblistclose(words);
4616
CB_LISTCLOSE(words);
2657
4617
return keys;
2658
4618
}
2659
4619
2660
4620
2661
4621
/* Retrieve a map object of keywords. */
2662
int est_db_put_keywords(ESTDB *db, int id, CBMAP *kwords){
4622
int est_db_put_keywords(ESTDB *db, int id, CBMAP *kwords, double weight){
4623
const char *kbuf;
2663
4624
char *mbuf;
2664
int msiz, err;
2665
assert(db && id > 0 && kwords);
4625
int err, ksiz, pair[2], msiz;
4626
assert(db && id > 0 && kwords && weight >= 0.0);
2666
4627
if(!dpwritable(db->metadb)){
2667
4628
db->ecode = ESTEACCES;
2668
4629
return FALSE;
2669
4630
}
4631
if(crvsiz(db->attrdb, (char *)&id, sizeof(int)) == -1){
4632
db->ecode = ESTENOITEM;
4633
return FALSE;
4634
}
4635
err = FALSE;
4636
if(!est_db_out_keywords(db, id) && db->ecode != ESTENOITEM) err = TRUE;
4637
pair[0] = id;
4638
cbmapiterinit(kwords);
4639
while((kbuf = cbmapiternext(kwords, &ksiz)) != NULL){
4640
if(ksiz < 1 || (kbuf[0] >= '\0' && kbuf[0] <= ' ')) continue;
4641
pair[1] = (int)(atoi(cbmapiterval(kbuf, NULL)) * weight);
4642
cbmapputcat(db->auxcc, kbuf, ksiz, (char *)pair, sizeof(pair));
4643
db->icsiz += sizeof(pair);
4644
}
2670
4645
mbuf = cbmapdump(kwords, &msiz);
2671
err = FALSE;
2672
if(!crput(db->kwddb, (char *)&id, sizeof(int), mbuf, msiz, CR_DOVER)){
4646
if(!est_crput(db->kwddb, db->zmode, id, mbuf, msiz, CR_DOVER)){
2673
4647
db->ecode = ESTEDB;
2674
4648
db->fatal = TRUE;
2675
4649
err = TRUE;
2679
4653
}
2680
4654
2681
4655
4656
/* Remove keywords of a document. */
4657
int est_db_out_keywords(ESTDB *db, int id){
4658
CBMAP *kwords;
4659
const char *word;
4660
char wbuf[ESTWORDMAXLEN+3], *tbuf;
4661
int wsiz;
4662
assert(db && id > 0);
4663
if(!dpwritable(db->metadb)){
4664
db->ecode = ESTEACCES;
4665
return FALSE;
4666
}
4667
if(id >= ESTPDOCIDMIN){
4668
db->ecode = ESTEINVAL;
4669
return FALSE;
4670
}
4671
if((kwords = est_db_get_keywords(db, id)) != NULL){
4672
cbmapiterinit(kwords);
4673
while((word = cbmapiternext(kwords, &wsiz)) != NULL){
4674
if(wsiz > ESTWORDMAXLEN){
4675
tbuf = cbsprintf(" %s", word);
4676
cbmapput(db->outcc, tbuf, wsiz + 1, "", 0, FALSE);
4677
free(tbuf);
4678
} else {
4679
sprintf(wbuf, " %s", word);
4680
cbmapput(db->outcc, wbuf, wsiz + 1, "", 0, FALSE);
4681
}
4682
}
4683
cbmapclose(kwords);
4684
}
4685
if(!est_crout(db->kwddb, id)){
4686
if(dpecode == DP_ENOITEM){
4687
db->ecode = ESTENOITEM;
4688
} else {
4689
db->ecode = ESTEDB;
4690
db->fatal = TRUE;
4691
}
4692
return FALSE;
4693
}
4694
cbmapout(db->veccc, (char *)&id, sizeof(int));
4695
return TRUE;
4696
}
4697
4698
2682
4699
/* Retrieve a map object of keywords. */
2683
4700
CBMAP *est_db_get_keywords(ESTDB *db, int id){
2684
4701
CBMAP *kwords;
2690
4707
cbmapmove(db->veccc, (char *)&id, sizeof(int), FALSE);
2691
4708
return cbmapload(cbuf, csiz);
2692
4709
}
2693
if(!(mbuf = crget(db->kwddb, (char *)&id, sizeof(int), 0, -1, &msiz))) return NULL;
4710
if(!(mbuf = est_crget(db->kwddb, db->zmode, id, &msiz))){
4711
db->ecode = ESTENOITEM;
4712
return NULL;
4713
}
2694
4714
kwords = cbmapload(mbuf, msiz);
2695
cbmapputvbuf(db->veccc, (char *)&id, sizeof(int), mbuf, msiz);
4715
if(db->vcmnum > 0) cbmapput(db->veccc, (char *)&id, sizeof(int), mbuf, msiz, TRUE);
4716
free(mbuf);
2696
4717
if(cbmaprnum(db->veccc) > db->vcmnum){
2697
4718
num = cbmaprnum(db->veccc) * 0.1 + 1;
2698
4719
cbmapiterinit(db->veccc);
2757
4778
/* Initialize the word iterator of a database. */
2758
4779
int est_db_word_iter_init(ESTDB *db){
2759
4780
assert(db);
2760
return vlcurfirst(db->fwmdb) || dpecode == DP_ENOITEM;
4781
if(!vlcurfirst(db->fwmdb) && dpecode != DP_ENOITEM){
4782
db->ecode = ESTEDB;
4783
db->fatal = TRUE;
4784
return FALSE;
4785
}
4786
return TRUE;
2761
4787
}
2762
4788
2763
4789
2765
4791
char *est_db_word_iter_next(ESTDB *db){
2766
4792
char *word;
2767
4793
assert(db);
2768
if(!(word = vlcurkey(db->fwmdb, NULL))) return NULL;
4794
if(!(word = vlcurkey(db->fwmdb, NULL))){
4795
if(dpecode == DP_ENOITEM){
4796
db->ecode = ESTENOITEM;
4797
} else {
4798
db->ecode = ESTEDB;
4799
db->fatal = TRUE;
4800
}
4801
return NULL;
4802
}
2769
4803
vlcurnext(db->fwmdb);
2770
4804
return word;
2771
4805
}
2773
4807
2774
4808
/* Get the size of the record of a word. */
2775
4809
int est_db_word_rec_size(ESTDB *db, const char *word){
2776
assert(db && word);
2777
return est_idx_vsiz(db->idxdb, word, strlen(word));
4810
int num;
4811
assert(db && word);
4812
if(!cbmapget(db->idxcc, word, -1, &num)) num = 0;
4813
return est_idx_vsiz(db->idxdb, word, strlen(word)) + num;
4814
}
4815
4816
4817
/* Get the number of unique keywords in a database. */
4818
int est_db_keyword_num(ESTDB *db){
4819
int wnum;
4820
assert(db);
4821
wnum = vlrnum(db->xfmdb);
4822
return wnum > 0 ? wnum : 0;
4823
}
4824
4825
4826
/* Initialize the keyword iterator of a database. */
4827
int est_db_keyword_iter_init(ESTDB *db){
4828
assert(db);
4829
if(!vlcurfirst(db->xfmdb) && dpecode != DP_ENOITEM){
4830
db->ecode = ESTEDB;
4831
db->fatal = TRUE;
4832
return FALSE;
4833
}
4834
return TRUE;
4835
}
4836
4837
4838
/* Get the next keyword of the word iterator of a database. */
4839
char *est_db_keyword_iter_next(ESTDB *db){
4840
char *word;
4841
assert(db);
4842
if(!(word = vlcurkey(db->xfmdb, NULL))){
4843
if(dpecode == DP_ENOITEM){
4844
db->ecode = ESTENOITEM;
4845
} else {
4846
db->ecode = ESTEDB;
4847
db->fatal = TRUE;
4848
}
4849
return NULL;
4850
}
4851
vlcurnext(db->xfmdb);
4852
return word;
4853
}
4854
4855
4856
/* Get the size of the record of a keyword. */
4857
int est_db_keyword_rec_size(ESTDB *db, const char *word){
4858
const char *kbuf;
4859
assert(db && word);
4860
return (kbuf = vlgetcache(db->xfmdb, word, -1, NULL)) != NULL ? atoi(kbuf) : 0;
4861
}
4862
4863
4864
/* Search documents corresponding a keyword for a database. */
4865
int *est_db_keyword_search(ESTDB *db, const char *word, int *nump){
4866
int i, *res, rnum;
4867
assert(db && word && nump);
4868
if(!(res = (int *)vlget(db->auxdb, word, -1, &rnum))){
4869
*nump = 0;
4870
CB_MALLOC(res, 1);
4871
return res;
4872
}
4873
rnum /= sizeof(int) * 2;
4874
for(i = 0; i < rnum; i++){
4875
res[i] = res[i*2];
4876
}
4877
*nump = rnum;
4878
return res;
2778
4879
}
2779
4880
2780
4881
2788
4889
/* Get the size of used cache region. */
2789
4890
int est_db_used_cache_size(ESTDB *db){
2790
4891
assert(db);
2791
return (db->icsiz + cbmaprnum(db->idxcc) * (sizeof(CBMAPDATUM) + ESTWORDAVGLEN)) * ESTMEMIRATIO;
4892
return (db->icsiz + (cbmaprnum(db->idxcc) + cbmaprnum(db->auxcc)) *
4893
(sizeof(CBMAPDATUM) + ESTWORDAVGLEN)) * ESTMEMIRATIO;
2792
4894
}
2793
4895
2794
4896
2806
4908
2807
4909
2808
4910
/* Set the callback function for database events. */
2809
void est_db_set_informer(ESTDB *db, void (*func)(const char *)){
4911
void est_db_set_informer(ESTDB *db, void (*func)(const char *, void *), void *opaque){
2810
4912
assert(db && func);
2811
db->cbinfo = func;
4913
db->infocb = func;
4914
db->infoop = opaque;
2812
4915
est_db_inform(db, "status");
2813
4916
}
2814
4917
2815
4918
2816
4919
/* Fill the cache for keys for TF-IDF. */
2817
4920
void est_db_fill_key_cache(ESTDB *db){
2818
char *kbuf, *msg;
4921
const char *kbuf;
4922
char *msg;
2819
4923
int i, ksiz, vsiz;
2820
4924
assert(db);
2821
4925
vlcurfirst(db->fwmdb);
2822
for(i = 0; (kbuf = vlcurkey(db->fwmdb, &ksiz)) != NULL; i++){
4926
for(i = 0; (kbuf = vlcurkeycache(db->fwmdb, &ksiz)) != NULL; i++){
2823
4927
vsiz = est_idx_vsiz(db->idxdb, kbuf, ksiz);
2824
4928
cbmapput(db->keycc, kbuf, ksiz, (char *)&vsiz, sizeof(int), TRUE);
2825
free(kbuf);
2826
4929
vlcurnext(db->fwmdb);
2827
4930
if(i % ESTCCCBFREQ == 0){
2828
4931
msg = cbsprintf("filling the key cache for TF-IDF (%d)", i + 1);
2864
4967
int i, num, size, vsiz;
2865
4968
assert(db);
2866
4969
if(max < 0) max = INT_MAX;
2867
words = cblistopen();
4970
CB_LISTOPEN(words);
2868
4971
cbmapiterinit(db->rescc);
2869
4972
while((word = cbmapiternext(db->rescc, &size)) != NULL){
2870
vbuf = cbmapget(db->rescc, word, size, &vsiz);
2871
if(vsiz == sizeof(ESTSCORE) && ((ESTSCORE *)vbuf)->id == -1) cblistpush(words, word, size);
4973
CB_MAPITERVAL(vbuf, word, vsiz);
4974
if(vsiz == sizeof(ESTSCORE) && ((ESTSCORE *)vbuf)->id == -1) CB_LISTPUSH(words, word, size);
2872
4975
}
2873
4976
num = CB_LISTNUM(words);
2874
4977
for(i = 0; i < max && i < num; i++){
2875
4978
word = cblistval(words, num - i - 1, &size);
2876
free(est_search_union(db, word, 1, &size, NULL, TRUE));
4979
free(est_search_union(db, word, 1, NULL, &size, NULL, TRUE, -1, NULL));
2877
4980
}
2878
cblistclose(words);
4981
CB_LISTCLOSE(words);
2879
4982
}
2880
4983
2881
4984
2882
/* Get list of words in the result cache. */
4985
/* Get a list of words in the result cache. */
2883
4986
CBLIST *est_db_list_rescc(ESTDB *db){
2884
4987
CBLIST *words;
2885
4988
const char *word;
2886
4989
int size;
2887
4990
assert(db);
2888
words = cblistopen();
4991
CB_LISTOPEN(words);
2889
4992
cbmapiterinit(db->rescc);
2890
4993
while((word = cbmapiternext(db->rescc, &size)) != NULL){
2891
4994
cblistunshift(words, word, size);
2894
4997
}
2895
4998
2896
4999
5000
/* Get the number of pseudo documents in a database. */
5001
int est_db_pseudo_doc_num(ESTDB *db){
5002
assert(db);
5003
return cblistnum(db->pdocs);
5004
}
5005
5006
5007
/* Get a list of expressions of attribute indexes of a database. */
5008
CBLIST *est_db_attr_index_exprs(ESTDB *db){
5009
ESTATTRIDX *attridx;
5010
CBLIST *list;
5011
const char *kbuf;
5012
char *expr;
5013
assert(db);
5014
list = cblistopen();
5015
cbmapiterinit(db->aidxs);
5016
while((kbuf = cbmapiternext(db->aidxs, NULL)) != NULL){
5017
attridx = (ESTATTRIDX *)cbmapiterval(kbuf, NULL);
5018
switch(attridx->type){
5019
case ESTIDXATTRSTR:
5020
expr = cbsprintf("%s=str", kbuf);
5021
break;
5022
case ESTIDXATTRNUM:
5023
expr = cbsprintf("%s=num", kbuf);
5024
break;
5025
default:
5026
expr = cbsprintf("%s=seq", kbuf);
5027
break;
5028
}
5029
CB_LISTPUSHBUF(list, expr, strlen(expr));
5030
}
5031
return list;
5032
}
5033
5034
2897
5035
/* Interrupt long time processing. */
2898
5036
void est_db_interrupt(ESTDB *db){
2899
5037
assert(db);
2901
5039
}
2902
5040
2903
5041
5042
/* Repair a broken database directory. */
5043
int est_db_repair(const char *name, int options, int *ecp){
5044
ESTDB *db;
5045
DEPOT *depot, *metadb;
5046
CURIA *curia, *attrdb, *textdb, *kwddb;
5047
VILLA *villa, *listdb;
5048
CBLIST *list;
5049
CBMAP *aidxs, *attrs;
5050
ESTATTRIDX attridx, *attridxp;
5051
void *aidxdb;
5052
const char *elem, *abuf;
5053
char path[ESTPATHBUFSIZ], *kbuf, vbuf[ESTNUMBUFSIZ], *dec, *mbuf;
5054
int i, err, idmax, flags, zmode, dnum, dseq, ksiz, vsiz, type, id, msiz, esiz, asiz;
5055
assert(name && ecp);
5056
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTMETADBNAME);
5057
if(est_inode(path) == -1){
5058
*ecp = ESTEIO;
5059
return FALSE;
5060
}
5061
if(!(options & ESTRPSTRICT) && (depot= dpopen(path, DP_OWRITER, -1)) != NULL){
5062
dpclose(depot);
5063
} else {
5064
dprepair(path);
5065
}
5066
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTIDXDBNAME);
5067
if(est_inode(path) == -1){
5068
*ecp = ESTEIO;
5069
return FALSE;
5070
}
5071
if((list = cbdirlist(path)) != NULL){
5072
for(i = 1; i < CB_LISTNUM(list); i++){
5073
elem = CB_LISTVAL(list, i);
5074
if(elem[0] < '0' || elem[0] > '9') continue;
5075
sprintf(path, "%s%c%s%c%s", name, ESTPATHCHR, ESTIDXDBNAME, ESTPATHCHR, elem);
5076
if(!(options & ESTRPSTRICT) && (villa = vlopen(path, VL_OWRITER, VL_CMPLEX)) != NULL){
5077
vlclose(villa);
5078
} else {
5079
vlrepair(path, VL_CMPLEX);
5080
}
5081
}
5082
CB_LISTCLOSE(list);
5083
}
5084
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTFWMDBNAME);
5085
if(est_inode(path) == -1){
5086
*ecp = ESTEIO;
5087
return FALSE;
5088
}
5089
if(!(options & ESTRPSTRICT) && (villa = vlopen(path, VL_OWRITER, VL_CMPLEX)) != NULL){
5090
vlclose(villa);
5091
} else {
5092
vlrepair(path, VL_CMPLEX);
5093
}
5094
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTAUXDBNAME);
5095
if(est_inode(path) == -1){
5096
*ecp = ESTEIO;
5097
return FALSE;
5098
}
5099
if(!(options & ESTRPSTRICT) && (villa = vlopen(path, VL_OWRITER, VL_CMPLEX)) != NULL){
5100
vlclose(villa);
5101
} else {
5102
vlrepair(path, VL_CMPLEX);
5103
}
5104
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTXFMDBNAME);
5105
if(est_inode(path) == -1){
5106
*ecp = ESTEIO;
5107
return FALSE;
5108
}
5109
if(!(options & ESTRPSTRICT) && (villa = vlopen(path, VL_OWRITER, VL_CMPLEX)) != NULL){
5110
vlclose(villa);
5111
} else {
5112
vlrepair(path, VL_CMPLEX);
5113
}
5114
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTATTRDBNAME);
5115
if(est_inode(path) == -1){
5116
*ecp = ESTEIO;
5117
return FALSE;
5118
}
5119
if(!(options & ESTRPSTRICT) && (curia = cropen(path, CR_OWRITER, -1, -1)) != NULL){
5120
crclose(curia);
5121
} else {
5122
crrepair(path);
5123
}
5124
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTTEXTDBNAME);
5125
if(est_inode(path) == -1){
5126
*ecp = ESTEIO;
5127
return FALSE;
5128
}
5129
if(!(options & ESTRPSTRICT) && (curia = cropen(path, CR_OWRITER, -1, -1)) != NULL){
5130
crclose(curia);
5131
} else {
5132
crrepair(path);
5133
}
5134
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTKWDDBNAME);
5135
if(est_inode(path) == -1){
5136
*ecp = ESTEIO;
5137
return FALSE;
5138
}
5139
if(!(options & ESTRPSTRICT) && (curia = cropen(path, CR_OWRITER, -1, -1)) != NULL){
5140
crclose(curia);
5141
} else {
5142
crrepair(path);
5143
}
5144
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTLISTDBNAME);
5145
if(est_inode(path) == -1){
5146
*ecp = ESTEIO;
5147
return FALSE;
5148
}
5149
if(!(options & ESTRPSTRICT) && (villa = vlopen(path, VL_OWRITER, VL_CMPLEX)) != NULL){
5150
vlclose(villa);
5151
} else {
5152
vlrepair(path, VL_CMPLEX);
5153
}
5154
if((list = cbdirlist(name)) != NULL){
5155
for(i = 0; i < CB_LISTNUM(list); i++){
5156
elem = CB_LISTVAL(list, i);
5157
if(cbstrfwmatch(elem, ESTAISEQPREF)){
5158
sprintf(path, "%s%c%s", name, ESTPATHCHR, elem);
5159
if(!(options & ESTRPSTRICT) && (depot = dpopen(path, DP_OWRITER, -1)) != NULL){
5160
dpclose(depot);
5161
} else {
5162
dprepair(path);
5163
}
5164
} else if(cbstrfwmatch(elem, ESTAISTRPREF)){
5165
sprintf(path, "%s%c%s", name, ESTPATHCHR, elem);
5166
if(!(options & ESTRPSTRICT) && (villa = vlopen(path, VL_OWRITER, VL_CMPLEX)) != NULL){
5167
vlclose(villa);
5168
} else {
5169
vlrepair(path, VL_CMPLEX);
5170
}
5171
} else if(cbstrfwmatch(elem, ESTAINUMPREF)){
5172
sprintf(path, "%s%c%s", name, ESTPATHCHR, elem);
5173
if(!(options & ESTRPSTRICT) && (villa = vlopen(path, VL_OWRITER, VL_CMPLEX)) != NULL){
5174
vlclose(villa);
5175
} else {
5176
vlrepair(path, VL_CMPLEX);
5177
}
5178
}
5179
}
5180
CB_LISTCLOSE(list);
5181
}
5182
if((options & ESTRPSHODDY) && (db = est_db_open(name, ESTDBWRITER, ecp)) != NULL){
5183
if(!est_db_close(db, ecp)) return FALSE;
5184
return TRUE;
5185
}
5186
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTMETADBNAME);
5187
metadb = dpopen(path, DP_OWRITER, -1);
5188
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTATTRDBNAME);
5189
attrdb = cropen(path, CR_OWRITER, -1, -1);
5190
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTTEXTDBNAME);
5191
textdb = cropen(path, CR_OWRITER, -1, -1);
5192
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTKWDDBNAME);
5193
kwddb = cropen(path, CR_OWRITER, -1, -1);
5194
sprintf(path, "%s%c%s", name, ESTPATHCHR, ESTLISTDBNAME);
5195
listdb = vlopen(path, VL_OWRITER, VL_CMPLEX);
5196
if(!attrdb || !textdb || !kwddb || !listdb){
5197
if(listdb) vlclose(listdb);
5198
if(kwddb) crclose(kwddb);
5199
if(textdb) crclose(textdb);
5200
if(attrdb) crclose(attrdb);
5201
if(metadb) dpclose(metadb);
5202
*ecp = ESTEDB;
5203
return FALSE;
5204
}
5205
aidxs = cbmapopenex(ESTMINIBNUM);
5206
if((list = cbdirlist(name)) != NULL){
5207
for(i = 0; i < CB_LISTNUM(list); i++){
5208
elem = CB_LISTVAL(list, i);
5209
dec = NULL;
5210
type = -1;
5211
if(cbstrfwmatch(elem, ESTAISEQPREF)){
5212
dec = est_hex_decode(elem + strlen(ESTAISEQPREF));
5213
type = ESTIDXATTRSEQ;
5214
} else if(cbstrfwmatch(elem, ESTAISTRPREF)){
5215
dec = est_hex_decode(elem + strlen(ESTAISTRPREF));
5216
type = ESTIDXATTRSTR;
5217
} else if(cbstrfwmatch(elem, ESTAINUMPREF)){
5218
dec = est_hex_decode(elem + strlen(ESTAINUMPREF));
5219
type = ESTIDXATTRNUM;
5220
}
5221
if(dec){
5222
sprintf(path, "%s%c%s", name, ESTPATHCHR, elem);
5223
switch(type){
5224
case ESTIDXATTRSTR:
5225
if((aidxdb = vlopen(path, VL_OWRITER, VL_CMPLEX)) != NULL){
5226
vlsettuning(aidxdb, ESTAIDXLRM, ESTAIDXNIM, ESTAIDXLCN, ESTAIDXNCN);
5227
vlsetfbpsiz(aidxdb, ESTAIDXVLFBP);
5228
attridx.db = aidxdb;
5229
attridx.type = type;
5230
cbmapput(aidxs, dec, -1, (char *)&attridx, sizeof(ESTATTRIDX), FALSE);
5231
}
5232
break;
5233
case ESTIDXATTRNUM:
5234
if((aidxdb = vlopen(path, VL_OWRITER, est_aidx_numcmp)) != NULL){
5235
vlsettuning(aidxdb, ESTAIDXLRM, ESTAIDXNIM, ESTAIDXLCN, ESTAIDXNCN);
5236
vlsetfbpsiz(aidxdb, ESTAIDXVLFBP);
5237
attridx.db = aidxdb;
5238
attridx.type = type;
5239
cbmapput(aidxs, dec, -1, (char *)&attridx, sizeof(ESTATTRIDX), FALSE);
5240
}
5241
break;
5242
default:
5243
if((aidxdb = dpopen(path, DP_OWRITER, crbnum(attrdb) / ESTAIBDIAM)) != NULL){
5244
dpsetfbpsiz(aidxdb, ESTAIDXDPFBP);
5245
attridx.db = aidxdb;
5246
attridx.type = type;
5247
cbmapput(aidxs, dec, -1, (char *)&attridx, sizeof(ESTATTRIDX), FALSE);
5248
}
5249
break;
5250
}
5251
free(dec);
5252
}
5253
}
5254
CB_LISTCLOSE(list);
5255
}
5256
err = FALSE;
5257
idmax = 0;
5258
if((vsiz = dpgetwb(metadb, ESTKEYDNUM, -1, 0, ESTNUMBUFSIZ - 1, vbuf)) > 0){
5259
vbuf[vsiz] = '\0';
5260
idmax = atoi(vbuf);
5261
}
5262
flags = dpgetflags(metadb);
5263
zmode = 0;
5264
if(flags & ESTDFZLIB){
5265
zmode = ESTDFZLIB;
5266
} else if(flags & ESTDFLZO){
5267
zmode = ESTDFLZO;
5268
} else if(flags & ESTDFBZIP){
5269
zmode = ESTDFBZIP;
5270
}
5271
dnum = 0;
5272
dseq = 0;
5273
CB_LISTOPEN(list);
5274
if(!criterinit(attrdb)) err = TRUE;
5275
while((kbuf = criternext(attrdb, &ksiz)) != NULL){
5276
if(ksiz == sizeof(int) && (id = *(int *)kbuf) > 0 && id <= idmax &&
5277
crvsiz(attrdb, kbuf, ksiz) > 0 && crvsiz(textdb, kbuf, ksiz) > 0){
5278
dnum++;
5279
if(dseq < id) dseq = id;
5280
if(options & ESTRPSTRICT){
5281
if((mbuf = est_crget(attrdb, zmode, id, &msiz)) != NULL){
5282
attrs = cbmapload(mbuf, msiz);
5283
if((elem = cbmapget(attrs, ESTDATTRURI, -1, &esiz)) != NULL){
5284
vsiz = sprintf(vbuf, "%d", id);
5285
vlput(listdb, elem, esiz, vbuf, vsiz, VL_DKEEP);
5286
}
5287
if(cbmaprnum(aidxs) > 0){
5288
cbmapiterinit(aidxs);
5289
while((abuf = cbmapiternext(aidxs, &asiz)) != NULL){
5290
if(!(elem = cbmapget(attrs, abuf, asiz, &esiz))) continue;
5291
attridxp = (ESTATTRIDX *)cbmapiterval(abuf, NULL);
5292
switch(attridxp->type){
5293
case ESTIDXATTRSTR:
5294
case ESTIDXATTRNUM:
5295
est_aidx_attr_put(attridxp->db, id, elem, esiz);
5296
break;
5297
default:
5298
est_aidx_seq_put(attridxp->db, id, elem, esiz);
5299
break;
5300
}
5301
}
5302
}
5303
cbmapclose(attrs);
5304
free(mbuf);
5305
}
5306
}
5307
} else {
5308
CB_LISTPUSH(list, kbuf, ksiz);
5309
}
5310
free(kbuf);
5311
}
5312
if(dpecode != DP_ENOITEM) err = TRUE;
5313
for(i = 0; i < CB_LISTNUM(list); i++){
5314
elem = CB_LISTVAL2(list, i, esiz);
5315
crout(attrdb, elem, esiz);
5316
crout(textdb, elem, esiz);
5317
crout(kwddb, elem, esiz);
5318
}
5319
CB_LISTCLOSE(list);
5320
sprintf(vbuf, "%d", dseq);
5321
if(!dpput(metadb, ESTKEYDSEQ, -1, vbuf, -1, DP_DOVER)) err = TRUE;
5322
sprintf(vbuf, "%d", dnum);
5323
if(!dpput(metadb, ESTKEYDNUM, -1, vbuf, -1, DP_DOVER)) err = TRUE;
5324
cbmapiterinit(aidxs);
5325
while((elem = cbmapiternext(aidxs, NULL)) != NULL){
5326
attridxp = (ESTATTRIDX *)cbmapiterval(elem, NULL);
5327
switch(attridxp->type){
5328
case ESTIDXATTRSTR:
5329
case ESTIDXATTRNUM:
5330
if(!vlclose(attridxp->db)) err = TRUE;
5331
break;
5332
default:
5333
if(!dpclose(attridxp->db)) err = TRUE;
5334
break;
5335
}
5336
}
5337
cbmapclose(aidxs);
5338
if(!vlclose(listdb)) err = TRUE;
5339
if(!crclose(kwddb)) err = TRUE;
5340
if(!crclose(textdb)) err = TRUE;
5341
if(!crclose(attrdb)) err = TRUE;
5342
if(!dpclose(metadb)) err = TRUE;
5343
if(err){
5344
*ecp = ESTEDB;
5345
return FALSE;
5346
}
5347
return err ? FALSE : TRUE;
5348
}
5349
5350
2904
5351
/* Extract words for snippet from hints of search. */
2905
5352
CBLIST *est_hints_to_words(CBMAP *hints){
2906
5353
CBLIST *words;
2907
5354
const char *kbuf;
2908
5355
int ksiz;
2909
5356
assert(hints);
2910
words = cblistopen();
5357
CB_LISTOPEN(words);
2911
5358
cbmapiterinit(hints);
2912
5359
while((kbuf = cbmapiternext(hints, &ksiz)) != NULL){
2913
if(ksiz < 1 || atoi(cbmapget(hints, kbuf, ksiz, NULL)) < 1) continue;
2914
cblistpush(words, kbuf, ksiz);
5360
if(ksiz < 1 || atoi(cbmapget(hints, kbuf, ksiz, NULL)) < 0) continue;
5361
CB_LISTPUSH(words, kbuf, ksiz);
2915
5362
}
2916
5363
return words;
2917
5364
}
2918
5365
2919
5366
5367
/* Reset the environment of the process. */
5368
void est_proc_env_reset(void){
5369
char *value, *pbuf;
5370
cbstdiobin();
5371
putenv("LANG=C");
5372
putenv("LANGUAGE=C");
5373
putenv("LC_CTYPE=C");
5374
putenv("LC_COLLATE=C");
5375
putenv("LC_TIME=C");
5376
putenv("LC_NUMERIC=C");
5377
putenv("LC_MONETARY=C");
5378
putenv("LC_ALL=C");
5379
putenv("EST_VERSION=" _EST_VERSION);
5380
if((value = getenv("PATH")) != NULL){
5381
if(ESTPATHCHR == '\\'){
5382
pbuf = cbsprintf("PATH=%s;C:\\hyperestraier;D:\\hyperestraier;E:\\hyperestraier", value);
5383
} else {
5384
pbuf = cbsprintf("PATH=%s:/bin:/usr/bin:/usr/local/bin", value);
5385
}
5386
putenv(pbuf);
5387
cbglobalgc(pbuf, free);
5388
}
5389
}
5390
5391
2920
5392
/* Make a directory. */
2921
5393
int est_mkdir(const char *path){
2922
5394
#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_)
2937
5409
assert(path);
2938
5410
if((files = cbdirlist(path)) != NULL){
2939
5411
for(i = 0; i < CB_LISTNUM(files); i++){
2940
file = CB_LISTVAL(files, i, NULL);
5412
file = CB_LISTVAL(files, i);
2941
5413
if(!strcmp(file, ESTCDIRSTR) || !strcmp(file, ESTPDIRSTR)) continue;
2942
5414
sprintf(pbuf, "%s%c%s", path, ESTPATHCHR, file);
2943
5415
if(unlink(pbuf) == -1) est_rmdir_rec(pbuf);
2944
5416
}
2945
cblistclose(files);
5417
CB_LISTCLOSE(files);
2946
5418
}
2947
5419
return rmdir(path) == 0;
2948
5420
}
2977
5449
}
2978
5450
2979
5451
5452
/* Get the inode number of a file. */
5453
int est_inode(const char *path){
5454
#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_)
5455
char pbuf[ESTPATHBUFSIZ*2], *p;
5456
int inode;
5457
struct stat sbuf;
5458
if(stat(path, &sbuf) == -1) return -1;
5459
if(GetFullPathName(path, ESTPATHBUFSIZ*2, pbuf, &p) != 0){
5460
inode = 11003;
5461
for(p = pbuf; *p != '\0'; p++){
5462
inode = inode * 31 + *(unsigned char *)p;
5463
}
5464
return (inode * 911) & 0x7FFF;
5465
}
5466
return -1;
5467
#else
5468
struct stat sbuf;
5469
if(stat(path, &sbuf) == -1) return -1;
5470
return sbuf.st_ino;
5471
#endif
5472
}
5473
5474
2980
5475
/* Get the time of day in milliseconds. */
2981
5476
double est_gettimeofday(void){
2982
5477
#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_)
3070
5565
}
3071
5566
3072
5567
5568
/* Get the load ratio of the physical memory. */
5569
double est_memory_usage(void){
5570
#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) || defined(_SYS_CYGWIN_)
5571
MEMORYSTATUS sbuf;
5572
sbuf.dwLength = sizeof(MEMORYSTATUS);
5573
GlobalMemoryStatus(&sbuf);
5574
return sbuf.dwMemoryLoad / 100.0;
5575
#else
5576
return 0.0;
5577
#endif
5578
}
5579
5580
3073
5581
/* get the media type of an extention */
3074
5582
const char *est_ext_type(const char *ext){
3075
5583
static const char *list[] = {
3129
5637
/* Set a seed vector from a map object. */
3130
5638
void est_vector_set_seed(CBMAP *svmap, int *svec, int vnum){
3131
5639
const char *kbuf;
3132
int i, ksiz;
5640
int nnum, ksiz;
3133
5641
assert(svmap && svec && vnum > 0);
3134
5642
cbmapiterinit(svmap);
3135
for(i = 0; i < vnum; i++){
5643
nnum = 0;
5644
while(nnum < vnum){
3136
5645
if((kbuf = cbmapiternext(svmap, &ksiz)) != NULL){
3137
svec[i] = atoi(cbmapget(svmap, kbuf, ksiz, NULL));
5646
if(ksiz < 1) continue;
5647
svec[nnum++] = atoi(cbmapiterval(kbuf, NULL));
3138
5648
} else {
3139
svec[i] = 0;
5649
svec[nnum++] = 0;
3140
5650
}
3141
5651
}
3142
5652
}
3189
5699
*************************************************************************************************/
3190
5700
3191
5701
5702
/* Encode a string into hexadecimal.
5703
`str' specifies a string.
5704
The return value is the result hexadecimal string. */
5705
static char *est_hex_encode(const char *str){
5706
char *res, *wp;
5707
assert(str);
5708
CB_MALLOC(res, strlen(str) * 2 + 1);
5709
wp = res;
5710
while(*str != '\0'){
5711
wp += sprintf(wp, "%02X", *(unsigned char *)str);
5712
str++;
5713
}
5714
*wp = '\0';
5715
return res;
5716
}
5717
5718
5719
/* Decode a hexadecimal string into original one.
5720
`str' specifies a hexadecimal string.
5721
The return value is the original string. */
5722
static char *est_hex_decode(const char *str){
5723
char *res, *wp;
5724
int i, len;
5725
assert(str);
5726
len = strlen(str);
5727
CB_MALLOC(res, len + 1);
5728
wp = res;
5729
for(i = 0; i < len; i += 2){
5730
*(wp++) = (str[i] >= 'A' ? str[i] - 'A' + 10 : str[i] - '0') * 16 +
5731
(str[i+1] >= 'A' ? str[i+1] - 'A' + 10 : str[i+1] - '0');
5732
}
5733
*wp = '\0';
5734
return res;
5735
}
5736
5737
3192
5738
/* Count the number of missing characters when converting.
3193
5739
`ptr' specifies the pointer to a region.
3194
5740
`size' specifies the size of the region.
3235
5781
for(i = 0; i < size - 1; i += 2){
3236
5782
b1 = utext[i];
3237
5783
b2 = utext[i+1];
3238
if(b1 == 0x0 && (b2 <= 0x8 || (b2 >= 0x0e && b2 <= 0x1f))){
3239
/* control characters */
3240
utext[wi] = 0x0;
3241
utext[wi+1] = 0x20;
3242
} else if(b1 == 0x0 && b2 == 0xa0){
3243
/* no-break space */
3244
utext[wi] = 0x0;
3245
utext[wi+1] = 0x20;
3246
} else if(b1 == 0x20 && b2 == 0x2){
3247
/* en space */
3248
utext[wi] = 0x0;
3249
utext[wi+1] = 0x20;
3250
} else if(b1 == 0x20 && b2 == 0x3){
3251
/* em space */
3252
utext[wi] = 0x0;
3253
utext[wi+1] = 0x20;
3254
} else if(b1 == 0x20 && b2 == 0x9){
3255
/* thin space */
3256
utext[wi] = 0x0;
3257
utext[wi+1] = 0x20;
3258
} else if(b1 == 0x30 && b2 == 0x0){
3259
/* fullwidth space */
3260
utext[wi] = 0x0;
3261
utext[wi+1] = 0x20;
5784
if(b1 == 0x0){
5785
if(b2 <= 0x8 || (b2 >= 0x0e && b2 <= 0x1f)){
5786
/* control characters */
5787
utext[wi++] = 0x0;
5788
utext[wi++] = 0x20;
5789
} else if(b2 == 0xa0){
5790
/* no-break space */
5791
utext[wi++] = 0x0;
5792
utext[wi++] = 0x20;
5793
} else {
5794
/* (otherwise) */
5795
utext[wi++] = b1;
5796
utext[wi++] = b2;
5797
}
5798
} else if(b1 == 0x20){
5799
if(b2 == 0x2){
5800
/* en space */
5801
utext[wi++] = 0x0;
5802
utext[wi++] = 0x20;
5803
} else if(b2 == 0x3){
5804
/* em space */
5805
utext[wi++] = 0x0;
5806
utext[wi++] = 0x20;
5807
} else if(b2 == 0x9){
5808
/* thin space */
5809
utext[wi++] = 0x0;
5810
utext[wi++] = 0x20;
5811
} else if(b2 == 0x10){
5812
/* hyphen */
5813
utext[wi++] = 0x0;
5814
utext[wi++] = 0x2d;
5815
} else if(b2 == 0x15){
5816
/* fullwidth horizontal line */
5817
utext[wi++] = 0x0;
5818
utext[wi++] = 0x2d;
5819
} else if(b2 == 0x19){
5820
/* apostrophe */
5821
utext[wi++] = 0x0;
5822
utext[wi++] = 0x27;
5823
} else if(b2 == 0x33){
5824
/* double quotes */
5825
utext[wi++] = 0x0;
5826
utext[wi++] = 0x22;
5827
} else {
5828
/* (otherwise) */
5829
utext[wi++] = b1;
5830
utext[wi++] = b2;
5831
}
5832
} else if(b1 == 0x22){
5833
if(b2 == 0x12){
5834
/* minus sign */
5835
utext[wi++] = 0x0;
5836
utext[wi++] = 0x2d;
5837
} else {
5838
/* (otherwise) */
5839
utext[wi++] = b1;
5840
utext[wi++] = b2;
5841
}
5842
} else if(b1 == 0x30){
5843
if(b2 == 0x0){
5844
/* fullwidth space */
5845
utext[wi++] = 0x0;
5846
utext[wi++] = 0x20;
5847
} else {
5848
/* (otherwise) */
5849
utext[wi++] = b1;
5850
utext[wi++] = b2;
5851
}
3262
5852
} else if(b1 == 0xff){
3263
if(b2 >= 0x21 && b2 <= 0x3a){
5853
if(b2 == 0x01){
5854
/* fullwidth exclamation */
5855
utext[wi++] = 0x0;
5856
utext[wi++] = 0x21;
5857
} else if(b2 == 0x03){
5858
/* fullwidth igeta */
5859
utext[wi++] = 0x0;
5860
utext[wi++] = 0x23;
5861
} else if(b2 == 0x04){
5862
/* fullwidth dollar */
5863
utext[wi++] = 0x0;
5864
utext[wi++] = 0x24;
5865
} else if(b2 == 0x05){
5866
/* fullwidth parcent */
5867
utext[wi++] = 0x0;
5868
utext[wi++] = 0x25;
5869
} else if(b2 == 0x06){
5870
/* fullwidth ampersand */
5871
utext[wi++] = 0x0;
5872
utext[wi++] = 0x26;
5873
} else if(b2 == 0x0a){
5874
/* fullwidth asterisk */
5875
utext[wi++] = 0x0;
5876
utext[wi++] = 0x2a;
5877
} else if(b2 == 0x0b){
5878
/* fullwidth plus */
5879
utext[wi++] = 0x0;
5880
utext[wi++] = 0x2b;
5881
} else if(b2 == 0x0c){
5882
/* fullwidth comma */
5883
utext[wi++] = 0x0;
5884
utext[wi++] = 0x2c;
5885
} else if(b2 == 0x0e){
5886
/* fullwidth period */
5887
utext[wi++] = 0x0;
5888
utext[wi++] = 0x2e;
5889
} else if(b2 == 0x0f){
5890
/* fullwidth slash */
5891
utext[wi++] = 0x0;
5892
utext[wi++] = 0x2f;
5893
} else if(b2 == 0x1a){
5894
/* fullwidth colon */
5895
utext[wi++] = 0x0;
5896
utext[wi++] = 0x3a;
5897
} else if(b2 == 0x1b){
5898
/* fullwidth semicolon */
5899
utext[wi++] = 0x0;
5900
utext[wi++] = 0x3b;
5901
} else if(b2 == 0x1d){
5902
/* fullwidth equal */
5903
utext[wi++] = 0x0;
5904
utext[wi++] = 0x3d;
5905
} else if(b2 == 0x1f){
5906
/* fullwidth question */
5907
utext[wi++] = 0x0;
5908
utext[wi++] = 0x3f;
5909
} else if(b2 == 0x20){
5910
/* fullwidth atmark */
5911
utext[wi++] = 0x0;
5912
utext[wi++] = 0x40;
5913
} else if(b2 == 0x3c){
5914
/* fullwidth backslash */
5915
utext[wi++] = 0x0;
5916
utext[wi++] = 0x5c;
5917
} else if(b2 == 0x3e){
5918
/* fullwidth circumflex */
5919
utext[wi++] = 0x0;
5920
utext[wi++] = 0x5e;
5921
} else if(b2 == 0x3f){
5922
/* fullwidth underscore */
5923
utext[wi++] = 0x0;
5924
utext[wi++] = 0x5f;
5925
} else if(b2 == 0x5c){
5926
/* fullwidth vertical line */
5927
utext[wi++] = 0x0;
5928
utext[wi++] = 0x7c;
5929
} else if(b2 >= 0x21 && b2 <= 0x3a){
3264
5930
/* fullwidth alphabets */
3265
utext[wi] = 0x0;
3266
utext[wi+1] = b2 - 0x21 + 0x41;
5931
utext[wi++] = 0x0;
5932
utext[wi++] = b2 - 0x21 + 0x41;
3267
5933
} else if(b2 >= 0x41 && b2 <= 0x5a){
3268
5934
/* fullwidth small alphabets */
3269
utext[wi] = 0x0;
3270
utext[wi+1] = b2 - 0x41 + 0x61;
5935
utext[wi++] = 0x0;
5936
utext[wi++] = b2 - 0x41 + 0x61;
3271
5937
} else if(b2 >= 0x10 && b2 <= 0x19){
3272
5938
/* fullwidth numbers */
3273
utext[wi] = 0x0;
3274
utext[wi+1] = b2 - 0x10 + 0x30;
5939
utext[wi++] = 0x0;
5940
utext[wi++] = b2 - 0x10 + 0x30;
3275
5941
} else if(b2 == 0x61){
3276
5942
/* halfwidth full stop */
3277
utext[wi] = 0x30;
3278
utext[wi+1] = 0x2;
5943
utext[wi++] = 0x30;
5944
utext[wi++] = 0x2;
3279
5945
} else if(b2 == 0x62){
3280
5946
/* halfwidth left corner */
3281
utext[wi] = 0x30;
3282
utext[wi+1] = 0xc;
5947
utext[wi++] = 0x30;
5948
utext[wi++] = 0xc;
3283
5949
} else if(b2 == 0x63){
3284
5950
/* halfwidth right corner */
3285
utext[wi] = 0x30;
3286
utext[wi+1] = 0xd;
5951
utext[wi++] = 0x30;
5952
utext[wi++] = 0xd;
3287
5953
} else if(b2 == 0x64){
3288
5954
/* halfwidth comma */
3289
utext[wi] = 0x30;
3290
utext[wi+1] = 0x1;
5955
utext[wi++] = 0x30;
5956
utext[wi++] = 0x1;
3291
5957
} else if(b2 == 0x65){
3292
5958
/* halfwidth middle dot */
3293
utext[wi] = 0x30;
3294
utext[wi+1] = 0xfb;
5959
utext[wi++] = 0x30;
5960
utext[wi++] = 0xfb;
3295
5961
} else if(b2 == 0x66){
3296
5962
/* halfwidth wo */
3297
utext[wi] = 0x30;
3298
utext[wi+1] = 0xf2;
5963
utext[wi++] = 0x30;
5964
utext[wi++] = 0xf2;
3299
5965
} else if(b2 >= 0x67 && b2 <= 0x6b){
3300
5966
/* halfwidth small a-o */
3301
utext[wi] = 0x30;
3302
utext[wi+1] = (b2 - 0x67) * 2 + 0xa1;
5967
utext[wi++] = 0x30;
5968
utext[wi++] = (b2 - 0x67) * 2 + 0xa1;
3303
5969
} else if(b2 >= 0x6c && b2 <= 0x6e){
3304
5970
/* halfwidth small ya-yo */
3305
utext[wi] = 0x30;
3306
utext[wi+1] = (b2 - 0x6c) * 2 + 0xe3;
5971
utext[wi++] = 0x30;
5972
utext[wi++] = (b2 - 0x6c) * 2 + 0xe3;
3307
5973
} else if(b2 == 0x6f){
3308
5974
/* halfwidth small tu */
3309
utext[wi] = 0x30;
3310
utext[wi+1] = 0xc3;
5975
utext[wi++] = 0x30;
5976
utext[wi++] = 0xc3;
3311
5977
} else if(b2 == 0x70){
3312
5978
/* halfwidth prolonged mark */
3313
utext[wi] = 0x30;
3314
utext[wi+1] = 0xfc;
5979
utext[wi++] = 0x30;
5980
utext[wi++] = 0xfc;
3315
5981
} else if(b2 >= 0x71 && b2 <= 0x75){
3316
5982
/* halfwidth a-o */
3317
utext[wi] = 0x30;
3318
utext[wi+1] = (b2 - 0x71) * 2 + 0xa2;
5983
utext[wi++] = 0x30;
5984
utext[wi++] = (b2 - 0x71) * 2 + 0xa2;
3319
5985
if(i + 2 < size - 1 && b2 == 0x73 && utext[i+2] == 0xff && utext[i+3] == 0x9e){
3320
utext[wi+1] = 0xf4;
5986
utext[wi-1] = 0xf4;
3321
5987
i += 2;
3322
5988
}
3323
5989
} else if(b2 >= 0x76 && b2 <= 0x7a){
3324
5990
/* halfwidth ka-ko */
3325
utext[wi] = 0x30;
3326
utext[wi+1] = (b2 - 0x76) * 2 + 0xab;
5991
utext[wi++] = 0x30;
5992
utext[wi++] = (b2 - 0x76) * 2 + 0xab;
3327
5993
if(i + 2 < size - 1 && utext[i+2] == 0xff && utext[i+3] == 0x9e){
3328
utext[wi+1] += 1;
5994
utext[wi-1] += 1;
3329
5995
i += 2;
3330
5996
}
3331
5997
} else if(b2 >= 0x7b && b2 <= 0x7f){
3332
5998
/* halfwidth sa-so */
3333
utext[wi] = 0x30;
3334
utext[wi+1] = (b2 - 0x7b) * 2 + 0xb5;
5999
utext[wi++] = 0x30;
6000
utext[wi++] = (b2 - 0x7b) * 2 + 0xb5;
3335
6001
if(i + 2 < size - 1 && utext[i+2] == 0xff && utext[i+3] == 0x9e){
3336
utext[wi+1] += 1;
6002
utext[wi-1] += 1;
3337
6003
i += 2;
3338
6004
}
3339
6005
} else if(b2 >= 0x80 && b2 <= 0x84){
3340
6006
/* halfwidth ta-to */
3341
utext[wi] = 0x30;
3342
utext[wi+1] = (b2 - 0x80) * 2 + 0xbf + (b2 >= 0x82 ? 1 : 0);
6007
utext[wi++] = 0x30;
6008
utext[wi++] = (b2 - 0x80) * 2 + 0xbf + (b2 >= 0x82 ? 1 : 0);
3343
6009
if(i + 2 < size - 1 && utext[i+2] == 0xff && utext[i+3] == 0x9e){
3344
utext[wi+1] += 1;
6010
utext[wi-1] += 1;
3345
6011
i += 2;
3346
6012
}
3347
6013
} else if(b2 >= 0x85 && b2 <= 0x89){
3348
6014
/* halfwidth na-no */
3349
utext[wi] = 0x30;
3350
utext[wi+1] = b2 - 0x85 + 0xca;
6015
utext[wi++] = 0x30;
6016
utext[wi++] = b2 - 0x85 + 0xca;
3351
6017
} else if(b2 >= 0x8a && b2 <= 0x8e){
3352
6018
/* halfwidth ha-ho */
3353
utext[wi] = 0x30;
3354
utext[wi+1] = (b2 - 0x8a) * 3 + 0xcf;
6019
utext[wi++] = 0x30;
6020
utext[wi++] = (b2 - 0x8a) * 3 + 0xcf;
3355
6021
if(i + 2 < size - 1){
3356
6022
if(utext[i+2] == 0xff && utext[i+3] == 0x9e){
3357
utext[wi+1] += 1;
6023
utext[wi-1] += 1;
3358
6024
i += 2;
3359
6025
} else if(utext[i+2] == 0xff && utext[i+3] == 0x9f){
3360
utext[wi+1] += 2;
6026
utext[wi-1] += 2;
3361
6027
i += 2;
3362
6028
}
3363
6029
}
3364
6030
} else if(b2 >= 0x8f && b2 <= 0x93){
3365
6031
/* halfwidth ma-mo */
3366
utext[wi] = 0x30;
3367
utext[wi+1] = b2 - 0x8f + 0xde;
6032
utext[wi++] = 0x30;
6033
utext[wi++] = b2 - 0x8f + 0xde;
3368
6034
} else if(b2 >= 0x94 && b2 <= 0x96){
3369
6035
/* halfwidth ya-yo */
3370
utext[wi] = 0x30;
3371
utext[wi+1] = (b2 - 0x94) * 2 + 0xe4;
6036
utext[wi++] = 0x30;
6037
utext[wi++] = (b2 - 0x94) * 2 + 0xe4;
3372
6038
} else if(b2 >= 0x97 && b2 <= 0x9b){
3373
6039
/* halfwidth ra-ro */
3374
utext[wi] = 0x30;
3375
utext[wi+1] = b2 - 0x97 + 0xe9;
6040
utext[wi++] = 0x30;
6041
utext[wi++] = b2 - 0x97 + 0xe9;
3376
6042
} else if(b2 == 0x9c){
3377
6043
/* halfwidth wa */
3378
utext[wi] = 0x30;
3379
utext[wi+1] = 0xef;
6044
utext[wi++] = 0x30;
6045
utext[wi++] = 0xef;
3380
6046
} else if(b2 == 0x9d){
3381
6047
/* halfwidth wo */
3382
utext[wi] = 0x30;
3383
utext[wi+1] = 0xf3;
6048
utext[wi++] = 0x30;
6049
utext[wi++] = 0xf3;
3384
6050
} else {
3385
utext[wi] = b1;
3386
utext[wi+1] = b2;
6051
/* (otherwise) */
6052
utext[wi++] = b1;
6053
utext[wi++] = b2;
3387
6054
}
3388
6055
} else {
3389
utext[wi] = b1;
3390
utext[wi+1] = b2;
6056
/* (otherwise) */
6057
utext[wi++] = b1;
6058
utext[wi++] = b2;
3391
6059
}
3392
wi += 2;
3393
6060
}
3394
6061
*sp = wi;
3395
6062
}
3403
6070
int i;
3404
6071
for(i = 0; i < size; i += 2){
3405
6072
if(utext[i] == 0x0){
3406
if(utext[i+1] >= 'A' && utext[i+1] <= 'Z'){
6073
if(utext[i+1] < ' '){
6074
/* functional spaces */
6075
if(!funcspc) utext[i+1] = ' ';
6076
} else if(utext[i+1] >= 'A' && utext[i+1] <= 'Z'){
3407
6077
/* ascii */
3408
6078
utext[i+1] += 'a' - 'A';
3409
} else if((utext[i+1] >= 0xc0 && utext[i+1] <= 0xd6) ||
3410
(utext[i+1] >= 0xd8 && utext[i+1] <= 0xde)){
6079
} else if(utext[i+1] >= 0xc0){
3411
6080
/* latin-1 supplement */
3412
utext[i+1] += 0x20;
3413
} else if(!funcspc && utext[i+1] < ' '){
3414
/* functional spaces */
3415
utext[i+1] = ' ';
6081
if((utext[i+1] >= 0xc0 && utext[i+1] <= 0xd6) ||
6082
(utext[i+1] >= 0xd8 && utext[i+1] <= 0xde)) utext[i+1] += 0x20;
6083
if(utext[i+1] >= 0xe0 && utext[i+1] <= 0xe5){
6084
utext[i+1] = 'a';
6085
} else if(utext[i+1] == 0xe7){
6086
utext[i+1] = 'c';
6087
} else if(utext[i+1] >= 0xe8 && utext[i+1] <= 0xeb){
6088
utext[i+1] = 'e';
6089
} else if(utext[i+1] >= 0xec && utext[i+1] <= 0xef){
6090
utext[i+1] = 'i';
6091
} else if(utext[i+1] == 0xf1){
6092
utext[i+1] = 'n';
6093
} else if((utext[i+1] >= 0xf2 && utext[i+1] <= 0xf6) || utext[i+1] == 0xf8){
6094
utext[i+1] = 'o';
6095
} else if(utext[i+1] >= 0xf9 && utext[i+1] <= 0xfc){
6096
utext[i+1] = 'u';
6097
} else if(utext[i+1] == 0xfd || utext[i+1] == 0xff){
6098
utext[i+1] = 'y';
6099
}
3416
6100
}
3417
6101
} else if(utext[i] == 0x1){
6102
/* latin extended-a */
3418
6103
if((utext[i+1] <= 0x36 && utext[i+1] % 2 == 0) ||
3419
6104
(utext[i+1] >= 0x39 && utext[i+1] <= 0x47 && utext[i+1] % 2 == 1) ||
3420
6105
(utext[i+1] >= 0x4a && utext[i+1] <= 0x76 && utext[i+1] % 2 == 0) ||
3421
(utext[i+1] >= 0x79 && utext[i+1] <= 0x7d && utext[i+1] % 2 == 1)){
3422
/* latin extended-a */
6106
(utext[i+1] >= 0x79 && utext[i+1] <= 0x7d && utext[i+1] % 2 == 1))
3423
6107
utext[i+1] += 0x1;
3424
} else if(utext[i+1] == 0x78){
3425
/* y with umlaut */
3426
utext[i] = 0x0;
3427
utext[i+1] = 0xff;
6108
if(utext[i+1] <= 0x05){
6109
utext[i] = 0x0;
6110
utext[i+1] = 'a';
6111
} else if(utext[i+1] >= 0x06 && utext[i+1] <= 0x0d){
6112
utext[i] = 0x0;
6113
utext[i+1] = 'c';
6114
} else if(utext[i+1] >= 0x0e && utext[i+1] <= 0x11){
6115
utext[i] = 0x0;
6116
utext[i+1] = 'd';
6117
} else if(utext[i+1] >= 0x12 && utext[i+1] <= 0x1b){
6118
utext[i] = 0x0;
6119
utext[i+1] = 'e';
6120
} else if(utext[i+1] >= 0x1c && utext[i+1] <= 0x23){
6121
utext[i] = 0x0;
6122
utext[i+1] = 'g';
6123
} else if(utext[i+1] >= 0x24 && utext[i+1] <= 0x27){
6124
utext[i] = 0x0;
6125
utext[i+1] = 'h';
6126
} else if(utext[i+1] >= 0x28 && utext[i+1] <= 0x31){
6127
utext[i] = 0x0;
6128
utext[i+1] = 'i';
6129
} else if(utext[i+1] >= 0x34 && utext[i+1] <= 0x35){
6130
utext[i] = 0x0;
6131
utext[i+1] = 'j';
6132
} else if(utext[i+1] >= 0x36 && utext[i+1] <= 0x38){
6133
utext[i] = 0x0;
6134
utext[i+1] = 'k';
6135
} else if(utext[i+1] >= 0x39 && utext[i+1] <= 0x42){
6136
utext[i] = 0x0;
6137
utext[i+1] = 'l';
6138
} else if(utext[i+1] >= 0x43 && utext[i+1] <= 0x4b){
6139
utext[i] = 0x0;
6140
utext[i+1] = 'n';
6141
} else if(utext[i+1] >= 0x4c && utext[i+1] <= 0x51){
6142
utext[i] = 0x0;
6143
utext[i+1] = 'o';
6144
} else if(utext[i+1] >= 0x54 && utext[i+1] <= 0x59){
6145
utext[i] = 0x0;
6146
utext[i+1] = 'r';
6147
} else if((utext[i+1] >= 0x5a && utext[i+1] <= 0x61) || utext[i+1] == 0x7f){
6148
utext[i] = 0x0;
6149
utext[i+1] = 's';
6150
} else if(utext[i+1] >= 0x62 && utext[i+1] <= 0x67){
6151
utext[i] = 0x0;
6152
utext[i+1] = 't';
6153
} else if(utext[i+1] >= 0x68 && utext[i+1] <= 0x73){
6154
utext[i] = 0x0;
6155
utext[i+1] = 'u';
6156
} else if(utext[i+1] >= 0x74 && utext[i+1] <= 0x75){
6157
utext[i] = 0x0;
6158
utext[i+1] = 'w';
6159
} else if(utext[i+1] >= 0x76 && utext[i+1] <= 0x78){
6160
utext[i] = 0x0;
6161
utext[i+1] = 'y';
6162
} else if(utext[i+1] >= 0x79 && utext[i+1] <= 0x7e){
6163
utext[i] = 0x0;
6164
utext[i+1] = 'z';
3428
6165
}
3429
6166
} else if(utext[i] == 0x3){
3430
if(utext[i+1] >= 0x91 && utext[i+1] <= 0xa9){
3431
/* greek */
3432
utext[i+1] += 0x20;
3433
}
6167
/* greek */
6168
if(utext[i+1] >= 0x91 && utext[i+1] <= 0xa9) utext[i+1] += 0x20;
3434
6169
} else if(utext[i] == 0x4){
6170
/* cyrillic */
3435
6171
if(utext[i+1] >= 0x10 && utext[i+1] <= 0x2f){
3436
/* cyrillic */
3437
6172
utext[i+1] += 0x20;
3438
6173
} else if(utext[i+1] <= 0x0f){
3439
/* cyrillic with mark */
3440
6174
utext[i+1] += 0x50;
3441
6175
}
3442
6176
} else if(utext[i] == 0xff){
6177
/* special */
3443
6178
if(utext[i+1] >= 0xf0){
3444
/* special */
3445
6179
utext[i] = 0x0;
3446
6180
utext[i+1] = ' ';
3447
6181
}
3466
6200
if(c >= 0x0600 && c <= 0x08ff) return ESTEASTALPH;
3467
6201
/* south and south east asia */
3468
6202
if((c >= 0x0900 && c <= 0x109f) || (c >= 0x1700 && c <= 0x1cff)) return ESTEASTALPH;
3469
/* cjk */
3470
if((c >= 0x1100 && c <= 0x11ff) || (c >= 0x2e80 && c <= 0xd7af) ||
6203
/* cjk and surrogates */
6204
if((c >= 0x1100 && c <= 0x11ff) || (c >= 0x2e80 && c <= 0xdfff) ||
3471
6205
(c >= 0xf900 && c <= 0xfaff) || (c >= 0xff00 && c <= 0xffef)) return ESTEASTALPH;
3472
6206
/* asian presentation forms */
3473
6207
if((c >= 0xfb50 && c <= 0xfdff) || (c >= 0xfe30 && c <= 0xfe4f) ||
3486
6220
}
3487
6221
3488
6222
6223
/* Categorize a character for character category analyzer.
6224
`c' specifies the UCS number of a character.
6225
The return value is the category of the character. */
6226
static int est_char_category_chrcat(int c){
6227
/* ascii space */
6228
if(c <= 0x0020) return ESTSPACECHR;
6229
/* ascii alnum */
6230
if((c >= 0x0030 && c <= 0x0039) || (c >= 0x0041 && c <= 0x005a) ||
6231
(c >= 0x0061 && c <= 0x007a)) return ESTWESTALPH;
6232
/* latin */
6233
if((c >= 0x00c0 && c <= 0x00ff && c != 0x00d7 && c != 0x00f7) || (c >= 0x0100 && c <= 0x017f))
6234
return ESTWESTALPH;
6235
/* arabic and syrian */
6236
if(c >= 0x0600 && c <= 0x08ff) return ESTEASTALPH;
6237
/* south and south east asia */
6238
if((c >= 0x0900 && c <= 0x109f) || (c >= 0x1700 && c <= 0x1cff)) return ESTEASTALPH;
6239
/* hiragana */
6240
if(c >= 0x3040 && c <= 0x309f) return ESTHIRAGANA;
6241
/* katakana */
6242
if(c >= 0x30a0 && c <= 0x30ff) return ESTKATAKANA;
6243
/* hangul */
6244
if((c >= 0x1100 && c <= 0x11ff) || (c >= 0x3130 && c <= 0x318f) ||
6245
(c >= 0xac00 && c <= 0xd7af)) return ESTHANGUL;
6246
/* kanji */
6247
if(c >= 0x4e00 && c <= 0x9faf) return ESTKANJI;
6248
/* other cjk and surrogates */
6249
if((c >= 0x2e80 && c <= 0xdfff) || (c >= 0xf900 && c <= 0xfaff) ||
6250
(c >= 0xff00 && c <= 0xffef)) return ESTEASTALPH;
6251
/* asian presentation forms */
6252
if((c >= 0xfb50 && c <= 0xfdff) || (c >= 0xfe30 && c <= 0xfe4f) ||
6253
(c >= 0xfe70 && c <= 0xfeff)) return ESTEASTALPH;
6254
/* others */
6255
return ESTDELIMCHR;
6256
}
6257
6258
6259
/* Make a snippet of an arbitrary string.
6260
`word' specifies a list object of words to be highlight.
6261
`wwidth' specifies whole width of the result.
6262
`hwidth' specifies width of strings picked up from the beginning of the text.
6263
`awidth' specifies width of strings picked up around each highlighted word.
6264
The return value is a snippet string of the string. */
6265
static char *est_make_snippet(const char *str, int len, const CBLIST *words,
6266
int wwidth, int hwidth, int awidth){
6267
CBDATUM *res;
6268
CBMAP *counts;
6269
CBLIST *rwords;
6270
const char *word, *cval;
6271
const unsigned char *rword;
6272
unsigned char *rtext, *ctext;
6273
int i, j, k, bi, size, wsiz, rwsiz, mywidth, awsiz, csiz;
6274
assert(str && len >= 0 && words && wwidth >= 0 && hwidth >= 0 && awidth >= 0);
6275
CB_DATUMOPEN(res);
6276
CB_LISTOPEN(rwords);
6277
for(i = 0; i < CB_LISTNUM(words); i++){
6278
word = CB_LISTVAL2(words, i, wsiz);
6279
if(wsiz < 1 || !strcmp(word, ESTOPUVSET)) continue;
6280
rtext = (unsigned char *)est_uconv_in(word, wsiz, &size);
6281
est_canonicalize_text(rtext, size, TRUE);
6282
CB_LISTPUSHBUF(rwords, (char *)rtext, size);
6283
}
6284
rtext = (unsigned char *)est_uconv_in(str, len, &size);
6285
ctext = (unsigned char *)cbmemdup((char *)rtext, size);
6286
est_canonicalize_text(ctext, size, FALSE);
6287
mywidth = hwidth;
6288
if(CB_LISTNUM(rwords) < 1) mywidth *= 3;
6289
if(mywidth > wwidth) mywidth = wwidth;
6290
for(i = 0; i < size && mywidth > 0; i += 2){
6291
mywidth -= est_char_category(rtext[i] * 0x100 + rtext[i+1]) == ESTEASTALPH ? 2 : 1;
6292
}
6293
awsiz = size - i;
6294
if(awsiz > ESTWORDMAXLEN) awsiz = ESTWORDMAXLEN;
6295
est_snippet_add_text(rtext, ctext, i, awsiz, res, rwords);
6296
wwidth -= hwidth;
6297
bi = i + 2;
6298
CB_DATUMCAT(res, "\n", 1);
6299
if(awidth > 0){
6300
counts = cbmapopenex(ESTMINIBNUM);
6301
for(i = bi; i < size && wwidth >= 0; i += 2){
6302
for(j = 0; j < CB_LISTNUM(rwords); j++){
6303
rword = (unsigned char *)CB_LISTVAL2(rwords, j, rwsiz);
6304
if(est_str_fwmatch_wide(ctext + i, size - i, rword, rwsiz) > 0 &&
6305
(!(cval = cbmapget(counts, (char *)rword, rwsiz, &csiz)) ||
6306
csiz < (wwidth > awidth * 1.2 ? 2 : 1))){
6307
cbmapputcat(counts, (char *)rword, rwsiz, "*", 1);
6308
if(cbmaprnum(counts) >= CB_LISTNUM(rwords)){
6309
cbmapclose(counts);
6310
counts = cbmapopenex(ESTMINIBNUM);
6311
}
6312
mywidth = awidth / 2 + 1;
6313
for(k = i - 2; k >= bi && mywidth >= 0; k -= 2){
6314
mywidth -= est_char_category(rtext[k] * 0x100 + rtext[k+1]) == ESTEASTALPH ? 2 : 1;
6315
}
6316
bi = k;
6317
mywidth = awidth / 2 + 1;
6318
for(k = i + rwsiz + 2; k < size && mywidth >= 0; k += 2){
6319
mywidth -= est_char_category(rtext[k] * 0x100 + rtext[k+1]) == ESTEASTALPH ? 2 : 1;
6320
}
6321
if(k > size) k = size;
6322
est_snippet_add_text(rtext + bi, ctext + bi, k - bi, 0, res, rwords);
6323
wwidth -= awidth + rwsiz / 2;
6324
bi = k + 2;
6325
i = bi - 2;
6326
CB_DATUMCAT(res, "\n", 1);
6327
break;
6328
}
6329
}
6330
}
6331
cbmapclose(counts);
6332
}
6333
free(ctext);
6334
free(rtext);
6335
CB_LISTCLOSE(rwords);
6336
return cbdatumtomalloc(res, NULL);
6337
}
6338
6339
6340
/* Check whether a string is compsed of CJK characters only.
6341
`str' specifies a string of UTF-8.
6342
The return value is whether the string is compsed of CJK characters only. */
6343
static int est_check_cjk_only(const char *str){
6344
const unsigned char *rp;
6345
int size;
6346
rp = (unsigned char *)str;
6347
size = strlen(str);
6348
while(rp < (unsigned char *)str + size){
6349
if(*rp < 0x7f){
6350
return FALSE;
6351
} else if(*rp < 0xdf){
6352
return FALSE;
6353
} else if(*rp < 0xf0){
6354
if(rp >= (unsigned char *)str + size - 2) break;
6355
rp += 3;
6356
} else if(*rp < 0xf8){
6357
if(rp >= (unsigned char *)str + size - 3) break;
6358
rp += 4;
6359
} else if(*rp < 0xfb){
6360
if(rp >= (unsigned char *)str + size - 4) break;
6361
rp += 5;
6362
} else if(*rp < 0xfd){
6363
if(rp >= (unsigned char *)str + size - 5) break;
6364
rp += 6;
6365
} else {
6366
break;
6367
}
6368
}
6369
return TRUE;
6370
}
6371
6372
3489
6373
/* Convert a simplified phrase into complete form.
3490
6374
`sphrase' specifies a simplified phrase.
3491
6375
The return value is the complete form of the phrase. */
3492
static char *est_phrase_from_thumb(const char *sphrase){
6376
static char *est_phrase_from_simple(const char *sphrase){
3493
6377
CBDATUM *datum;
3494
6378
const char *oper, *rp, *pv;
3495
6379
unsigned char *utext;
3496
6380
char *rtext;
3497
6381
int size, quote, lw;
3498
6382
assert(sphrase);
3499
datum = cbdatumopen("", 0);
6383
CB_DATUMOPEN(datum);
3500
6384
utext = (unsigned char *)est_uconv_in(sphrase, strlen(sphrase), &size);
3501
6385
est_normalize_text(utext, size, &size);
3502
6386
est_canonicalize_text(utext, size, FALSE);
3508
6392
for(rp = rtext; *rp != '\0'; rp++){
3509
6393
if(*rp == '"'){
3510
6394
if(oper){
3511
cbdatumcat(datum, oper, -1);
6395
CB_DATUMCAT(datum, oper, strlen(oper));
3512
6396
oper = NULL;
3513
6397
}
3514
6398
quote = !quote;
3515
6399
continue;
3516
6400
}
3517
6401
if(quote){
3518
cbdatumcat(datum, rp, 1);
6402
CB_DATUMCAT(datum, rp, 1);
3519
6403
continue;
3520
6404
}
3521
6405
switch(*rp){
3537
6421
break;
3538
6422
default:
3539
6423
if(oper){
3540
cbdatumcat(datum, oper, -1);
6424
CB_DATUMCAT(datum, oper, strlen(oper));
3541
6425
oper = NULL;
3542
6426
}
3543
6427
if(!lw){
3547
6431
}
3548
6432
if(pv > rp + 1 && pv[-1] == '*'){
3549
6433
if(rp[0] == '*'){
3550
cbdatumcat(datum, ESTOPWCRX " ", -1);
6434
CB_DATUMCAT(datum, ESTOPWCRX " ", strlen(ESTOPWCRX) + 1);
3551
6435
} else {
3552
cbdatumcat(datum, ESTOPWCBW " ", -1);
6436
CB_DATUMCAT(datum, ESTOPWCBW " ", strlen(ESTOPWCBW) + 1);
3553
6437
}
3554
6438
} else if(pv > rp + 1 && rp[0] == '*'){
3555
6439
if(pv[-1] == '*'){
3556
cbdatumcat(datum, ESTOPWCRX " ", -1);
6440
CB_DATUMCAT(datum, ESTOPWCRX " ", strlen(ESTOPWCRX) + 1);
3557
6441
} else {
3558
cbdatumcat(datum, ESTOPWCEW " ", -1);
6442
CB_DATUMCAT(datum, ESTOPWCEW " ", strlen(ESTOPWCEW) + 1);
3559
6443
}
3560
6444
}
3561
6445
}
3562
if(*rp != '*' || (lw && rp[1] != '\0' && rp[1] != ' ')) cbdatumcat(datum, rp, 1);
3563
lw = TRUE;
3564
}
3565
}
6446
if(*rp != '*' || (lw && rp[1] != '\0' && rp[1] != ' ')) CB_DATUMCAT(datum, rp, 1);
6447
lw = TRUE;
6448
}
6449
}
6450
free(rtext);
6451
free(utext);
6452
return cbdatumtomalloc(datum, NULL);
6453
}
6454
6455
6456
/* Convert a rough phrase into complete form.
6457
`rphrase' specifies a simplified phrase.
6458
The return value is the complete form of the phrase. */
6459
static char *est_phrase_from_rough(const char *rphrase){
6460
CBDATUM *datum;
6461
const char *oper, *rp;
6462
unsigned char *utext;
6463
char *rtext;
6464
int size, quote, lw;
6465
assert(rphrase);
6466
CB_DATUMOPEN(datum);
6467
utext = (unsigned char *)est_uconv_in(rphrase, strlen(rphrase), &size);
6468
est_normalize_text(utext, size, &size);
6469
est_canonicalize_text(utext, size, FALSE);
6470
rtext = est_uconv_out((char *)utext, size, NULL);
6471
cbstrsqzspc(rtext);
6472
quote = FALSE;
6473
oper = NULL;
6474
lw = FALSE;
6475
for(rp = rtext; *rp != '\0'; rp++){
6476
if(*rp == '"'){
6477
if(oper){
6478
CB_DATUMCAT(datum, oper, strlen(oper));
6479
oper = NULL;
6480
}
6481
quote = !quote;
6482
continue;
6483
}
6484
if(quote){
6485
CB_DATUMCAT(datum, rp, 1);
6486
continue;
6487
}
6488
switch(*rp){
6489
case ' ':
6490
if(!oper) oper = " AND ";
6491
lw = FALSE;
6492
break;
6493
case '&':
6494
oper = " AND ";
6495
lw = FALSE;
6496
break;
6497
case '|':
6498
oper = " OR ";
6499
lw = FALSE;
6500
break;
6501
case '-':
6502
if(lw){
6503
CB_DATUMCAT(datum, rp, 1);
6504
} else {
6505
oper = " ANDNOT ";
6506
}
6507
break;
6508
default:
6509
if(oper){
6510
CB_DATUMCAT(datum, oper, strlen(oper));
6511
oper = NULL;
6512
}
6513
CB_DATUMCAT(datum, rp, 1);
6514
lw = TRUE;
6515
}
6516
}
6517
free(rtext);
6518
free(utext);
6519
return cbdatumtomalloc(datum, NULL);
6520
}
6521
6522
6523
/* Convert a union phrase into complete form.
6524
`uphrase' specifies a simplified phrase.
6525
The return value is the complete form of the phrase. */
6526
static char *est_phrase_from_union(const char *uphrase){
6527
CBDATUM *datum;
6528
CBLIST *terms;
6529
const char *term;
6530
unsigned char *utext;
6531
char *rtext;
6532
int i, size;
6533
assert(uphrase);
6534
CB_DATUMOPEN(datum);
6535
utext = (unsigned char *)est_uconv_in(uphrase, strlen(uphrase), &size);
6536
est_normalize_text(utext, size, &size);
6537
est_canonicalize_text(utext, size, FALSE);
6538
rtext = est_uconv_out((char *)utext, size, NULL);
6539
cbstrsqzspc(rtext);
6540
terms = cbsplit(rtext, -1, " ");
6541
for(i = 0; i < CB_LISTNUM(terms); i++){
6542
term = CB_LISTVAL2(terms, i, size);
6543
if(size < 1) continue;
6544
if(CB_DATUMSIZE(datum) > 0) CB_DATUMCAT(datum, " OR ", 4);
6545
CB_DATUMCAT(datum, term, size);
6546
}
6547
CB_LISTCLOSE(terms);
6548
free(rtext);
6549
free(utext);
6550
return cbdatumtomalloc(datum, NULL);
6551
}
6552
6553
6554
/* Convert a intersection phrase into complete form.
6555
`iphrase' specifies a simplified phrase.
6556
The return value is the complete form of the phrase. */
6557
static char *est_phrase_from_isect(const char *iphrase){
6558
CBDATUM *datum;
6559
CBLIST *terms;
6560
const char *term;
6561
unsigned char *utext;
6562
char *rtext;
6563
int i, size;
6564
assert(iphrase);
6565
CB_DATUMOPEN(datum);
6566
utext = (unsigned char *)est_uconv_in(iphrase, strlen(iphrase), &size);
6567
est_normalize_text(utext, size, &size);
6568
est_canonicalize_text(utext, size, FALSE);
6569
rtext = est_uconv_out((char *)utext, size, NULL);
6570
cbstrsqzspc(rtext);
6571
terms = cbsplit(rtext, -1, " ");
6572
for(i = 0; i < CB_LISTNUM(terms); i++){
6573
term = CB_LISTVAL2(terms, i, size);
6574
if(size < 1) continue;
6575
if(CB_DATUMSIZE(datum) > 0) CB_DATUMCAT(datum, " AND ", 5);
6576
CB_DATUMCAT(datum, term, size);
6577
}
6578
CB_LISTCLOSE(terms);
3566
6579
free(rtext);
3567
6580
free(utext);
3568
6581
return cbdatumtomalloc(datum, NULL);
3584
6597
bi = 0;
3585
6598
for(i = 0; i < size; i += 2){
3586
6599
for(j = 0; j < CB_LISTNUM(rwords); j++){
3587
rword = (unsigned char *)CB_LISTVAL2(rwords, j, &rwsiz);
6600
rword = (unsigned char *)CB_LISTVAL2(rwords, j, rwsiz);
3588
6601
if((step = est_str_fwmatch_wide(ctext + i, size + awsiz - i, rword, rwsiz)) > 0){
3589
6602
if(i - bi > 0){
3590
6603
orig = est_uconv_out((char *)rtext + bi, i - bi, &osiz);
3591
cbdatumcat(res, orig, osiz);
3592
cbdatumcat(res, "\n", 1);
6604
CB_DATUMCAT(res, orig, osiz);
6605
CB_DATUMCAT(res, "\n", 1);
3593
6606
free(orig);
3594
6607
}
3595
6608
orig = est_uconv_out((char *)rtext + i, step, &osiz);
3596
cbdatumcat(res, orig, osiz);
6609
CB_DATUMCAT(res, orig, osiz);
3597
6610
free(orig);
3598
cbdatumcat(res, "\t", 1);
6611
CB_DATUMCAT(res, "\t", 1);
3599
6612
orig = est_uconv_out((char *)rword, rwsiz, &osiz);
3600
cbdatumcat(res, orig, osiz);
6613
CB_DATUMCAT(res, orig, osiz);
3601
6614
free(orig);
3602
cbdatumcat(res, "\n", 1);
6615
CB_DATUMCAT(res, "\n", 1);
3603
6616
bi = i + step;
3604
6617
i = bi - 2;
3605
6618
break;
3608
6621
}
3609
6622
if(i - bi > 0){
3610
6623
orig = est_uconv_out((char *)rtext + bi, i - bi, &osiz);
3611
cbdatumcat(res, orig, osiz);
3612
cbdatumcat(res, "\n", 1);
6624
CB_DATUMCAT(res, orig, osiz);
6625
CB_DATUMCAT(res, "\n", 1);
3613
6626
free(orig);
3614
6627
}
3615
6628
}
3648
6661
}
3649
6662
3650
6663
6664
/* Find the first occurrence of a substring ignoring space characters.
6665
`haystack' specifies a target string.
6666
`needle' specifies a substring.
6667
The the pointer to the first occurrence. */
6668
static char *est_strstr_sparse(const char *haystack, const char *needle){
6669
const char *hp, *np;
6670
assert(haystack && needle);
6671
while(*needle > '\0' && *needle <= ' '){
6672
needle++;
6673
}
6674
if(needle[0] == '\0') return (char *)haystack;
6675
while((haystack = strchr(haystack, *needle)) != NULL){
6676
hp = haystack;
6677
np = needle;
6678
while(TRUE){
6679
while(*hp > '\0' && *hp <= ' '){
6680
hp++;
6681
}
6682
while(*np > '\0' && *np <= ' '){
6683
np++;
6684
}
6685
if(*np == '\0') return (char *)haystack;
6686
if(*hp != *np || *hp == '\0') break;
6687
hp++;
6688
np++;
6689
}
6690
haystack++;
6691
}
6692
return NULL;
6693
}
6694
6695
6696
/* Get the last ID number in an index record.
6697
`vbuf' specifies the pointer to the value of a record.
6698
`vsiz' specifies the size of the value.
6699
`smode' specifies a mode of score type.
6700
The return value is the last ID number in a record. */
6701
static int est_idx_rec_last_id(const char *vbuf, int vsiz, int smode){
6702
const char *rp, *ep, *sp;
6703
int cid, vnum, vstep;
6704
assert(vbuf && vsiz >= 0);
6705
cid = 0;
6706
rp = vbuf;
6707
ep = vbuf + vsiz;
6708
while(rp < ep){
6709
EST_READ_VNUMBUF(rp, vnum, vstep);
6710
cid += vnum + 1;
6711
rp += vstep;
6712
sp = rp;
6713
switch(smode){
6714
case ESTDFSCVOID:
6715
break;
6716
default:
6717
rp++;
6718
break;
6719
case ESTDFSCINT:
6720
case ESTDFSCASIS:
6721
rp += sizeof(int);
6722
break;
6723
}
6724
while(*rp != 0x00){
6725
rp += 2;
6726
}
6727
rp++;
6728
}
6729
return cid;
6730
}
6731
6732
6733
/* Encode a raw index record into a gap form.
6734
`datum' specifies a datum to store the result.
6735
`vbuf' specifies the pointer to the value of a raw index record.
6736
`vsiz' specifies the size of the value of the record.
6737
`lid' specifies the last ID number in the existing record.
6738
`smode' specifies a mode of score type. */
6739
static void est_encode_idx_rec(CBDATUM *datum, const char *vbuf, int vsiz, int lid, int smode){
6740
const char *rp, *ep, *sp;
6741
char nbuf[ESTNUMBUFSIZ];
6742
int cid, vstep;
6743
assert(datum && vbuf && vsiz >= 0);
6744
rp = vbuf;
6745
ep = vbuf + vsiz;
6746
while(rp < ep){
6747
EST_READ_VNUMBUF(rp, cid, vstep);
6748
rp += vstep;
6749
sp = rp;
6750
switch(smode){
6751
case ESTDFSCVOID:
6752
break;
6753
default:
6754
rp++;
6755
break;
6756
case ESTDFSCINT:
6757
case ESTDFSCASIS:
6758
rp += sizeof(int);
6759
break;
6760
}
6761
while(*rp != 0x00){
6762
rp += 2;
6763
}
6764
rp++;
6765
EST_SET_VNUMBUF(vstep, nbuf, cid - lid - 1);
6766
CB_DATUMCAT(datum, nbuf, vstep);
6767
CB_DATUMCAT(datum, sp, rp - sp);
6768
lid = cid;
6769
}
6770
}
6771
6772
6773
/* Decode a gap index record into a raw form.
6774
`datum' specifies a datum to store the result.
6775
`vbuf' specifies the pointer to the value of a gap index record.
6776
`vsiz' specifies the size of the value of the record.
6777
`smode' specifies a mode of score type. */
6778
static void est_decode_idx_rec(CBDATUM *datum, const char *vbuf, int vsiz, int smode){
6779
const char *rp, *ep, *sp;
6780
char nbuf[ESTNUMBUFSIZ];
6781
int cid, vnum, vstep;
6782
assert(datum && vbuf && vsiz >= 0);
6783
rp = vbuf;
6784
ep = vbuf + vsiz;
6785
cid = 0;
6786
while(rp < ep){
6787
EST_READ_VNUMBUF(rp, vnum, vstep);
6788
cid += vnum + 1;
6789
rp += vstep;
6790
sp = rp;
6791
switch(smode){
6792
case ESTDFSCVOID:
6793
break;
6794
default:
6795
rp++;
6796
break;
6797
case ESTDFSCINT:
6798
case ESTDFSCASIS:
6799
rp += sizeof(int);
6800
break;
6801
}
6802
while(*rp != 0x00){
6803
rp += 2;
6804
}
6805
rp++;
6806
EST_SET_VNUMBUF(vstep, nbuf, cid);
6807
CB_DATUMCAT(datum, nbuf, vstep);
6808
CB_DATUMCAT(datum, sp, rp - sp);
6809
}
6810
}
6811
6812
3651
6813
/* Open the inverted index.
3652
6814
`name' specifies the name of a directory.
3653
6815
`omode' specifies an open mode of Villa.
3656
6818
static ESTIDX *est_idx_open(const char *name, int omode, int dnum){
3657
6819
ESTIDX *idx;
3658
6820
CBLIST *files;
6821
const char *file;
3659
6822
char path[ESTPATHBUFSIZ];
3660
int i;
6823
int i, crdnum;
3661
6824
assert(name && dnum > 0);
3662
6825
if(dnum > ESTIDXDMAX) dnum = ESTIDXDMAX;
3663
6826
CB_MALLOC(idx, sizeof(ESTIDX));
3664
6827
if((omode & VL_OCREAT) && !est_mkdir(name) && errno != EEXIST) return NULL;
3665
6828
if((omode & VL_OTRUNC) && (files = cbdirlist(name)) != NULL){
3666
6829
for(i = 0; i < CB_LISTNUM(files); i++){
3667
sprintf(path, "%s%c%s", name, ESTPATHCHR, CB_LISTVAL(files, i, NULL));
3668
unlink(path);
6830
file = CB_LISTVAL(files, i);
6831
if(!strcmp(file, ESTCDIRSTR) || !strcmp(file, ESTPDIRSTR)) continue;
6832
sprintf(path, "%s%c%s", name, ESTPATHCHR, file);
6833
if(unlink(path) == -1) est_rmdir_rec(path);
3669
6834
}
3670
cblistclose(files);
6835
CB_LISTCLOSE(files);
3671
6836
}
3672
6837
for(i = 0; i < dnum; i++){
3673
6838
sprintf(path, "%s%c%04d", name, ESTPATHCHR, i + 1);
6839
crdnum = vlcrdnum;
6840
vlcrdnum = ESTVLCRDNUM;
3674
6841
if(!(idx->dbs[i] = vlopen(path, omode, VL_CMPLEX))){
3675
6842
while(--i >= 0){
3676
6843
vlclose(idx->dbs[i]);
3677
6844
}
6845
vlcrdnum = crdnum;
3678
6846
return NULL;
3679
6847
}
6848
vlcrdnum = crdnum;
3680
6849
}
3681
6850
idx->name = cbmemdup(name, -1);
3682
6851
idx->omode = omode;
3704
6873
3705
6874
/* Set the tuning parameters of the inverted index.
3706
6875
`idx' specifies an object of the inverted index.
6876
`lrecmax' specifies the max number of records in a leaf node of B+ tree.
6877
`nidxmax' specifies the max number of indexes in a non-leaf node of B+ tree.
6878
`lcnum' specifies the max number of caching leaf nodes.
6879
`ncnum' specifies the max number of caching non-leaf nodes.
6880
`fbpsiz' specifies the size of the free block pool.
3707
6881
Other parameters are same with `vlsettuning' of Villa. */
3708
static void est_idx_set_tuning(ESTIDX *idx, int lrecmax, int nidxmax, int lcnum, int ncnum){
6882
static void est_idx_set_tuning(ESTIDX *idx, int lrecmax, int nidxmax, int lcnum, int ncnum,
6883
int fbpsiz){
3709
6884
int i;
3710
6885
assert(idx);
3711
6886
for(i = 0; i < idx->dnum; i++){
3712
6887
vlsettuning(idx->dbs[i], lrecmax, nidxmax, lcnum, ncnum);
6888
if(fbpsiz > 0) vlsetfbpsiz(idx->dbs[i], fbpsiz);
3713
6889
}
3714
6890
}
3715
6891
3718
6894
`idx' specifies an object of the inverted index. */
3719
6895
static void est_idx_increment(ESTIDX *idx){
3720
6896
char path[ESTPATHBUFSIZ];
3721
int i, min, size;
6897
int i, min, size, crdnum;
3722
6898
assert(idx);
3723
6899
min = INT_MAX;
3724
6900
for(i = 0; i < idx->dnum; i++){
3730
6906
return;
3731
6907
}
3732
6908
sprintf(path, "%s%c%04d", idx->name, ESTPATHCHR, idx->dnum + 1);
6909
crdnum = vlcrdnum;
6910
vlcrdnum = ESTVLCRDNUM;
3733
6911
if((idx->dbs[idx->dnum] = vlopen(path, idx->omode | VL_OCREAT | VL_OTRUNC, VL_CMPLEX)) != NULL){
3734
6912
idx->cdb = idx->dbs[idx->dnum];
3735
6913
idx->dnum++;
3736
6914
}
6915
vlcrdnum = crdnum;
3737
6916
}
3738
6917
3739
6918
3750
6929
`word' specifies a word.
3751
6930
`vbuf' specifies the pointer to the value of a record.
3752
6931
`vsiz' specifies the size of the value.
6932
`smode' specifies a mode of score type.
3753
6933
The return value is true if success, else it is false. */
3754
static int est_idx_add(ESTIDX *idx, const char *word, int wsiz, const char *vbuf, int vsiz){
6934
static int est_idx_add(ESTIDX *idx, const char *word, int wsiz,
6935
const char *vbuf, int vsiz, int smode){
6936
CBDATUM *datum;
6937
const char *obuf;
6938
int rv, lid, osiz;
3755
6939
assert(idx && word && wsiz >= 0 && vbuf && vsiz >= 0);
3756
return vlput(idx->cdb, word, wsiz, vbuf, vsiz, VL_DDUP);
6940
CB_DATUMOPEN(datum);
6941
lid = 0;
6942
if((obuf = vlgetcache(idx->cdb, word, wsiz, &osiz)) != NULL)
6943
lid = est_idx_rec_last_id(obuf, osiz, smode);
6944
est_encode_idx_rec(datum, vbuf, vsiz, lid, smode);
6945
rv = vlput(idx->cdb, word, wsiz, CB_DATUMPTR(datum), CB_DATUMSIZE(datum), VL_DCAT);
6946
CB_DATUMCLOSE(datum);
6947
return rv;
3757
6948
}
3758
6949
3759
6950
3767
6958
static int est_idx_put_one(ESTIDX *idx, int inum, const char *word, int wsiz,
3768
6959
const char *vbuf, int vsiz){
3769
6960
assert(idx && inum >= 0 && word && wsiz >= 0 && vbuf && vsiz >= 0);
3770
if(!vloutlist(idx->dbs[inum], word, wsiz) && dpecode != DP_ENOITEM) return FALSE;
3771
return vsiz < 1 ? TRUE : vlput(idx->dbs[inum], word, wsiz, vbuf, vsiz, VL_DDUP);
6961
return vsiz > 0 ? vlput(idx->dbs[inum], word, wsiz, vbuf, vsiz, VL_DOVER) :
6962
(vlout(idx->dbs[inum], word, wsiz) || dpecode == DP_ENOITEM);
3772
6963
}
3773
6964
3774
6965
3783
6974
assert(idx && word && wsiz >= 0);
3784
6975
err = FALSE;
3785
6976
for(i = 0; i < idx->dnum; i++){
3786
if(!vloutlist(idx->dbs[i], word, wsiz) && dpecode != DP_ENOITEM) err = TRUE;
6977
if(!vlout(idx->dbs[i], word, wsiz) && dpecode != DP_ENOITEM) err = TRUE;
3787
6978
}
3788
6979
return err ? FALSE : TRUE;
3789
6980
}
3795
6986
`wsiz' specifies the size of the word.
3796
6987
`sp' specifies the pointer to a variable to which the size of the region of the return value
3797
6988
is assigned.
6989
`smode' specifies a mode of score type.
3798
6990
The return value is the pointer to the region of the value of the corresponding record.
3799
6991
if no item correspongs, empty region is returned. */
3800
static char *est_idx_get(ESTIDX *idx, const char *word, int wsiz, int *sp){
6992
static char *est_idx_scan(ESTIDX *idx, const char *word, int wsiz, int *sp, int smode){
3801
6993
CBDATUM *datum;
3802
char *vbuf;
6994
const char *vbuf;
3803
6995
int i, vsiz;
3804
6996
assert(idx && word && wsiz >= 0 && sp);
3805
datum = cbdatumopen("", 0);
6997
CB_DATUMOPEN(datum);
3806
6998
for(i = 0; i < idx->dnum; i++){
3807
if(!(vbuf = vlgetcat(idx->dbs[i], word, wsiz, &vsiz))) continue;
3808
cbdatumcat(datum, vbuf, vsiz);
3809
free(vbuf);
6999
if(!(vbuf = vlgetcache(idx->dbs[i], word, wsiz, &vsiz))) continue;
7000
est_decode_idx_rec(datum, vbuf, vsiz, smode);
3810
7001
}
3811
7002
return cbdatumtomalloc(datum, sp);
3812
7003
}
3820
7011
`sp' specifies the pointer to a variable to which the size of the region of the return value
3821
7012
is assigned.
3822
7013
The return value is the pointer to the region of the value of the corresponding record.
3823
if no item correspongs, `NULL' is returned. */
3824
static char *est_idx_get_one(ESTIDX *idx, int inum, const char *word, int wsiz, int *sp){
7014
if no item correspongs, `NULL' is returned. Because the region of the return value is
7015
volatile, it sould be copied immediately. */
7016
static const char *est_idx_get_one(ESTIDX *idx, int inum, const char *word, int wsiz, int *sp){
3825
7017
assert(idx && inum >= 0 && word && wsiz >= 0 && sp);
3826
return vlgetcat(idx->dbs[inum], word, wsiz, sp);
7018
return vlgetcache(idx->dbs[inum], word, wsiz, sp);
3827
7019
}
3828
7020
3829
7021
3834
7026
The return value is the size of the value of the corresponding record.
3835
7027
if no item correspongs, 0 is returned. */
3836
7028
static int est_idx_vsiz(ESTIDX *idx, const char *word, int wsiz){
3837
char *vbuf;
3838
7029
int i, sum, vsiz;
3839
7030
assert(idx && word && wsiz >= 0);
3840
7031
sum = 0;
3841
7032
for(i = 0; i < idx->dnum; i++){
3842
if(!(vbuf = vlgetcat(idx->dbs[i], word, wsiz, &vsiz))) continue;
7033
if((vsiz = vlvsiz(idx->dbs[i], word, wsiz)) < 1) continue;
3843
7034
sum += vsiz;
3844
free(vbuf);
3845
7035
}
3846
7036
return sum;
3847
7037
}
3880
7070
}
3881
7071
3882
7072
7073
/* Synchronize updating contents of the inverted index on memory.
7074
`idx' specifies an object of the inverted index.
7075
The return value is true if success, else it is false. */
7076
static int est_idx_memflush(ESTIDX *idx){
7077
int i;
7078
assert(idx);
7079
for(i = 0; i < idx->dnum; i++){
7080
if(!vlmemflush(idx->dbs[i])) return FALSE;
7081
}
7082
return TRUE;
7083
}
7084
7085
3883
7086
/* Syncronize the inverted index.
3884
7087
`idx' specifies an object of the inverted index.
3885
The return value is the size of the inverted index. */
7088
The return value is true if success, else it is false. */
3886
7089
static int est_idx_sync(ESTIDX *idx){
3887
7090
int i;
3888
7091
assert(idx);
3895
7098
3896
7099
/* Optimize the inverted index.
3897
7100
`idx' specifies an object of the inverted index.
3898
The return value is the size of the inverted index. */
7101
The return value is true if success, else it is false. */
3899
7102
static int est_idx_optimize(ESTIDX *idx){
3900
7103
int i;
3901
7104
assert(idx);
3921
7124
}
3922
7125
3923
7126
7127
/* Store a record related to the ID number of a document.
7128
`curia' specifies a database object.
7129
`zmode' specifies a compression mode.
7130
`id' specifies the ID number of a document.
7131
`vbuf' specifies the pointer to the value of a record.
7132
`vsiz' specifies the size of the value.
7133
The return value is true if success, else it is false. */
7134
static int est_crput(CURIA *curia, int zmode, int id, const char *vbuf, int vsiz, int dmode){
7135
char *zbuf;
7136
int zsiz;
7137
assert(curia && id > 0 && vbuf && vsiz >= 0);
7138
switch(zmode){
7139
case ESTDFZLIB:
7140
if(!(zbuf = est_deflate(vbuf, vsiz, &zsiz, -1))){
7141
dpecode = ESTEMISC;
7142
return FALSE;
7143
}
7144
if(!crput(curia, (char *)&id, sizeof(int), zbuf, zsiz, dmode)){
7145
free(zbuf);
7146
return FALSE;
7147
}
7148
free(zbuf);
7149
break;
7150
case ESTDFLZO:
7151
if(!(zbuf = est_lzoencode(vbuf, vsiz, &zsiz))){
7152
dpecode = ESTEMISC;
7153
return FALSE;
7154
}
7155
if(!crput(curia, (char *)&id, sizeof(int), zbuf, zsiz, dmode)){
7156
free(zbuf);
7157
return FALSE;
7158
}
7159
free(zbuf);
7160
break;
7161
case ESTDFBZIP:
7162
if(!(zbuf = est_bzencode(vbuf, vsiz, &zsiz))){
7163
dpecode = ESTEMISC;
7164
return FALSE;
7165
}
7166
if(!crput(curia, (char *)&id, sizeof(int), zbuf, zsiz, dmode)){
7167
free(zbuf);
7168
return FALSE;
7169
}
7170
free(zbuf);
7171
break;
7172
default:
7173
if(!crput(curia, (char *)&id, sizeof(int), vbuf, vsiz, dmode)) return FALSE;
7174
break;
7175
}
7176
return TRUE;
7177
}
7178
7179
7180
/* Remove a record related to the ID number of a document.
7181
`curia' specifies a database object.
7182
`id' specifies the ID number of a document.
7183
The return value is true if success, else it is false. */
7184
static int est_crout(CURIA *curia, int id){
7185
assert(curia && id > 0);
7186
return crout(curia, (char *)&id, sizeof(int));
7187
}
7188
7189
7190
/* Get a record related to the ID number of a document.
7191
`curia' specifies a database object.
7192
`zmode' specifies a compression mode.
7193
`id' specifies the ID number of a document.
7194
`sp' specifies the pointer to a variable to which the size of the region of the return value
7195
is assigned.
7196
The return value is the pointer to the region of the value of the corresponding record. */
7197
static char *est_crget(CURIA *curia, int zmode, int id, int *sp){
7198
char *zbuf, *vbuf;
7199
int zsiz;
7200
assert(curia && id > 0 && sp);
7201
switch(zmode){
7202
case ESTDFZLIB:
7203
if(!(zbuf = crget(curia, (char *)&id, sizeof(int), 0, -1, &zsiz))) return NULL;
7204
if(!(vbuf = est_inflate(zbuf, zsiz, sp, -1))){
7205
free(zbuf);
7206
return NULL;
7207
}
7208
free(zbuf);
7209
break;
7210
case ESTDFLZO:
7211
if(!(zbuf = crget(curia, (char *)&id, sizeof(int), 0, -1, &zsiz))) return NULL;
7212
if(!(vbuf = est_lzodecode(zbuf, zsiz, sp))){
7213
free(zbuf);
7214
return NULL;
7215
}
7216
free(zbuf);
7217
break;
7218
case ESTDFBZIP:
7219
if(!(zbuf = crget(curia, (char *)&id, sizeof(int), 0, -1, &zsiz))) return NULL;
7220
if(!(vbuf = est_bzdecode(zbuf, zsiz, sp))){
7221
free(zbuf);
7222
return NULL;
7223
}
7224
free(zbuf);
7225
break;
7226
default:
7227
if(!(vbuf = crget(curia, (char *)&id, sizeof(int), 0, -1, sp))) return NULL;
7228
break;
7229
}
7230
return vbuf;
7231
}
7232
7233
7234
/* Add an attribute of a document to a sequencial attribute index.
7235
`db' specifies a handle of a sequencial attribute index.
7236
`id' specifies the ID number of a document.
7237
`vbuf' specifies the pointer to the attribute value.
7238
`vsiz' specifies the size of the attribute value.
7239
The return value is true if success, else it is false. */
7240
static int est_aidx_seq_put(DEPOT *db, int id, const char *vbuf, int vsiz){
7241
int err;
7242
assert(db && id >= 0 && vbuf && vsiz >= 0);
7243
err = FALSE;
7244
if(!dpput(db, (char *)&id, sizeof(int), vbuf, vsiz, DP_DKEEP)) err = TRUE;
7245
return err ? FALSE : TRUE;
7246
}
7247
7248
7249
/* Remove an attribute of a document from a sequencial attribute index.
7250
`db' specifies a handle of a sequencial attribute index.
7251
`id' specifies the ID number of a document.
7252
The return value is true if success, else it is false. */
7253
static int est_aidx_seq_out(DEPOT *db, int id){
7254
int err;
7255
assert(db && id >= 0);
7256
err = FALSE;
7257
if(!dpout(db, (char *)&id, sizeof(int))) err = TRUE;
7258
return err ? FALSE : TRUE;
7259
}
7260
7261
7262
/* Retrieve the value of an attribute of a document in a sequencial attribute index.
7263
`db' specifies a handle of a sequencial attribute index.
7264
`id' specifies the ID number of a document.
7265
The return value is the value of the attribute or `NULL' if no attribute. */
7266
static char *est_aidx_seq_get(DEPOT *db, int id, int *sp){
7267
assert(db && id >= 0 && sp);
7268
return dpget(db, (char *)&id, sizeof(int), 0, -1, sp);
7269
}
7270
7271
7272
/* Narrow scores of search candidates with a sequencial attribute index.
7273
`db' specifies a handle of a sequencial attribute index.
7274
`pdocs' specifies a list of pseudo documents.
7275
`cop' specifies the pointer to the operator.
7276
`sign' specifies the sign of operation.
7277
`oval' specifies the operation value.
7278
`osiz' specifies the size of the operation value
7279
`sval' specifies the operation value of small cases.
7280
`ssiz' specifies the size of the operation value of small cases.
7281
`regex' specifies the regular expressions.
7282
`onum' specifies the numeric value.
7283
`scores' specifies an array of scores of search candidates.
7284
`snum' specifies the number of the array.
7285
`limit' specifies the limit number to check.
7286
`restp' specifies the pointer to a variable to which rest number to be checked is assigned.
7287
The return value is the new number of the array. */
7288
static int est_aidx_seq_narrow(DEPOT *db, const CBLIST *pdocs, const char *cop, int sign,
7289
const char *oval, int osiz, const char *sval, int ssiz,
7290
const void *regex, int onum, ESTSCORE *scores, int snum,
7291
int limit, int *restp){
7292
char vbuf[ESTAIKBUFSIZ];
7293
int i, nnum, vsiz;
7294
assert(db && cop && oval && osiz >= 0 && scores && snum >= 0 && limit >= 0 && restp);
7295
nnum = 0;
7296
for(i = 0; i < snum; i++){
7297
if(nnum >= limit){
7298
*restp = snum - i;
7299
break;
7300
}
7301
if(scores[i].id >= ESTPDOCIDMIN){
7302
scores[nnum].id = scores[i].id;
7303
scores[nnum].score = scores[i].score;
7304
nnum++;
7305
continue;
7306
}
7307
if((vsiz = dpgetwb(db, (char *)&(scores[i].id), sizeof(int), 0, ESTAIKBUFSIZ - 1, vbuf)) < 0)
7308
continue;
7309
vbuf[vsiz] = '\0';
7310
if(est_match_attr(vbuf, vsiz, cop, sign, oval, osiz, sval, ssiz, regex, onum)){
7311
scores[nnum].id = scores[i].id;
7312
scores[nnum].score = scores[i].score;
7313
nnum++;
7314
}
7315
}
7316
return nnum;
7317
}
7318
7319
7320
/* Compare two record in numeric order.
7321
`aptr' specifies the pointer to the region of one key.
7322
`asiz' specifies the size of the region of one key.
7323
`bptr' specifies the pointer to the region of the other key.
7324
`bsiz' specifies the size of the region of the other key.
7325
The return value is positive if the former is big, negative if the latter is big, 0 if both
7326
are equivalent. */
7327
static int est_aidx_numcmp(const char *aptr, int asiz, const char *bptr, int bsiz){
7328
int rv;
7329
if((rv = cbstrmktime(aptr) - cbstrmktime(bptr)) != 0) return rv;
7330
return VL_CMPLEX(aptr, asiz, bptr, bsiz);
7331
}
7332
7333
7334
/* Add an attribute of a document to an attribute narrowing index.
7335
`db' specifies a handle of an attribute narrowing index.
7336
`id' specifies the ID number of a document.
7337
`vbuf' specifies the pointer to the attribute value.
7338
`vsiz' specifies the size of the attribute value.
7339
The return value is true if success, else it is false. */
7340
static int est_aidx_attr_put(VILLA *db, int id, const char *vbuf, int vsiz){
7341
char *tbuf;
7342
int err, tsiz;
7343
assert(db && id >= 0 && vbuf && vsiz >= 0);
7344
err = FALSE;
7345
tsiz = vsiz + sizeof(int) + 1;
7346
CB_MALLOC(tbuf, tsiz);
7347
memcpy(tbuf, vbuf, vsiz + 1);
7348
memcpy(tbuf + vsiz + 1, &id, sizeof(int));
7349
if(!vlput(db, tbuf, tsiz, "", 0, VL_DKEEP)) err = TRUE;
7350
free(tbuf);
7351
return err ? FALSE : TRUE;
7352
}
7353
7354
7355
/* Remove an attribute of a document from an attribute narrowing index.
7356
`db' specifies a handle of an attribute narrowing index.
7357
`id' specifies the ID number of a document.
7358
`vbuf' specifies the pointer to the attribute value.
7359
`vsiz' specifies the size of the attribute value.
7360
The return value is true if success, else it is false. */
7361
static int est_aidx_attr_out(VILLA *db, int id, const char *vbuf, int vsiz){
7362
char *tbuf;
7363
int err, tsiz;
7364
assert(db && id >= 0 && vbuf && vsiz >= 0);
7365
err = FALSE;
7366
tsiz = vsiz + sizeof(int) + 1;
7367
CB_MALLOC(tbuf, tsiz);
7368
memcpy(tbuf, vbuf, vsiz + 1);
7369
memcpy(tbuf + vsiz + 1, &id, sizeof(int));
7370
if(!vlout(db, tbuf, tsiz)) err = TRUE;
7371
free(tbuf);
7372
return err ? FALSE : TRUE;
7373
}
7374
7375
7376
/* Narrow scores of search candidates with an attribute narrowing index.
7377
`db' specifies a handle of an attribute narrowing index.
7378
`pdocs' specifies a list of pseudo documents.
7379
`cop' specifies the pointer to the operator.
7380
`sign' specifies the sign of operation.
7381
`oval' specifies the operation value.
7382
`osiz' specifies the size of the operation value
7383
`sval' specifies the operation value of small cases.
7384
`ssiz' specifies the size of the operation value of small cases.
7385
`regex' specifies the regular expressions.
7386
`onum' specifies the numeric value.
7387
`scores' specifies an array of scores of search candidates.
7388
`snum' specifies the number of the array.
7389
The return value is the new number of the array. */
7390
static int est_aidx_attr_narrow(VILLA *db, const CBLIST *pdocs, const char *cop, int sign,
7391
const char *oval, int osiz, const char *sval, int ssiz,
7392
const void *regex, int onum, ESTSCORE *scores, int snum){
7393
CBDATUM *abuf;
7394
CBLIST *tokens;
7395
const char *kbuf;
7396
char numbuf[ESTNUMBUFSIZ], *tmp, *wp;
7397
int i, j, ksiz, len, esc, jmp, id, nnum, *ary, anum;
7398
time_t lower, upper;
7399
assert(db && pdocs && cop && oval && osiz >= 0 && scores && snum >= 0);
7400
CB_DATUMOPEN(abuf);
7401
if(cop == ESTOPSTROREQ && sign && !sval){
7402
tokens = cbsplit(oval, osiz, " ,");
7403
cblistsort(tokens);
7404
for(i = 0; i < CB_LISTNUM(tokens); i++){
7405
oval = CB_LISTVAL2(tokens, i, osiz);
7406
if(osiz < 1) continue;
7407
vlcurjump(db, oval, osiz, VL_JFORWARD);
7408
while((kbuf = vlcurkeycache(db, &ksiz)) != NULL && !strcmp(kbuf, oval)){
7409
CB_DATUMCAT(abuf, kbuf + ksiz - sizeof(int), sizeof(int));
7410
vlcurnext(db);
7411
}
7412
}
7413
CB_LISTCLOSE(tokens);
7414
} else if(cop == ESTOPNUMBT && sign && !sval){
7415
CB_MEMDUP(tmp, oval, osiz);
7416
if((wp = strchr(tmp, ' ')) != NULL || (wp = strchr(tmp, '\t')) != NULL){
7417
*(wp++) = '\0';
7418
while(*wp == ' ' || *wp == '\t'){
7419
wp++;
7420
}
7421
lower = cbstrmktime(tmp);
7422
upper = cbstrmktime(wp);
7423
} else {
7424
lower = cbstrmktime(tmp);
7425
upper = INT_MAX;
7426
}
7427
len = sprintf(numbuf, "%.0f", (double)lower);
7428
vlcurjump(db, numbuf, len, VL_JFORWARD);
7429
while((kbuf = vlcurkeycache(db, &ksiz)) != NULL && cbstrmktime(kbuf) <= upper){
7430
CB_DATUMCAT(abuf, kbuf + ksiz - sizeof(int), sizeof(int));
7431
vlcurnext(db);
7432
}
7433
free(tmp);
7434
} else if(!sign || sval){
7435
esc = INT_MAX;
7436
jmp = INT_MAX;
7437
if(sign && (cop == ESTOPSTREQ || cop == ESTOPSTRBW) && osiz > 0){
7438
if(*sval > 0x0 && *sval < 0x7f){
7439
numbuf[0] = *sval;
7440
numbuf[1] = '\0';
7441
esc = *(unsigned char *)sval;
7442
if(*sval >= 'a' && *sval <= 'z'){
7443
numbuf[0] -= 'a' - 'A';
7444
jmp = *sval - 'a' + 'A';
7445
}
7446
vlcurjump(db, numbuf, 1, VL_JFORWARD);
7447
} else if(*(unsigned char *)sval >= 0xc0){
7448
numbuf[0] = *sval;
7449
numbuf[1] = '\0';
7450
esc = *(unsigned char *)sval;
7451
vlcurjump(db, numbuf, 1, VL_JFORWARD);
7452
} else {
7453
vlcurfirst(db);
7454
}
7455
} else {
7456
vlcurfirst(db);
7457
}
7458
while((kbuf = vlcurkeycache(db, &ksiz)) != NULL){
7459
if(est_match_attr(kbuf, ksiz - sizeof(int) - 1,
7460
cop, sign, oval, osiz, sval, ssiz, regex, onum))
7461
CB_DATUMCAT(abuf, kbuf + ksiz - sizeof(int), sizeof(int));
7462
if(*(unsigned char *)kbuf > jmp && *(unsigned char *)kbuf < *(unsigned char *)sval){
7463
numbuf[0] = *sval;
7464
numbuf[1] = '\0';
7465
vlcurjump(db, numbuf, 1, VL_JFORWARD);
7466
jmp = INT_MAX;
7467
} else if(*(unsigned char *)kbuf > esc){
7468
break;
7469
} else {
7470
vlcurnext(db);
7471
}
7472
}
7473
} else {
7474
if(cop == ESTOPSTREQ || cop == ESTOPSTRBW ||
7475
cop == ESTOPNUMEQ || cop == ESTOPNUMGT || cop == ESTOPNUMGE){
7476
vlcurjump(db, oval, osiz, VL_JFORWARD);
7477
if(cop == ESTOPNUMGT){
7478
while((kbuf = vlcurkeycache(db, NULL)) != NULL && cbstrmktime(kbuf) <= onum){
7479
vlcurnext(db);
7480
}
7481
}
7482
} else if(cop == ESTOPNUMLT || cop == ESTOPNUMLE){
7483
len = sprintf(numbuf, "%.0f", (double)cbstrmktime(oval) + 1);
7484
vlcurjump(db, numbuf, len, VL_JBACKWARD);
7485
if(cop == ESTOPNUMLT){
7486
while((kbuf = vlcurkeycache(db, NULL)) != NULL && cbstrmktime(kbuf) >= onum){
7487
vlcurprev(db);
7488
}
7489
}
7490
} else {
7491
vlcurfirst(db);
7492
}
7493
while((kbuf = vlcurkeycache(db, &ksiz)) != NULL){
7494
if(est_match_attr(kbuf, ksiz - sizeof(int) - 1,
7495
cop, TRUE, oval, osiz, sval, ssiz, regex, onum)){
7496
CB_DATUMCAT(abuf, kbuf + ksiz - sizeof(int), sizeof(int));
7497
} else if(cop == ESTOPSTREQ || cop == ESTOPSTRBW || cop == ESTOPNUMEQ){
7498
break;
7499
}
7500
if(cop == ESTOPNUMLT || cop == ESTOPNUMLE){
7501
vlcurprev(db);
7502
} else {
7503
vlcurnext(db);
7504
}
7505
}
7506
}
7507
for(i = 0; i < CB_LISTNUM(pdocs); i++){
7508
id = ESTPDOCIDMIN + i;
7509
CB_DATUMCAT(abuf, &id, sizeof(int));
7510
}
7511
nnum = 0;
7512
ary = (int *)CB_DATUMPTR(abuf);
7513
anum = CB_DATUMSIZE(abuf) / sizeof(int);
7514
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
7515
qsort(ary, anum, sizeof(int), est_int_compare);
7516
for(i = 0, j = 0; i < snum; i++){
7517
while(j < anum && ary[j] < scores[i].id){
7518
j++;
7519
}
7520
if(j < anum && scores[i].id == ary[j]){
7521
scores[nnum].id = scores[i].id;
7522
scores[nnum].score = scores[i].score;
7523
nnum++;
7524
}
7525
}
7526
CB_DATUMCLOSE(abuf);
7527
return nnum;
7528
}
7529
7530
7531
/* Compare two integers.
7532
`ap' specifies the pointer to one element.
7533
`bp' specifies the pointer to the other element.
7534
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
7535
static int est_int_compare(const void *ap, const void *bp){
7536
assert(ap && bp);
7537
return *(int *)ap - *(int *)bp;
7538
}
7539
7540
7541
/* Compare elements of a record for effective compression.
7542
`ap' specifies the pointer to one element.
7543
`bp' specifies the pointer to the other element.
7544
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
7545
static int est_short_compare(const void *ap, const void *bp){
7546
assert(ap && bp);
7547
return ((((unsigned char *)ap)[0] << 8) + ((unsigned char *)ap)[1]) -
7548
((((unsigned char *)bp)[0] << 8) + ((unsigned char *)bp)[1]);
7549
}
7550
7551
7552
/* Clean up the inode map.
7553
`arg' specifies a dummy argument. */
7554
static void est_inodes_delete(void *arg){
7555
#if defined(NDEBUG)
7556
ESTDB *db;
7557
const char *kbuf;
7558
int ecode;
7559
assert(arg);
7560
if(cbmaprnum(est_inodes) > 0){
7561
cbmapiterinit(est_inodes);
7562
while((kbuf = cbmapiternext(est_inodes, NULL)) != NULL){
7563
db = *(ESTDB **)cbmapiterval(kbuf, NULL);
7564
est_db_set_informer(db, est_inodes_delete_informer, NULL);
7565
est_db_close(db, &ecode);
7566
}
7567
}
7568
cbmapclose(est_inodes);
7569
#else
7570
ESTDB *db;
7571
const char *kbuf;
7572
int ecode;
7573
assert(arg);
7574
if(cbmaprnum(est_inodes) > 0){
7575
cbmapiterinit(est_inodes);
7576
while((kbuf = cbmapiternext(est_inodes, NULL)) != NULL){
7577
db = *(ESTDB **)cbmapiterval(kbuf, NULL);
7578
fprintf(stderr, "\nWARNING: %s is not closed.\n\n", cbmemdup(est_db_name(db), -1));
7579
est_db_set_informer(db, est_inodes_delete_informer, NULL);
7580
est_db_close(db, &ecode);
7581
}
7582
}
7583
cbmapclose(est_inodes);
7584
#endif
7585
}
7586
7587
7588
/* Inform a database event while clening up database handles.
7589
`msg' specifies the message of each event.
7590
`opaque' is ignored. */
7591
static void est_inodes_delete_informer(const char *msg, void *opaque){
7592
#if !defined(NDEBUG)
7593
fprintf(stderr, "estraier: %s\n", msg);
7594
#endif
7595
}
7596
7597
3924
7598
/* Write meta data to the database.
3925
7599
`db' specifies a database object.
3926
7600
The return value is true if success, else it is false. */
3935
7609
if(!dpput(db->metadb, ESTKEYDSEQ, -1, vbuf, -1, DP_DOVER)) err = TRUE;
3936
7610
sprintf(vbuf, "%d", db->dnum);
3937
7611
if(!dpput(db->metadb, ESTKEYDNUM, -1, vbuf, -1, DP_DOVER)) err = TRUE;
3938
sprintf(vbuf, "%d", db->amode);
3939
if(!dpput(db->metadb, ESTKEYAMODE, -1, vbuf, -1, DP_DOVER)) err = TRUE;
3940
7612
if(db->metacc){
3941
7613
sbuf = cbmapdump(db->metacc, &ssiz);
3942
7614
if(!dpput(db->metadb, ESTKEYMETA, -1, sbuf, ssiz, DP_DOVER)) err = TRUE;
3955
7627
`info' specifies an extra message. */
3956
7628
static void est_db_inform(ESTDB *db, const char *info){
3957
7629
char *msg;
3958
assert(db);
3959
if(!db->cbinfo) return;
7630
assert(db && info);
7631
if(!db->infocb) return;
3960
7632
msg = cbsprintf("%s: name=%s dnum=%d wnum=%d fsiz=%.0f crnum=%d csiz=%d dknum=%d",
3961
7633
info, db->name, db->dnum, vlrnum(db->fwmdb), (double)est_db_size(db),
3962
cbmaprnum(db->idxcc), est_db_used_cache_size(db), cbmaprnum(db->outcc));
3963
db->cbinfo(msg);
7634
cbmaprnum(db->idxcc) + cbmaprnum(db->auxcc), est_db_used_cache_size(db),
7635
cbmaprnum(db->outcc));
7636
db->infocb(msg, db->infoop);
3964
7637
free(msg);
3965
7638
}
3966
7639
3980
7653
}
3981
7654
3982
7655
7656
/* Score a document object matching the phrase of a search condition object definitely.
7657
`db' specifies a database object.
7658
`doc' specifies a document object.
7659
`cond' specifies a search condition object.
7660
`scp' specifies the pointer to a variable to which the score is assigned.
7661
The return value is true if the document matches the phrase of the condition object
7662
definitely, else it is false. */
7663
static int est_db_score_doc(ESTDB *db, ESTDOC *doc, ESTCOND *cond, int *scp){
7664
struct { char *word; int num; } wsets[ESTSCANWNUM], nsets[ESTSCANWNUM];
7665
CBLIST *terms, *words;
7666
const char *term, *text, *rp;
7667
unsigned char *rbuf;
7668
char *tmp;
7669
int i, j, k, sc, wsnum, nsnum, asiz, tsiz, add, rsiz, hit;
7670
double tune;
7671
assert(db && doc && cond && scp);
7672
*scp = 0;
7673
if(!cond->phrase || cbstrfwmatch(cond->phrase, ESTOPSIMILAR) ||
7674
cbstrfwmatch(cond->phrase, ESTOPID) || cbstrfwmatch(cond->phrase, ESTOPURI)) return FALSE;
7675
if(!doc->dtexts) CB_LISTOPEN(doc->dtexts);
7676
switch(cond->pmode){
7677
default:
7678
terms = est_phrase_terms(cond->phrase);
7679
break;
7680
case ESTPMSIMPLE:
7681
tmp = est_phrase_from_simple(cond->phrase);
7682
terms = est_phrase_terms(tmp);
7683
free(tmp);
7684
break;
7685
case ESTPMROUGH:
7686
tmp = est_phrase_from_rough(cond->phrase);
7687
terms = est_phrase_terms(tmp);
7688
free(tmp);
7689
break;
7690
case ESTPMUNION:
7691
tmp = est_phrase_from_union(cond->phrase);
7692
terms = est_phrase_terms(tmp);
7693
free(tmp);
7694
break;
7695
case ESTPMISECT:
7696
tmp = est_phrase_from_isect(cond->phrase);
7697
terms = est_phrase_terms(tmp);
7698
free(tmp);
7699
break;
7700
}
7701
wsnum = 0;
7702
nsnum = 0;
7703
add = TRUE;
7704
for(i = 0; i < CB_LISTNUM(terms); i++){
7705
term = CB_LISTVAL(terms, i);
7706
if(!strcmp(term, ESTOPISECT)){
7707
add = TRUE;
7708
} else if(!strcmp(term, ESTOPDIFF)){
7709
add = FALSE;
7710
} else if(strcmp(term, ESTOPUVSET)){
7711
if(term[0] == ' '){
7712
term++;
7713
if(term[0] == 'b'){
7714
term++;
7715
} else if(term[0] == 'e'){
7716
term++;
7717
}
7718
}
7719
words = cbsplit(term, -1, "\t");
7720
if(add){
7721
while(wsnum < ESTSCANWNUM && CB_LISTNUM(words) > 0){
7722
wsets[wsnum].word = cblistshift(words, NULL);
7723
wsets[wsnum].num = i;
7724
wsnum++;
7725
}
7726
} else {
7727
while(nsnum < ESTSCANWNUM && CB_LISTNUM(words) > 0){
7728
nsets[nsnum].word = cblistshift(words, NULL);
7729
nsets[nsnum].num = i;
7730
nsnum++;
7731
}
7732
}
7733
CB_LISTCLOSE(words);
7734
}
7735
}
7736
asiz = 0;
7737
sc = 0;
7738
if((rp = cbmapget(doc->attrs, "\t", 1, NULL)) != NULL) sc = -1 - atoi(rp);
7739
for(i = -1; i < CB_LISTNUM(doc->dtexts); i++){
7740
if(i < 0){
7741
if(!doc->attrs || !(text = cbmapget(doc->attrs, "", 0, NULL))) continue;
7742
asiz += strlen(text);
7743
} else {
7744
text = CB_LISTVAL2(doc->dtexts, i, tsiz);
7745
asiz += tsiz;
7746
}
7747
rbuf = (unsigned char *)est_uconv_in(text, strlen(text), &rsiz);
7748
est_canonicalize_text(rbuf, rsiz, FALSE);
7749
tmp = est_uconv_out((char *)rbuf, rsiz, &rsiz);
7750
for(j = 0; j < wsnum; j++){
7751
if(!wsets[j].word) continue;
7752
if((rp = est_strstr_sparse(tmp, wsets[j].word)) != NULL){
7753
if(sc >= 0){
7754
do {
7755
sc += 16;
7756
rp += strlen(wsets[j].word);
7757
} while((rp = est_strstr_sparse(rp, wsets[j].word)) != NULL);
7758
}
7759
for(k = 0; k < wsnum; k++){
7760
if(!wsets[k].word) continue;
7761
if(wsets[k].num == wsets[j].num){
7762
free(wsets[k].word);
7763
wsets[k].word = NULL;
7764
}
7765
}
7766
}
7767
}
7768
for(j = 0; j < nsnum; j++){
7769
if(!nsets[j].word) continue;
7770
if((rp = est_strstr_sparse(tmp, nsets[j].word)) != NULL){
7771
for(k = 0; k < nsnum; k++){
7772
if(!nsets[k].word) continue;
7773
if(nsets[k].num == nsets[j].num){
7774
free(nsets[k].word);
7775
nsets[k].word = NULL;
7776
}
7777
}
7778
}
7779
}
7780
free(tmp);
7781
free(rbuf);
7782
}
7783
hit = TRUE;
7784
for(i = 0; i < wsnum; i++){
7785
if(!wsets[i].word) continue;
7786
free(wsets[i].word);
7787
hit = FALSE;
7788
}
7789
for(i = 0; i < nsnum; i++){
7790
if(!nsets[i].word){
7791
hit = FALSE;
7792
continue;
7793
}
7794
free(nsets[i].word);
7795
}
7796
CB_LISTCLOSE(terms);
7797
if(sc < 0) sc = -1 - sc;
7798
tune = sqrt(asiz / 8.0 + 128) / 16.0;
7799
switch(db->smode){
7800
case ESTDFSCVOID:
7801
sc = 0;
7802
break;
7803
default:
7804
sc /= tune;
7805
if(sc >= 0x80) sc += (0x80 - sc) * 0.75;
7806
if(sc >= 0xc0) sc += (0xc0 - sc) * 0.75;
7807
sc = sc < 0xff ? sc : 0xff;
7808
break;
7809
case ESTDFSCINT:
7810
sc /= tune;
7811
break;
7812
case ESTDFSCASIS:
7813
break;
7814
}
7815
*scp = sc;
7816
return hit;
7817
}
7818
7819
7820
/* Get the ID of a document specified by URI from pseudo indexes.
7821
`db' specifies a database object.
7822
`uri' specifies the URI of a registered document.
7823
The return value is the ID of the document. On error, -1 is returned. */
7824
static int est_pidx_uri_to_id(ESTDB *db, const char *uri){
7825
ESTDOC *doc;
7826
const char *vbuf;
7827
int i, vsiz;
7828
assert(db && uri);
7829
if(!db->puris){
7830
db->puris = cbmapopenex(CB_LISTNUM(db->pdocs) + 1);
7831
for(i = 0; i < CB_LISTNUM(db->pdocs); i++){
7832
if((doc = est_db_get_doc(db, ESTPDOCIDMIN + i, 0)) != NULL){
7833
if((vbuf = cbmapget(doc->attrs, ESTDATTRURI, -1, &vsiz)) != NULL)
7834
cbmapput(db->puris, vbuf, vsiz, (char *)&(doc->id), sizeof(int), FALSE);
7835
est_doc_delete(doc);
7836
}
7837
}
7838
}
7839
if((vbuf = cbmapget(db->puris, uri, -1, NULL)) != NULL) return *(int *)vbuf;
7840
return -1;
7841
}
7842
7843
3983
7844
/* Create a list of terms for search.
3984
7845
`phrase' specifies a search phrase.
3985
7846
The return value is a list object of the terms of the phrase. */
3990
7851
char *tbuf, *pbuf;
3991
7852
int i, tsiz, psiz, lw;
3992
7853
assert(phrase);
3993
terms = cblistopen();
7854
CB_LISTOPEN(terms);
3994
7855
tbuf = est_uconv_in(phrase, strlen(phrase), &tsiz);
3995
7856
est_normalize_text((unsigned char *)tbuf, tsiz, &tsiz);
3996
7857
pbuf = est_uconv_out(tbuf, tsiz, &psiz);
3997
7858
elems = cbsplit(pbuf, psiz, "\a\b\t\n\v\f\r ");
3998
datum = cbdatumopen("", 0);
7859
CB_DATUMOPEN(datum);
3999
7860
lw = FALSE;
4000
7861
for(i = 0; i < CB_LISTNUM(elems); i++){
4001
elem = CB_LISTVAL(elems, i, NULL);
7862
elem = CB_LISTVAL(elems, i);
4002
7863
if(elem[0] == '\0') continue;
4003
7864
if(!strcmp(elem, ESTOPUNION)){
4004
7865
if(CB_DATUMSIZE(datum) < 1) continue;
4005
if(lw) cbdatumcat(datum, "\t", -1);
7866
if(lw) CB_DATUMCAT(datum, "\t", 1);
4006
7867
lw = FALSE;
4007
7868
} else if(!strcmp(elem, ESTOPWCBW)){
4008
if(!lw) cbdatumcat(datum, " b", 2);
7869
if(!lw) CB_DATUMCAT(datum, " b", 2);
4009
7870
} else if(!strcmp(elem, ESTOPWCEW)){
4010
if(!lw) cbdatumcat(datum, " e", 2);
7871
if(!lw) CB_DATUMCAT(datum, " e", 2);
4011
7872
} else if(!strcmp(elem, ESTOPWCRX)){
4012
if(!lw) cbdatumcat(datum, " r", 2);
7873
if(!lw) CB_DATUMCAT(datum, " r", 2);
4013
7874
} else if(!strcmp(elem, ESTOPISECT) || !strcmp(elem, ESTOPDIFF)){
4014
7875
if(CB_DATUMSIZE(datum) < 1) continue;
4015
cblistpush(terms, CB_DATUMPTR(datum), CB_DATUMSIZE(datum));
4016
cbdatumsetsize(datum, 0);
4017
cblistpush(terms, elem, -1);
7876
CB_LISTPUSH(terms, CB_DATUMPTR(datum), CB_DATUMSIZE(datum));
7877
CB_DATUMSETSIZE(datum, 0);
7878
CB_LISTPUSH(terms, elem, strlen(elem));
4018
7879
lw = FALSE;
4019
7880
} else {
4020
if(CB_DATUMSIZE(datum) > 0 && lw) cbdatumcat(datum, " ", 1);
4021
cbdatumcat(datum, elem, -1);
7881
if(CB_DATUMSIZE(datum) > 0 && lw) CB_DATUMCAT(datum, " ", 1);
7882
CB_DATUMCAT(datum, elem, strlen(elem));
4022
7883
lw = TRUE;
4023
7884
}
4024
7885
}
4025
if(CB_DATUMSIZE(datum) > 0) cblistpush(terms, CB_DATUMPTR(datum), CB_DATUMSIZE(datum));
4026
cbdatumclose(datum);
4027
cblistclose(elems);
7886
if(CB_DATUMSIZE(datum) > 0) CB_LISTPUSH(terms, CB_DATUMPTR(datum), CB_DATUMSIZE(datum));
7887
CB_DATUMCLOSE(datum);
7888
CB_LISTCLOSE(elems);
4028
7889
free(pbuf);
4029
7890
free(tbuf);
4030
7891
for(i = 0; i < CB_LISTNUM(terms); i++){
4031
elem = CB_LISTVAL(terms, i, NULL);
7892
elem = CB_LISTVAL(terms, i);
4032
7893
if(!strcmp(elem, ESTOPUVSET) || !strcmp(elem, ESTOPISECT) ||
4033
7894
!strcmp(elem, ESTOPDIFF)) continue;
4034
7895
tbuf = est_uconv_in(elem, strlen(elem), &tsiz);
4039
7900
free(tbuf);
4040
7901
}
4041
7902
for(i = CB_LISTNUM(terms) - 1; i >= 0; i--){
4042
elem = CB_LISTVAL(terms, i, NULL);
7903
elem = CB_LISTVAL(terms, i);
4043
7904
if(strcmp(elem, ESTOPISECT) && strcmp(elem, ESTOPDIFF)) break;
4044
free(cblistpop(terms, NULL));
7905
CB_LISTDROP(terms);
4045
7906
}
4046
7907
return terms;
4047
7908
}
4048
7909
4049
7910
4050
/* Compare two scores by each ID.
7911
/* Compare two scores by each ID for ascending order.
4051
7912
`ap' specifies the pointer to one score.
4052
7913
`bp' specifies the pointer to the other score.
4053
7914
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
4054
static int est_score_compare_by_id(const void *ap, const void *bp){
7915
static int est_score_compare_by_id_asc(const void *ap, const void *bp){
4055
7916
assert(ap && bp);
4056
7917
return ((ESTSCORE *)ap)->id - ((ESTSCORE *)bp)->id;
4057
7918
}
4058
7919
4059
7920
4060
/* Compare two scores by each score point.
4061
`ap' specifies the pointer to one score.
4062
`bp' specifies the pointer to the other score.
4063
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
4064
static int est_score_compare_by_score(const void *ap, const void *bp){
7921
/* Compare two scores by each ID for descending order.
7922
`ap' specifies the pointer to one score.
7923
`bp' specifies the pointer to the other score.
7924
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
7925
static int est_score_compare_by_id_desc(const void *ap, const void *bp){
7926
assert(ap && bp);
7927
return ((ESTSCORE *)bp)->id - ((ESTSCORE *)ap)->id;
7928
}
7929
7930
7931
/* Compare two scores by each score point for ascending order.
7932
`ap' specifies the pointer to one score.
7933
`bp' specifies the pointer to the other score.
7934
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
7935
static int est_score_compare_by_score_asc(const void *ap, const void *bp){
7936
assert(ap && bp);
7937
return ((ESTSCORE *)ap)->score - ((ESTSCORE *)bp)->score;
7938
}
7939
7940
7941
/* Compare two scores by each score point for descending order.
7942
`ap' specifies the pointer to one score.
7943
`bp' specifies the pointer to the other score.
7944
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
7945
static int est_score_compare_by_score_desc(const void *ap, const void *bp){
4065
7946
assert(ap && bp);
4066
7947
return ((ESTSCORE *)bp)->score - ((ESTSCORE *)ap)->score;
4067
7948
}
4107
7988
}
4108
7989
4109
7990
7991
/* Compare two meta scores by each ID for ascending order.
7992
`ap' specifies the pointer to one meta score
7993
`bp' specifies the pointer to the other meta score
7994
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
7995
static int est_metascore_compare_by_id_asc(const void *ap, const void *bp){
7996
assert(ap && bp);
7997
return ((ESTMETASCORE *)ap)->id - ((ESTMETASCORE *)bp)->id;
7998
}
7999
8000
8001
/* Compare two meta scores by each ID for descending order.
8002
`ap' specifies the pointer to one meta score
8003
`bp' specifies the pointer to the other meta score
8004
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
8005
static int est_metascore_compare_by_id_desc(const void *ap, const void *bp){
8006
assert(ap && bp);
8007
return ((ESTMETASCORE *)bp)->id - ((ESTMETASCORE *)ap)->id;
8008
}
8009
8010
8011
/* Compare two meta scores by each score point for ascending order.
8012
`ap' specifies the pointer to one meta score
8013
`bp' specifies the pointer to the other meta score
8014
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
8015
static int est_metascore_compare_by_score_asc(const void *ap, const void *bp){
8016
assert(ap && bp);
8017
return ((ESTMETASCORE *)ap)->score - ((ESTMETASCORE *)bp)->score;
8018
}
8019
8020
8021
/* Compare two meta scores by each score point for descending order.
8022
`ap' specifies the pointer to one meta score
8023
`bp' specifies the pointer to the other meta score
8024
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
8025
static int est_metascore_compare_by_score_desc(const void *ap, const void *bp){
8026
assert(ap && bp);
8027
return ((ESTMETASCORE *)bp)->score - ((ESTMETASCORE *)ap)->score;
8028
}
8029
8030
8031
/* Compare two meta scores by attributes of strings for ascending order.
8032
`ap' specifies the pointer to one meta score
8033
`bp' specifies the pointer to the other meta score
8034
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
8035
static int est_metascore_compare_by_str_asc(const void *ap, const void *bp){
8036
assert(ap && bp);
8037
return strcmp(((ESTMETASCORE *)ap)->value, ((ESTMETASCORE *)bp)->value);
8038
}
8039
8040
8041
/* Compare two meta scores by attributes of strings for descending order.
8042
`ap' specifies the pointer to one meta score
8043
`bp' specifies the pointer to the other meta score
8044
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
8045
static int est_metascore_compare_by_str_desc(const void *ap, const void *bp){
8046
assert(ap && bp);
8047
return strcmp(((ESTMETASCORE *)bp)->value, ((ESTMETASCORE *)ap)->value);
8048
}
8049
8050
8051
/* Compare two meta scores by attributes of numbers for ascending order.
8052
`ap' specifies the pointer to one meta score
8053
`bp' specifies the pointer to the other meta score
8054
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
8055
static int est_metascore_compare_by_num_asc(const void *ap, const void *bp){
8056
assert(ap && bp);
8057
return (time_t)((ESTMETASCORE *)ap)->value - (time_t)((ESTMETASCORE *)bp)->value;
8058
}
8059
8060
8061
/* Compare two meta scores by attributes of numbers for descending order.
8062
`ap' specifies the pointer to one meta score
8063
`bp' specifies the pointer to the other meta score
8064
The return value is negative if one is small, positive if one is big, 0 if both are equal. */
8065
static int est_metascore_compare_by_num_desc(const void *ap, const void *bp){
8066
assert(ap && bp);
8067
return (time_t)((ESTMETASCORE *)bp)->value - (time_t)((ESTMETASCORE *)ap)->value;
8068
}
8069
8070
4110
8071
/* Get the universal set of documents in a database.
4111
8072
`db' specifies a database object.
4112
8073
`nump' specifies the pointer to which the number of elements in the result is assigned.
4113
8074
`hints' specifies a list object. If it is `NULL', it is not used.
4114
8075
`add' specifies whether the result to be treated in union or difference.
4115
The return value is an array whose elements are ID numbers of corresponding documents. */
8076
The return value is an array of score structures of corresponding documents. */
4116
8077
static ESTSCORE *est_search_uvset(ESTDB *db, int *nump, CBMAP *hints, int add){
4117
8078
ESTSCORE *scores;
4118
8079
char *vbuf, numbuf[ESTNUMBUFSIZ];
4129
8090
}
4130
8091
scores[snum].id = atoi(vbuf);
4131
8092
scores[snum].score = 0;
8093
scores[snum].value = NULL;
4132
8094
snum++;
4133
8095
free(vbuf);
4134
8096
vlcurnext(db->listdb);
4136
8098
*nump = snum;
4137
8099
if(hints){
4138
8100
sprintf(numbuf, "%d", snum * (add ? 1 : -1));
4139
cbmapput(hints, ESTOPUVSET, -1, numbuf, -1, FALSE);
8101
cbmapput(hints, ESTOPUVSET, -1, numbuf, -1, TRUE);
4140
8102
}
4141
8103
return scores;
4142
8104
}
4147
8109
`word' specifies a word.
4148
8110
`list' specifies a list object to contain the results. */
4149
8111
static void est_expand_word_bw(ESTDB *db, const char *word, CBLIST *list){
4150
char *kbuf;
4151
int ksiz;
8112
const char *kbuf;
8113
int num, ksiz;
4152
8114
assert(db && word && list);
8115
num = 0;
4153
8116
vlcurjump(db->fwmdb, word, -1, VL_JFORWARD);
4154
while((kbuf = vlcurkey(db->fwmdb, &ksiz)) != NULL){
4155
if(!cbstrfwmatch(kbuf, word)){
4156
free(kbuf);
4157
break;
4158
}
4159
cblistpushbuf(list, kbuf, ksiz);
8117
while((kbuf = vlcurkeycache(db->fwmdb, &ksiz)) != NULL){
8118
if(!cbstrfwmatch(kbuf, word)) break;
8119
CB_LISTPUSH(list, kbuf, ksiz);
8120
if(++num >= db->wildmax) break;
4160
8121
vlcurnext(db->fwmdb);
4161
8122
}
4162
8123
}
4167
8128
`word' specifies a word.
4168
8129
`list' specifies a list object to contain the results. */
4169
8130
static void est_expand_word_ew(ESTDB *db, const char *word, CBLIST *list){
4170
char *kbuf;
4171
int wsiz, ksiz;
8131
const char *kbuf;
8132
int num, wsiz, ksiz;
4172
8133
assert(db && word && list);
8134
num = 0;
4173
8135
wsiz = strlen(word);
4174
8136
vlcurfirst(db->fwmdb);
4175
while((kbuf = vlcurkey(db->fwmdb, &ksiz)) != NULL){
8137
while((kbuf = vlcurkeycache(db->fwmdb, &ksiz)) != NULL){
4176
8138
if(ksiz >= wsiz && !memcmp(kbuf + ksiz - wsiz, word, wsiz)){
4177
cblistpushbuf(list, kbuf, ksiz);
4178
} else {
4179
free(kbuf);
8139
CB_LISTPUSH(list, kbuf, ksiz);
8140
if(++num >= db->wildmax) break;
4180
8141
}
4181
8142
vlcurnext(db->fwmdb);
4182
8143
}
4189
8150
`list' specifies a list object to contain the results. */
4190
8151
static void est_expand_word_rx(ESTDB *db, const char *word, CBLIST *list){
4191
8152
void *regex;
4192
char *kbuf;
4193
int ksiz;
8153
const char *kbuf;
8154
int num, ksiz;
4194
8155
assert(db && word && list);
4195
8156
if(!(regex = est_regex_new(word))) return;
8157
num = 0;
4196
8158
vlcurfirst(db->fwmdb);
4197
while((kbuf = vlcurkey(db->fwmdb, &ksiz)) != NULL){
8159
while((kbuf = vlcurkeycache(db->fwmdb, &ksiz)) != NULL){
4198
8160
if(est_regex_match(regex, kbuf)){
4199
cblistpushbuf(list, kbuf, ksiz);
4200
} else {
4201
free(kbuf);
8161
CB_LISTPUSH(list, kbuf, ksiz);
8162
if(++num >= db->wildmax) break;
4202
8163
}
4203
8164
vlcurnext(db->fwmdb);
4204
8165
}
4206
8167
}
4207
8168
4208
8169
8170
/* Expand a keyword to keywords which begins with it.
8171
`db' specifies a database object.
8172
`word' specifies a word.
8173
`list' specifies a list object to contain the results. */
8174
static void est_expand_keyword_bw(ESTDB *db, const char *word, CBLIST *list){
8175
const char *kbuf;
8176
int num, ksiz;
8177
assert(db && word && list);
8178
num = 0;
8179
vlcurjump(db->xfmdb, word, -1, VL_JFORWARD);
8180
while((kbuf = vlcurkeycache(db->xfmdb, &ksiz)) != NULL){
8181
if(!cbstrfwmatch(kbuf, word)) break;
8182
CB_LISTPUSH(list, kbuf, ksiz);
8183
if(++num >= db->wildmax) break;
8184
vlcurnext(db->xfmdb);
8185
}
8186
}
8187
8188
8189
/* Expand a keyword to keywords which ends with it.
8190
`db' specifies a database object.
8191
`word' specifies a word.
8192
`list' specifies a list object to contain the results. */
8193
static void est_expand_keyword_ew(ESTDB *db, const char *word, CBLIST *list){
8194
const char *kbuf;
8195
int num, wsiz, ksiz;
8196
assert(db && word && list);
8197
num = 0;
8198
wsiz = strlen(word);
8199
vlcurfirst(db->xfmdb);
8200
while((kbuf = vlcurkeycache(db->xfmdb, &ksiz)) != NULL){
8201
if(ksiz >= wsiz && !memcmp(kbuf + ksiz - wsiz, word, wsiz)){
8202
CB_LISTPUSH(list, kbuf, ksiz);
8203
if(++num >= db->wildmax) break;
8204
}
8205
vlcurnext(db->xfmdb);
8206
}
8207
}
8208
8209
8210
/* Expand regular expressios to keywords which matches them.
8211
`db' specifies a database object.
8212
`word' specifies regular expressions.
8213
`list' specifies a list object to contain the results. */
8214
static void est_expand_keyword_rx(ESTDB *db, const char *word, CBLIST *list){
8215
void *regex;
8216
const char *kbuf;
8217
int num, ksiz;
8218
assert(db && word && list);
8219
if(!(regex = est_regex_new(word))) return;
8220
num = 0;
8221
vlcurfirst(db->xfmdb);
8222
while((kbuf = vlcurkeycache(db->xfmdb, &ksiz)) != NULL){
8223
if(est_regex_match(regex, kbuf)){
8224
CB_LISTPUSH(list, kbuf, ksiz);
8225
if(++num >= db->wildmax) break;
8226
}
8227
vlcurnext(db->xfmdb);
8228
}
8229
est_regex_delete(regex);
8230
}
8231
8232
4209
8233
/* Get a correspinding set of documents in a database.
4210
8234
`db' specifies a database object.
4211
8235
`term' specifies a union term.
4212
8236
`gstep' specifies number of steps of N-gram.
8237
`xpn' specifies the pointer to a function for query expansion. If it is `NULL', it is not
8238
used.
4213
8239
`nump' specifies the pointer to which the number of elements in the result is assigned.
4214
8240
`hints' specifies a list object. If it is `NULL', it is not used.
4215
8241
`add' specifies whether the result to be treated in union or difference.
4216
The return value is an array whose elements are ID numbers of corresponding documents. */
8242
`auxmin' specifies the minimum hits to adopt the auxiliary index. If it is not more than 0,
8243
the auxiliary index is not used.
8244
`auxwords' specifies a map object where keywords used with the auxiliary index are stored. If
8245
it is `NULL', it is not used.
8246
The return value is an array of score structures of corresponding documents. */
4217
8247
static ESTSCORE *est_search_union(ESTDB *db, const char *term, int gstep,
4218
int *nump, CBMAP *hints, int add){
8248
void (*xpn)(const char *, CBLIST *),
8249
int *nump, CBMAP *hints, int add, int auxmin, CBMAP *auxwords){
4219
8250
const ESTSCORE *cscores;
4220
8251
ESTSCORE *scores, *tscores;
8252
CBMAP *umap;
4221
8253
CBLIST *words, *grams;
4222
8254
const char *ckey, *word, *gram, *rp, *fnext, *snext, *cbuf;
4223
char *vbuf, numbuf[ESTNUMBUFSIZ];
8255
char *vbuf, *wbuf, numbuf[ESTNUMBUFSIZ];
4224
8256
int i, j, k, snum, smax, cksiz, single, tsmax, tsnum, vsiz, gcnum, gsiz, csiz, wgstep, nnum;
4225
int west, wild, mfsiz, mssiz, mfhash, mshash, tfhash, tshash, id, score, hit, hnum;
8257
int west, wild, mfsiz, mssiz, mfhash, mshash, tfhash, tshash, id, vstep, score, hit, hnum;
8258
double avg, sd, dif;
4226
8259
assert(db && term && gstep > 0 && nump);
4227
8260
smax = ESTALLOCUNIT;
4228
8261
CB_MALLOC(scores, smax * sizeof(ESTSCORE));
4229
8262
snum = 0;
4230
8263
words = cbsplit(term, -1, "\t");
8264
if(xpn){
8265
umap = cbmapopenex(ESTMINIBNUM);
8266
for(i = 0; i < CB_LISTNUM(words); i++){
8267
word = CB_LISTVAL(words, i);
8268
if(word[0] == '\0' || word[0] == ' ') continue;
8269
CB_LISTOPEN(grams);
8270
xpn(word, grams);
8271
for(j = 0; j < CB_LISTNUM(grams); j++){
8272
word = CB_LISTVAL(grams, j);
8273
cbmapput(umap, word, -1, "", 0, FALSE);
8274
}
8275
CB_LISTCLOSE(grams);
8276
}
8277
CB_LISTCLOSE(words);
8278
words = cbmapkeys(umap);
8279
cbmapclose(umap);
8280
}
4231
8281
for(i = 0; i < CB_LISTNUM(words); i++){
4232
ckey = CB_LISTVAL2(words, i, &cksiz);
8282
ckey = CB_LISTVAL2(words, i, cksiz);
8283
if(cksiz < 1) continue;
4233
8284
word = ckey;
8285
wbuf = NULL;
4234
8286
if((cscores = est_rescc_get(db, ckey, cksiz, &tsnum)) != NULL){
8287
if(word[0] == ' '){
8288
word++;
8289
if(word[0] != '\0') word++;
8290
}
4235
8291
if(hints){
4236
8292
sprintf(numbuf, "%d", tsnum * (add ? 1 : -1));
4237
cbmapput(hints, word, -1, numbuf, -1, FALSE);
8293
cbmapput(hints, word, -1, numbuf, -1, TRUE);
4238
8294
}
4239
8295
for(j = 0; j < tsnum; j++){
4240
8296
if(snum >= smax){
4245
8301
scores[snum].score = cscores[j].score;
4246
8302
snum++;
4247
8303
}
8304
} else if(!strchr(word + 1, ' ') && auxmin > 0 &&
8305
(tscores = est_search_keywords(db, word, auxmin, &tsnum)) != NULL){
8306
if(word[0] == ' '){
8307
word++;
8308
if(word[0] != '\0') word++;
8309
}
8310
if(hints){
8311
sprintf(numbuf, "%d", tsnum * (add ? 1 : -1));
8312
cbmapput(hints, word, -1, numbuf, -1, TRUE);
8313
}
8314
if(auxwords) cbmapput(auxwords, word, -1, "", 0, FALSE);
8315
for(j = 0; j < tsnum; j++){
8316
if(snum >= smax){
8317
smax *= 2;
8318
CB_REALLOC(scores, smax * sizeof(ESTSCORE));
8319
}
8320
scores[snum].id = tscores[j].id;
8321
scores[snum].score = tscores[j].score;
8322
snum++;
8323
}
8324
free(tscores);
4248
8325
} else {
4249
8326
wild = '\0';
4250
8327
if(word[0] == ' '){
4263
8340
west = ((unsigned char *)word)[0] <= 0xdf;
4264
8341
if(!west || db->amode) wild = '\0';
4265
8342
single = FALSE;
4266
grams = cblistopen();
8343
CB_LISTOPEN(grams);
4267
8344
switch(wild){
4268
8345
case 'b':
4269
8346
est_break_text(word, grams, TRUE, FALSE);
4270
cblistpush(grams, word, -1);
8347
CB_LISTPUSH(grams, word, strlen(word));
4271
8348
while(CB_LISTNUM(grams) > 1){
4272
free(cblistpop(grams, NULL));
8349
CB_LISTDROP(grams);
4273
8350
}
4274
word = CB_LISTVAL(grams, 0, NULL);
8351
wbuf = cbmemdup(CB_LISTVAL(grams, 0), -1);
8352
word = wbuf;
4275
8353
est_expand_word_bw(db, word, grams);
4276
8354
single = TRUE;
4277
8355
break;
4281
8359
while(CB_LISTNUM(grams) > 1){
4282
8360
free(cblistshift(grams, NULL));
4283
8361
}
4284
word = CB_LISTVAL(grams, 0, NULL);
8362
wbuf = cbmemdup(CB_LISTVAL(grams, 0), -1);
8363
word = wbuf;
4285
8364
est_expand_word_ew(db, word, grams);
4286
8365
single = TRUE;
4287
8366
break;
4295
8374
break;
4296
8375
default:
4297
8376
switch(db->amode){
4298
case ESTAMPERFNG:
8377
case ESTDFPERFNG:
4299
8378
est_break_text_perfng(word, grams, TRUE, FALSE);
4300
8379
break;
8380
case ESTDFCHRCAT:
8381
est_break_text_chrcat(word, grams, TRUE);
8382
break;
4301
8383
default:
4302
8384
est_break_text(word, grams, TRUE, FALSE);
4303
8385
break;
4316
8398
if(west && gstep <= 2) wgstep = 1;
4317
8399
for(j = 0; j < CB_LISTNUM(grams); j += wgstep){
4318
8400
gcnum++;
4319
gram = CB_LISTVAL2(grams, j, &gsiz);
8401
gram = CB_LISTVAL2(grams, j, gsiz);
4320
8402
fnext = cblistval(grams, j + 1, &mfsiz);
4321
8403
snext = cblistval(grams, j + 2, &mssiz);
4322
8404
mfhash = fnext ? dpinnerhash(fnext, mfsiz) % ESTJHASHNUM + 1: 0xff;
4323
8405
mshash = snext ? dpouterhash(snext, mssiz) % ESTJHASHNUM + 1: 0xff;
4324
vbuf = est_idx_get(db->idxdb, gram, gsiz, &vsiz);
8406
vbuf = est_idx_scan(db->idxdb, gram, gsiz, &vsiz, db->smode);
4325
8407
if((cbuf = cbmapget(db->idxcc, gram, gsiz, &csiz)) != NULL){
4326
if(vbuf){
4327
CB_REALLOC(vbuf, vsiz + csiz + 100);
4328
memcpy(vbuf + vsiz, cbuf, csiz);
4329
vsiz += csiz;
4330
} else {
4331
vbuf = cbmemdup(cbuf, csiz);
4332
vsiz = csiz;
4333
}
8408
CB_REALLOC(vbuf, vsiz + csiz + 1);
8409
memcpy(vbuf + vsiz, cbuf, csiz);
8410
vsiz += csiz;
4334
8411
}
4335
if(!vbuf) continue;
4336
8412
rp = vbuf;
4337
8413
while(rp < vbuf + vsiz){
4338
memcpy(&id, rp, sizeof(int));
4339
rp += sizeof(int);
4340
score = *(unsigned char *)rp + 1;
4341
rp++;
8414
EST_READ_VNUMBUF(rp, id, vstep);
8415
rp += vstep;
8416
switch(db->smode){
8417
case ESTDFSCVOID:
8418
score = 0;
8419
break;
8420
default:
8421
score = *(unsigned char *)rp;
8422
rp++;
8423
break;
8424
case ESTDFSCINT:
8425
case ESTDFSCASIS:
8426
memcpy(&score, rp, sizeof(int));
8427
rp += sizeof(int);
8428
break;
8429
}
4342
8430
hit = mfhash == 0xff && mshash == 0xff;
4343
while(rp < vbuf + vsiz){
8431
while(rp < vbuf + vsiz && *(unsigned char *)rp != 0x00){
4344
8432
tfhash = *(unsigned char *)rp;
4345
8433
rp++;
4346
8434
tshash = *(unsigned char *)rp;
4347
8435
rp++;
4348
8436
if((mfhash == 0xff || mfhash == tfhash) && (mshash == 0xff || mshash == tshash))
4349
8437
hit = TRUE;
4350
if(*(unsigned char *)rp == 0x00){
4351
rp++;
4352
break;
4353
}
4354
8438
}
8439
rp++;
4355
8440
if(hit || single){
4356
8441
if(tsnum >= tsmax){
4357
8442
tsmax *= 2;
4358
8443
CB_REALLOC(tscores, tsmax * sizeof(ESTSCORE));
4359
8444
}
4360
8445
tscores[tsnum].id = id;
4361
tscores[tsnum].score = score * 100;
8446
switch(db->smode){
8447
case ESTDFSCVOID:
8448
tscores[tsnum].score = rp - vbuf;
8449
break;
8450
default:
8451
tscores[tsnum].score = score * 100 + 10;
8452
break;
8453
case ESTDFSCASIS:
8454
tscores[tsnum].score = score;
8455
break;
8456
}
4362
8457
tsnum++;
4363
8458
}
4364
8459
}
4365
8460
free(vbuf);
4366
8461
}
4367
8462
if(gcnum > 1){
4368
qsort(tscores, tsnum, sizeof(ESTSCORE), est_score_compare_by_id);
8463
qsort(tscores, tsnum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
4369
8464
nnum = 0;
4370
8465
for(j = 0; j < tsnum; j++){
4371
8466
id = tscores[j].id;
4372
8467
score = tscores[j].score;
4373
8468
hnum = 1;
4374
for(k = j + 1; k < tsnum && tscores[k].id == id; k++){
4375
score += tscores[k].score;
4376
hnum++;
4377
}
4378
if(hnum >= gcnum || single){
4379
tscores[nnum].id = id;
4380
tscores[nnum].score = score / hnum;
4381
nnum++;
8469
if(db->smode == ESTDFSCASIS){
8470
for(k = j + 1; k < tsnum && tscores[k].id == id; k++){
8471
hnum++;
8472
}
8473
if(hnum >= gcnum || single){
8474
tscores[nnum].id = id;
8475
tscores[nnum].score = score;
8476
nnum++;
8477
}
8478
} else {
8479
for(k = j + 1; k < tsnum && tscores[k].id == id; k++){
8480
score += tscores[k].score;
8481
hnum++;
8482
}
8483
if(hnum >= gcnum || single){
8484
tscores[nnum].id = id;
8485
tscores[nnum].score = score / hnum;
8486
nnum++;
8487
}
4382
8488
}
4383
8489
j = k - 1;
4384
8490
}
4386
8492
}
4387
8493
if(hints){
4388
8494
sprintf(numbuf, "%d", tsnum * (add ? 1 : -1));
4389
cbmapput(hints, word, -1, numbuf, -1, FALSE);
8495
cbmapput(hints, word, -1, numbuf, -1, TRUE);
4390
8496
}
4391
cblistclose(grams);
8497
CB_LISTCLOSE(grams);
8498
if(!strchr(word, ' ') && auxmin > 0) est_weight_keywords(db, word, tscores, tsnum);
4392
8499
for(j = 0; j < tsnum; j++){
4393
8500
if(snum >= smax){
4394
8501
smax *= 2;
4400
8507
}
4401
8508
est_rescc_put(db, ckey, cksiz, tscores, tsnum);
4402
8509
}
8510
free(wbuf);
4403
8511
}
4404
cblistclose(words);
4405
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id);
8512
CB_LISTCLOSE(words);
8513
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
4406
8514
nnum = 0;
4407
8515
for(i = 0; i < snum; i++){
4408
8516
id = scores[i].id;
4414
8522
}
4415
8523
scores[nnum].id = id;
4416
8524
scores[nnum].score = score / hnum;
8525
scores[nnum].value = NULL;
4417
8526
nnum++;
4418
8527
i = j - 1;
4419
8528
}
4420
8529
*nump = nnum;
8530
if(db->smode != ESTDFSCASIS && nnum > 0){
8531
avg = 0.0;
8532
for(i = 0; i < nnum; i++){
8533
avg += scores[i].score;
8534
}
8535
avg /= nnum;
8536
sd = 0.0;
8537
for(i = 0; i < nnum; i++){
8538
dif = avg - scores[i].score;
8539
sd += dif * dif;
8540
}
8541
sd /= nnum;
8542
sd = sqrt(sd);
8543
if(sd < 0.1){
8544
for(i = 0; i < nnum; i++){
8545
scores[i].score = ESTSCOREUNIT / 2;
8546
}
8547
} else {
8548
for(i = 0; i < nnum; i++){
8549
scores[i].score = (int)(((scores[i].score - avg) * (ESTSCOREUNIT / 10.0) / sd) +
8550
ESTSCOREUNIT / 2.0);
8551
}
8552
}
8553
}
4421
8554
return scores;
4422
8555
}
4423
8556
4449
8582
static void est_rescc_put(ESTDB *db, const char *word, int size, ESTSCORE *scores, int num){
4450
8583
int i;
4451
8584
assert(db && word && size >= 0 && scores && num >= 0);
4452
cbmapputvbuf(db->rescc, word, size, (char *)scores, num * sizeof(ESTSCORE));
8585
if(db->rcmnum < 1) return;
8586
cbmapput(db->rescc, word, size, (char *)scores, num * sizeof(ESTSCORE), TRUE);
8587
free(scores);
4453
8588
if(cbmaprnum(db->rescc) > db->rcmnum){
4454
8589
num = db->rcmnum * 0.1 + 1;
4455
8590
cbmapiterinit(db->rescc);
4460
8595
}
4461
8596
4462
8597
8598
/* Search the auxiliary index.
8599
`db' specifies a database object.
8600
`word' specifies a search word.
8601
`min' specifies the minimum hits to adopt the auxiliary index.
8602
`nump' specifies the pointer to which the number of elements in the result is assigned.
8603
The return value is an array of score structures of corresponding documents. */
8604
static ESTSCORE *est_search_keywords(ESTDB *db, const char *word, int min, int *nump){
8605
ESTSCORE *scores;
8606
CBLIST *words;
8607
CBDATUM *rbuf;
8608
const int *res;
8609
int i, rnum, snum, wsiz, nnum, lid;
8610
assert(db && word && min >= 0 && nump);
8611
if(*word != ' ' && (res = (int *)vlgetcache(db->auxdb, word, -1, &rnum)) != NULL &&
8612
(rnum /= sizeof(int)) / 2 >= min){
8613
CB_MALLOC(scores, (rnum / 2) * sizeof(ESTSCORE) + 1);
8614
snum = 0;
8615
for(i = 0; i < rnum; i += 2){
8616
scores[snum].id = res[i];
8617
scores[snum].score = res[i+1];
8618
snum++;
8619
}
8620
*nump = snum;
8621
return scores;
8622
}
8623
CB_LISTOPEN(words);
8624
if(*word == ' '){
8625
word++;
8626
if(*word == 'b'){
8627
est_expand_keyword_bw(db, word + 1, words);
8628
} else if(*word == 'e'){
8629
est_expand_keyword_ew(db, word + 1, words);
8630
} else if(*word == 'r'){
8631
est_expand_keyword_rx(db, word + 1, words);
8632
}
8633
} else if(*(unsigned char *)word >= 0xe3){
8634
est_expand_keyword_bw(db, word, words);
8635
}
8636
CB_DATUMOPEN(rbuf);
8637
for(i = 0; i < CB_LISTNUM(words) &&
8638
CB_DATUMSIZE(rbuf) <= sizeof(int) * 2 * min * ESTAUXEXRAT; i++){
8639
word = CB_LISTVAL2(words, i, wsiz);
8640
if(!(res = (int *)vlgetcache(db->auxdb, word, wsiz, &rnum))) continue;
8641
CB_DATUMCAT(rbuf, (char *)res, rnum);
8642
}
8643
res = (int *)CB_DATUMPTR(rbuf);
8644
rnum = CB_DATUMSIZE(rbuf);
8645
if((rnum /= sizeof(int)) / 2 < min){
8646
CB_DATUMCLOSE(rbuf);
8647
CB_LISTCLOSE(words);
8648
return NULL;
8649
}
8650
CB_MALLOC(scores, (rnum / 2) * sizeof(ESTSCORE) + 1);
8651
snum = 0;
8652
for(i = 0; i < rnum; i += 2){
8653
scores[snum].id = res[i];
8654
scores[snum].score = res[i+1];
8655
snum++;
8656
}
8657
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
8658
nnum = 0;
8659
lid = -1;
8660
for(i = 0; i < snum; i++){
8661
if(nnum > 0 && scores[i].id == lid){
8662
scores[nnum-1].score += scores[i].score;
8663
continue;
8664
}
8665
scores[nnum].id = scores[i].id;
8666
scores[nnum].score = scores[i].score;
8667
nnum++;
8668
lid = scores[i].id;
8669
}
8670
CB_DATUMCLOSE(rbuf);
8671
CB_LISTCLOSE(words);
8672
*nump = nnum;
8673
return scores;
8674
}
8675
8676
8677
/* Weight scores with the auxiliary index.
8678
`db' specifies a database object.
8679
`word' specifies a search word.
8680
`scores' specifies an array of scores of search candidates.
8681
`snum' specifies the number of the array. */
8682
static void est_weight_keywords(ESTDB *db, const char *word, ESTSCORE *scores, int snum){
8683
ESTSCORE *kscores;
8684
const int *res;
8685
int i, knum, nnum;
8686
double rank;
8687
if(!(res = (int *)vlgetcache(db->auxdb, word, -1, &knum)) || knum < 2) return;
8688
knum /= sizeof(int);
8689
CB_MALLOC(kscores, knum / 2 * sizeof(ESTSCORE));
8690
rank = knum / 2 + 1;
8691
nnum = 0;
8692
for(i = 0; i < knum; i += 2){
8693
kscores[nnum].id = res[i];
8694
kscores[nnum].score = (pow(rank, 0.7) / 8.0 + 1.0) * 10000.0;
8695
nnum++;
8696
rank -= 1.0;
8697
}
8698
knum = nnum;
8699
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
8700
qsort(kscores, knum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
8701
nnum = 0;
8702
for(i = 0; i < snum; i++){
8703
while(nnum < knum && kscores[nnum].id < scores[i].id){
8704
nnum++;
8705
}
8706
if(nnum < knum && kscores[nnum].id == scores[i].id)
8707
scores[i].score *= kscores[nnum].score / 10000.0;
8708
}
8709
free(kscores);
8710
}
8711
8712
8713
/* Get scores correspinding a ranking search with an attribute narrowing index.
8714
`db' specifies a database object.
8715
`name' specifies the name of an attribute.
8716
`nump' specifies the pointer to which the number of elements in the result is assigned.
8717
The return value is an array of score structures of corresponding documents. */
8718
static ESTSCORE *est_search_rank(ESTDB *db, const char *name, int top, int *nump){
8719
ESTATTRIDX *attridx;
8720
ESTSCORE *scores;
8721
const char *kbuf;
8722
int snum, ksiz, id;
8723
assert(db && name && nump);
8724
if(top == 0 || !(attridx = (ESTATTRIDX *)cbmapget(db->aidxs, name, -1, NULL)) ||
8725
(attridx->type != ESTIDXATTRSTR && attridx->type != ESTIDXATTRNUM)){
8726
*nump = 0;
8727
return cbmalloc(1);
8728
}
8729
snum = abs(top);
8730
if(snum > db->dnum) snum = db->dnum;
8731
CB_MALLOC(scores, snum * sizeof(ESTSCORE) + 1);
8732
snum = 0;
8733
if(top > 0){
8734
vlcurfirst(attridx->db);
8735
while(snum < top && (kbuf = vlcurkeycache(attridx->db, &ksiz)) != NULL){
8736
if(ksiz < sizeof(int)){
8737
vlcurnext(attridx->db);
8738
continue;
8739
}
8740
memcpy(&id, kbuf + ksiz - sizeof(int), sizeof(int));
8741
if(id < 1){
8742
vlcurnext(attridx->db);
8743
continue;
8744
}
8745
scores[snum].id = id;
8746
scores[snum].score = 0;
8747
scores[snum].value = NULL;
8748
snum++;
8749
vlcurnext(attridx->db);
8750
}
8751
} else {
8752
top *= -1;
8753
vlcurlast(attridx->db);
8754
while(snum < top && (kbuf = vlcurkeycache(attridx->db, &ksiz)) != NULL){
8755
if(ksiz < sizeof(int)){
8756
vlcurprev(attridx->db);
8757
continue;
8758
}
8759
memcpy(&id, kbuf + ksiz - sizeof(int), sizeof(int));
8760
if(id < 1){
8761
vlcurprev(attridx->db);
8762
continue;
8763
}
8764
scores[snum].id = id;
8765
scores[snum].score = 0;
8766
scores[snum].value = NULL;
8767
snum++;
8768
vlcurprev(attridx->db);
8769
}
8770
}
8771
*nump = snum;
8772
return scores;
8773
}
8774
8775
8776
/* Get scores correspinding an attribute expression with an attribute narrowing index.
8777
`db' specifies a database object.
8778
`expr' specifies an attribute search expression.
8779
`nump' specifies the pointer to which the number of elements in the result is assigned.
8780
The return value is an array of score structures of corresponding documents or `NULL' if no
8781
index is available. */
8782
static ESTSCORE *est_search_aidx_attr(ESTDB *db, const char *expr, int *nump){
8783
ESTATTRIDX *attridx;
8784
ESTSCORE *scores;
8785
CBDATUM *abuf;
8786
CBLIST *tokens;
8787
void *regex;
8788
const char *cop, *pv, *kbuf, *tbuf;
8789
unsigned char *utmp;
8790
char *name, *oper, *val, *sval, *wp, numbuf[ESTNUMBUFSIZ];
8791
int i, nsiz, vsiz, ksiz, tsiz, sign, ic, ssiz, esc, jmp, len, *ary, anum;
8792
time_t num, lower, upper;
8793
assert(db && expr && nump);
8794
name = NULL;
8795
oper = NULL;
8796
val = NULL;
8797
nsiz = 0;
8798
vsiz = 0;
8799
while(*expr > 0 && *expr <= ' '){
8800
expr++;
8801
}
8802
if((pv = strchr(expr, ' ')) != NULL){
8803
nsiz = pv - expr;
8804
name = cbmemdup(expr, nsiz);
8805
expr = pv;
8806
while(*expr > 0 && *expr <= ' '){
8807
expr++;
8808
}
8809
if((pv = strchr(expr, ' ')) != NULL){
8810
oper = cbmemdup(expr, pv - expr);
8811
expr = pv;
8812
while(*expr > 0 && *expr <= ' '){
8813
expr++;
8814
}
8815
vsiz = strlen(expr);
8816
val = cbmemdup(expr, vsiz);
8817
} else {
8818
oper = cbmemdup(expr, -1);
8819
}
8820
} else {
8821
nsiz = strlen(expr);
8822
name = cbmemdup(expr, nsiz);
8823
}
8824
if(!oper){
8825
oper = cbmemdup("", 0);
8826
}
8827
if(!val){
8828
vsiz = 0;
8829
val = cbmemdup("", 0);
8830
}
8831
cop = oper;
8832
if(*cop == '!'){
8833
sign = FALSE;
8834
cop++;
8835
} else {
8836
sign = TRUE;
8837
}
8838
if(*cop == 'I' || *cop == 'i'){
8839
ic = !est_check_cjk_only(val);
8840
cop++;
8841
} else {
8842
ic = FALSE;
8843
}
8844
regex = NULL;
8845
if(!cbstricmp(cop, ESTOPSTREQ)){
8846
cop = ESTOPSTREQ;
8847
} else if(!cbstricmp(cop, ESTOPSTRNE)){
8848
cop = ESTOPSTRNE;
8849
} else if(!cbstricmp(cop, ESTOPSTRINC)){
8850
cop = ESTOPSTRINC;
8851
} else if(!cbstricmp(cop, ESTOPSTRBW)){
8852
cop = ESTOPSTRBW;
8853
} else if(!cbstricmp(cop, ESTOPSTREW)){
8854
cop = ESTOPSTREW;
8855
} else if(!cbstricmp(cop, ESTOPSTRAND)){
8856
cop = ESTOPSTRAND;
8857
} else if(!cbstricmp(cop, ESTOPSTROR)){
8858
cop = ESTOPSTROR;
8859
} else if(!cbstricmp(cop, ESTOPSTROREQ)){
8860
cop = ESTOPSTROREQ;
8861
} else if(!cbstricmp(cop, ESTOPSTRRX)){
8862
cop = ESTOPSTRRX;
8863
regex = est_regex_new(val);
8864
} else if(!cbstricmp(cop, ESTOPNUMEQ)){
8865
cop = ESTOPNUMEQ;
8866
} else if(!cbstricmp(cop, ESTOPNUMNE)){
8867
cop = ESTOPNUMNE;
8868
} else if(!cbstricmp(cop, ESTOPNUMGT)){
8869
cop = ESTOPNUMGT;
8870
} else if(!cbstricmp(cop, ESTOPNUMGE)){
8871
cop = ESTOPNUMGE;
8872
} else if(!cbstricmp(cop, ESTOPNUMLT)){
8873
cop = ESTOPNUMLT;
8874
} else if(!cbstricmp(cop, ESTOPNUMLE)){
8875
cop = ESTOPNUMLE;
8876
} else if(!cbstricmp(cop, ESTOPNUMBT)){
8877
cop = ESTOPNUMBT;
8878
} else {
8879
cop = ESTOPSTRINC;
8880
val[0] = '\0';
8881
vsiz = 0;
8882
}
8883
num = cbstrmktime(val);
8884
if(!(attridx = (ESTATTRIDX *)cbmapget(db->aidxs, name, nsiz, NULL)) ||
8885
(attridx->type != ESTIDXATTRSTR && attridx->type != ESTIDXATTRNUM) ||
8886
(attridx->type == ESTIDXATTRNUM &&
8887
cop != ESTOPNUMEQ && cop != ESTOPNUMNE && cop != ESTOPNUMGT && cop != ESTOPNUMGE &&
8888
cop != ESTOPNUMLT && cop != ESTOPNUMLE && cop != ESTOPNUMBT)){
8889
if(regex) est_regex_delete(regex);
8890
free(val);
8891
free(oper);
8892
free(name);
8893
return NULL;
8894
}
8895
CB_DATUMOPEN(abuf);
8896
if(!sign || ic){
8897
if(ic){
8898
utmp = (unsigned char *)est_uconv_in(val, vsiz, &tsiz);
8899
est_normalize_text(utmp, tsiz, &tsiz);
8900
est_canonicalize_text(utmp, tsiz, FALSE);
8901
sval = (char *)est_uconv_out((char *)utmp, tsiz, &ssiz);
8902
free(utmp);
8903
} else {
8904
sval = NULL;
8905
ssiz = 0;
8906
}
8907
esc = INT_MAX;
8908
jmp = INT_MAX;
8909
if(sign && (cop == ESTOPSTREQ || cop == ESTOPSTRBW) && vsiz > 0){
8910
if(*sval > 0x0 && *sval < 0x7f){
8911
numbuf[0] = *sval;
8912
numbuf[1] = '\0';
8913
esc = *(unsigned char *)sval;
8914
if(*sval >= 'a' && *sval <= 'z'){
8915
numbuf[0] -= 'a' - 'A';
8916
jmp = *sval - 'a' + 'A';
8917
}
8918
vlcurjump(attridx->db, numbuf, 1, VL_JFORWARD);
8919
} else if(*(unsigned char *)sval >= 0xc0){
8920
numbuf[0] = *sval;
8921
numbuf[1] = '\0';
8922
esc = *(unsigned char *)sval;
8923
vlcurjump(attridx->db, numbuf, 1, VL_JFORWARD);
8924
} else {
8925
vlcurfirst(attridx->db);
8926
}
8927
} else {
8928
vlcurfirst(attridx->db);
8929
}
8930
while((kbuf = vlcurkeycache(attridx->db, &ksiz)) != NULL){
8931
if(est_match_attr(kbuf, ksiz - sizeof(int) - 1,
8932
cop, sign, val, vsiz, sval, ssiz, regex, num))
8933
CB_DATUMCAT(abuf, kbuf + ksiz - sizeof(int), sizeof(int));
8934
if(*(unsigned char *)kbuf > jmp && *(unsigned char *)kbuf < *(unsigned char *)sval){
8935
numbuf[0] = *sval;
8936
numbuf[1] = '\0';
8937
vlcurjump(attridx->db, numbuf, 1, VL_JFORWARD);
8938
jmp = INT_MAX;
8939
} else if(*(unsigned char *)kbuf > esc){
8940
break;
8941
} else {
8942
vlcurnext(attridx->db);
8943
}
8944
}
8945
if(sval) free(sval);
8946
} else if(cop == ESTOPSTROREQ){
8947
tokens = cbsplit(val, vsiz, " ,");
8948
cblistsort(tokens);
8949
for(i = 0; i < CB_LISTNUM(tokens); i++){
8950
tbuf = CB_LISTVAL2(tokens, i, tsiz);
8951
vlcurjump(attridx->db, tbuf, tsiz, VL_JFORWARD);
8952
while((kbuf = vlcurkeycache(attridx->db, &ksiz)) != NULL && !strcmp(kbuf, tbuf)){
8953
CB_DATUMCAT(abuf, kbuf + ksiz - sizeof(int), sizeof(int));
8954
vlcurnext(attridx->db);
8955
}
8956
}
8957
CB_LISTCLOSE(tokens);
8958
} else if(cop == ESTOPNUMBT){
8959
if((wp = strchr(val, ' ')) != NULL || (wp = strchr(val, '\t')) != NULL){
8960
*(wp++) = '\0';
8961
while(*wp == ' ' || *wp == '\t'){
8962
wp++;
8963
}
8964
lower = cbstrmktime(val);
8965
upper = cbstrmktime(wp);
8966
} else {
8967
lower = cbstrmktime(val);
8968
upper = INT_MAX;
8969
}
8970
len = sprintf(numbuf, "%.0f", (double)lower);
8971
vlcurjump(attridx->db, numbuf, len, VL_JFORWARD);
8972
while((kbuf = vlcurkeycache(attridx->db, &ksiz)) != NULL && cbstrmktime(kbuf) <= upper){
8973
CB_DATUMCAT(abuf, kbuf + ksiz - sizeof(int), sizeof(int));
8974
vlcurnext(attridx->db);
8975
}
8976
} else {
8977
if(cop == ESTOPSTREQ || cop == ESTOPSTRBW ||
8978
cop == ESTOPNUMEQ || cop == ESTOPNUMGT || cop == ESTOPNUMGE){
8979
vlcurjump(attridx->db, val, vsiz, VL_JFORWARD);
8980
if(cop == ESTOPNUMGT){
8981
while((kbuf = vlcurkeycache(attridx->db, NULL)) != NULL && cbstrmktime(kbuf) <= num){
8982
vlcurnext(attridx->db);
8983
}
8984
}
8985
} else if(cop == ESTOPNUMLT || cop == ESTOPNUMLE){
8986
len = sprintf(numbuf, "%.0f", (double)cbstrmktime(val) + 1);
8987
vlcurjump(attridx->db, numbuf, len, VL_JBACKWARD);
8988
if(cop == ESTOPNUMLT){
8989
while((kbuf = vlcurkeycache(attridx->db, NULL)) != NULL && cbstrmktime(kbuf) >= num){
8990
vlcurprev(attridx->db);
8991
}
8992
}
8993
} else {
8994
vlcurfirst(attridx->db);
8995
}
8996
while((kbuf = vlcurkeycache(attridx->db, &ksiz)) != NULL){
8997
if(est_match_attr(kbuf, ksiz - sizeof(int) - 1,
8998
cop, TRUE, val, vsiz, NULL, 0, regex, num)){
8999
CB_DATUMCAT(abuf, kbuf + ksiz - sizeof(int), sizeof(int));
9000
} else if(cop == ESTOPSTREQ || cop == ESTOPSTRBW || cop == ESTOPNUMEQ){
9001
break;
9002
}
9003
if(cop == ESTOPNUMLT || cop == ESTOPNUMLE){
9004
vlcurprev(attridx->db);
9005
} else {
9006
vlcurnext(attridx->db);
9007
}
9008
}
9009
}
9010
ary = (int *)CB_DATUMPTR(abuf);
9011
anum = CB_DATUMSIZE(abuf) / sizeof(int);
9012
CB_MALLOC(scores, anum * sizeof(ESTSCORE) + 1);
9013
for(i = 0; i < anum; i++){
9014
scores[i].id = ary[i];
9015
scores[i].score = 0;
9016
scores[i].value = NULL;
9017
}
9018
*nump = anum;
9019
CB_DATUMCLOSE(abuf);
9020
if(regex) est_regex_delete(regex);
9021
free(val);
9022
free(oper);
9023
free(name);
9024
return scores;
9025
}
9026
9027
9028
/* Get a correspinding set of documents in pseudo indexes.
9029
`db' specifies a database object.
9030
`cond' specifies a search condition object.
9031
`scores' specifies an array of scores of search candidates.
9032
`nump' specifies the pointer to which the number of elements in the parameter and result is
9033
assigned.
9034
`ordattrs' specifies a map object into which ordering attributes are stored.
9035
The return value is an array of re-allocated score structures. */
9036
static ESTSCORE *est_search_pidxs(ESTDB *db, ESTCOND *cond, ESTSCORE *scores, int *nump,
9037
CBMAP *ordattrs){
9038
ESTCATTR *list;
9039
ESTDOC *doc;
9040
const char *otype, *lbuf, *vbuf;
9041
char *oname, *wp;
9042
int i, j, k, snum, anum, id, hit, sc, miss, lsiz, vsiz;
9043
double avg, sd, dif, tune;
9044
assert(db && cond && scores && nump && ordattrs);
9045
snum = *nump;
9046
CB_REALLOC(scores, (snum + CB_LISTNUM(db->pdocs)) * sizeof(ESTSCORE) + 1);
9047
if(cond->phrase){
9048
if(cbstrfwmatch(cond->phrase, ESTOPID)){
9049
return scores;
9050
} else if(cbstrfwmatch(cond->phrase, ESTOPURI)){
9051
return scores;
9052
} else if(cbstrfwmatch(cond->phrase, ESTOPSIMILAR)){
9053
return scores;
9054
}
9055
}
9056
oname = NULL;
9057
otype = NULL;
9058
if(cond->order){
9059
oname = cbmemdup(cond->order, -1);
9060
cbstrtrim(oname);
9061
otype = ESTORDSTRA;
9062
if((wp = strchr(oname, ' ')) != NULL){
9063
*(wp++) = '\0';
9064
while(*wp == ' '){
9065
wp++;
9066
}
9067
otype = wp;
9068
}
9069
}
9070
list = NULL;
9071
anum = -1;
9072
if(cond->attrs) list = est_make_cattr_list(cond->attrs, &anum);
9073
for(i = 0; i < CB_LISTNUM(db->pdocs); i++){
9074
id = ESTPDOCIDMIN + i;
9075
hit = FALSE;
9076
sc = 0;
9077
doc = NULL;
9078
if(!cond->phrase || cond->phrase[0] == '\0'){
9079
hit = cond->attrs ? TRUE : FALSE;
9080
} else if(cbstrfwmatch(cond->phrase, ESTOPUVSET)){
9081
hit = TRUE;
9082
} else {
9083
if((doc = est_db_get_doc(db, id, 0)) != NULL){
9084
hit = est_db_score_doc(db, doc, cond, &sc);
9085
} else {
9086
hit = FALSE;
9087
}
9088
}
9089
if(hit && list){
9090
if(!doc && !(doc = est_db_get_doc(db, id, 0))){
9091
hit = FALSE;
9092
} else {
9093
miss = FALSE;
9094
for(j = 0; !miss && j < anum; j++){
9095
if(list[j].nsiz < 1) continue;
9096
if(list[j].nlist){
9097
hit = FALSE;
9098
for(k = 0; k < CB_LISTNUM(list[j].nlist); k++){
9099
lbuf = CB_LISTVAL2(list[j].nlist, k, lsiz);
9100
if(lsiz < 1) continue;
9101
if(!(vbuf = cbmapget(doc->attrs, lbuf, lsiz, &vsiz))) continue;
9102
if(est_match_attr(vbuf, vsiz, list[j].cop, list[j].sign, list[j].val, list[j].vsiz,
9103
list[j].sval, list[j].ssiz, list[j].regex, list[j].num)){
9104
hit = TRUE;
9105
break;
9106
}
9107
}
9108
if(!hit) miss = TRUE;
9109
} else if(!(vbuf = cbmapget(doc->attrs, list[j].name, list[j].nsiz, &vsiz))){
9110
miss = TRUE;
9111
} else if(!est_match_attr(vbuf, vsiz, list[j].cop, list[j].sign,
9112
list[j].val, list[j].vsiz, list[j].sval, list[j].ssiz,
9113
list[j].regex, list[j].num)){
9114
miss = TRUE;
9115
}
9116
}
9117
hit = !miss;
9118
}
9119
}
9120
if(hit){
9121
scores[snum].id = id;
9122
scores[snum].score = sc;
9123
scores[snum].value = NULL;
9124
snum++;
9125
if(oname && (doc || (doc = est_db_get_doc(db, id, 0)) != NULL)){
9126
if(!(vbuf = cbmapget(doc->attrs, oname, -1, &vsiz))){
9127
vbuf = "";
9128
vsiz = 0;
9129
}
9130
cbmapput(ordattrs, (char *)&id, sizeof(int), vbuf, vsiz, FALSE);
9131
}
9132
}
9133
if(doc) est_doc_delete(doc);
9134
}
9135
if(list) est_free_cattr_list(list, anum);
9136
if(oname) free(oname);
9137
if(db->smode != ESTDFSCASIS && snum > *nump){
9138
avg = 0.0;
9139
for(i = *nump; i < snum; i++){
9140
avg += scores[i].score;
9141
}
9142
avg /= snum - *nump;
9143
sd = 0.0;
9144
for(i = *nump; i < snum; i++){
9145
dif = avg - scores[i].score;
9146
sd += dif * dif;
9147
}
9148
sd /= snum - *nump;
9149
sd = sqrt(sd);
9150
if(sd < 0.1){
9151
for(i = *nump; i < snum; i++){
9152
scores[i].score = ESTSCOREUNIT / 2;
9153
}
9154
} else {
9155
for(i = *nump; i < snum; i++){
9156
scores[i].score = (int)(((scores[i].score - avg) * (ESTSCOREUNIT / 10.0) / sd) +
9157
ESTSCOREUNIT / 2.0);
9158
}
9159
}
9160
if(cond->tfidf){
9161
tune = pow(snum - *nump + 64, 0.4);
9162
for(i = *nump; i < snum; i++){
9163
scores[i].score *= 100.0 / tune;
9164
}
9165
} else {
9166
for(i = *nump; i < snum; i++){
9167
scores[i].score *= 10;
9168
}
9169
}
9170
}
9171
*nump = snum;
9172
return scores;
9173
}
9174
9175
4463
9176
/* Narrow and sort scores of search candidates.
4464
9177
`db' specifies a database object.
4465
9178
`attrs' specifies a list object of narrowing attributes.
9179
`ign' specifies the offset of an attribute to be ignored.
4466
9180
`order' specifies an expression for sorting.
9181
`distinct' specifies the name of the distinct attribute.
4467
9182
`scores' specifies an array of scores of search candidates.
4468
9183
`snum' specifies the number of the array.
4469
9184
`limit' specifies the limit number to check.
4470
9185
`restp' specifies the pointer to a variable to which rest number to be checked is assigned.
9186
`ordattrs' specifies a map object of cached ordering attributes.
4471
9187
The return value is the new number of the array. */
4472
static int est_narrow_scores(ESTDB *db, const CBLIST *attrs, const char *order,
4473
ESTSCORE *scores, int snum, int limit, int *restp){
9188
static int est_narrow_scores(ESTDB *db, const CBLIST *attrs, int ign,
9189
const char *order, const char *distinct, ESTSCORE *scores, int snum,
9190
int limit, int *restp, CBMAP *ordattrs){
4474
9191
ESTCATTR *list;
4475
const char *otype, *cbuf, *rp, *pv, *ibuf;
4476
unsigned char *utmp;
9192
ESTATTRIDX *attridx;
9193
CBMAP *umap;
9194
const char *otype, *cbuf, *ibuf, *lbuf;
4477
9195
char *oname, *wp, *mbuf, *vbuf;
4478
int i, j, k, ci, oi, anum, tsiz, nnum, csiz, msiz, miss, vsiz, num, isiz, onlen;
9196
int i, j, k, ci, oi, anum, done, mixed, nnum, csiz, msiz;
9197
int miss, vsiz, num, isiz, lsiz, hit, onlen, dnlen;
4479
9198
time_t tval;
4480
assert(db && scores && snum >= 0 && restp);
9199
assert(db && scores && snum >= 0 && limit >= 0 && restp && ordattrs);
4481
9200
*restp = 0;
4482
9201
ci = -1;
4483
9202
oi = -1;
4488
9207
cbstrtrim(oname);
4489
9208
otype = ESTORDSTRA;
4490
9209
if((wp = strchr(oname, ' ')) != NULL){
4491
*wp = '\0';
4492
rp = wp + 1;
4493
while(*rp == ' '){
4494
rp++;
9210
*(wp++) = '\0';
9211
while(*wp == ' '){
9212
wp++;
4495
9213
}
4496
otype = rp;
9214
otype = wp;
4497
9215
}
4498
9216
}
4499
9217
if(attrs){
4500
anum = CB_LISTNUM(attrs);
4501
CB_MALLOC(list, sizeof(ESTCATTR) * anum + 1);
4502
for(i = 0; i < anum; i++){
4503
list[i].name = NULL;
4504
list[i].oper = NULL;
4505
list[i].val = NULL;
4506
rp = CB_LISTVAL(attrs, i, NULL);
4507
while(*rp > 0 && *rp <= ' '){
4508
rp++;
4509
}
4510
if((pv = strchr(rp, ' ')) != NULL){
4511
list[i].nsiz = pv - rp;
4512
list[i].name = cbmemdup(rp, list[i].nsiz);
4513
rp = pv;
4514
while(*rp > 0 && *rp <= ' '){
4515
rp++;
9218
list = est_make_cattr_list(attrs, &anum);
9219
if(cbmaprnum(db->aidxs) > 0){
9220
done = TRUE;
9221
mixed = FALSE;
9222
for(i = 0; i < anum; i++){
9223
if(i == ign) continue;
9224
if(!(attridx = (ESTATTRIDX *)cbmapget(db->aidxs, list[i].name, list[i].nsiz, NULL)) ||
9225
(attridx->type == ESTIDXATTRNUM &&
9226
list[i].cop != ESTOPNUMEQ && list[i].cop != ESTOPNUMNE &&
9227
list[i].cop != ESTOPNUMGT && list[i].cop != ESTOPNUMGE &&
9228
list[i].cop != ESTOPNUMLT && list[i].cop != ESTOPNUMLE &&
9229
list[i].cop != ESTOPNUMBT)){
9230
done = FALSE;
9231
continue;
4516
9232
}
4517
if((pv = strchr(rp, ' ')) != NULL){
4518
list[i].oper = cbmemdup(rp, pv - rp);
4519
rp = pv;
4520
while(*rp > 0 && *rp <= ' '){
4521
rp++;
9233
switch(attridx->type){
9234
case ESTIDXATTRSTR:
9235
case ESTIDXATTRNUM:
9236
snum = est_aidx_attr_narrow(attridx->db, db->pdocs, list[i].cop, list[i].sign,
9237
list[i].val, list[i].vsiz, list[i].sval, list[i].ssiz,
9238
list[i].regex, list[i].num, scores, snum);
9239
mixed = TRUE;
9240
break;
9241
default:
9242
if(done && i == anum - 1 && !order && mixed){
9243
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score_desc);
9244
mixed = FALSE;
4522
9245
}
4523
list[i].vsiz = strlen(rp);
4524
list[i].val = cbmemdup(rp, list[i].vsiz);
4525
} else {
4526
list[i].oper = cbmemdup(rp, -1);
9246
snum = est_aidx_seq_narrow(attridx->db, db->pdocs, list[i].cop, list[i].sign,
9247
list[i].val, list[i].vsiz, list[i].sval, list[i].ssiz,
9248
list[i].regex, list[i].num, scores, snum,
9249
done && i == anum - 1 ? limit : INT_MAX, restp);
9250
break;
4527
9251
}
4528
} else {
4529
list[i].nsiz = strlen(rp);
4530
list[i].name = cbmemdup(rp, list[i].nsiz);
4531
}
4532
if(!list[i].oper){
4533
list[i].oper = cbmemdup("", 0);
4534
}
4535
if(!list[i].val){
4536
list[i].vsiz = 0;
4537
list[i].val = cbmemdup("", 0);
4538
}
4539
}
4540
for(i = 0; i < anum; i++){
4541
rp = list[i].oper;
4542
if(*rp == '!'){
4543
list[i].sign = FALSE;
4544
rp++;
4545
} else {
4546
list[i].sign = TRUE;
4547
}
4548
if(*rp == 'I' || *rp == 'i'){
4549
utmp = (unsigned char *)est_uconv_in(list[i].val, list[i].vsiz, &tsiz);
4550
est_normalize_text(utmp, tsiz, &tsiz);
4551
est_canonicalize_text(utmp, tsiz, FALSE);
4552
list[i].sval = (char *)est_uconv_out((char *)utmp, tsiz, &(list[i].ssiz));
4553
free(utmp);
4554
rp++;
4555
} else {
4556
list[i].sval = NULL;
4557
list[i].ssiz = 0;
4558
}
4559
list[i].regex = NULL;
4560
list[i].num = cbstrmktime(list[i].val);
4561
if(!cbstricmp(rp, ESTOPSTREQ)){
4562
list[i].cop = ESTOPSTREQ;
4563
} else if(!cbstricmp(rp, ESTOPSTRNE)){
4564
list[i].cop = ESTOPSTRNE;
4565
} else if(!cbstricmp(rp, ESTOPSTRINC)){
4566
list[i].cop = ESTOPSTRINC;
4567
} else if(!cbstricmp(rp, ESTOPSTRBW)){
4568
list[i].cop = ESTOPSTRBW;
4569
} else if(!cbstricmp(rp, ESTOPSTREW)){
4570
list[i].cop = ESTOPSTREW;
4571
} else if(!cbstricmp(rp, ESTOPSTRAND)){
4572
list[i].cop = ESTOPSTRAND;
4573
} else if(!cbstricmp(rp, ESTOPSTROR)){
4574
list[i].cop = ESTOPSTROR;
4575
} else if(!cbstricmp(rp, ESTOPSTRRX)){
4576
list[i].cop = ESTOPSTRRX;
4577
list[i].regex = list[i].sval ? est_regex_new(list[i].sval) : est_regex_new(list[i].val);
4578
} else if(!cbstricmp(rp, ESTOPNUMEQ)){
4579
list[i].cop = ESTOPNUMEQ;
4580
} else if(!cbstricmp(rp, ESTOPNUMNE)){
4581
list[i].cop = ESTOPNUMNE;
4582
} else if(!cbstricmp(rp, ESTOPNUMGT)){
4583
list[i].cop = ESTOPNUMGT;
4584
} else if(!cbstricmp(rp, ESTOPNUMGE)){
4585
list[i].cop = ESTOPNUMGE;
4586
} else if(!cbstricmp(rp, ESTOPNUMLT)){
4587
list[i].cop = ESTOPNUMLT;
4588
} else if(!cbstricmp(rp, ESTOPNUMLE)){
4589
list[i].cop = ESTOPNUMLE;
4590
} else if(!cbstricmp(rp, ESTOPNUMBT)){
4591
list[i].cop = ESTOPNUMBT;
4592
} else {
4593
list[i].cop = NULL;
4594
}
9252
list[i].cop = ESTOPDUMMY;
9253
}
9254
if(mixed && !order) qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score_desc);
9255
} else {
9256
done = FALSE;
4595
9257
}
4596
9258
if(db->spacc){
4597
9259
for(i = 0; i < anum; i++){
4609
9271
}
4610
9272
}
4611
9273
}
4612
nnum = 0;
9274
if(!done){
9275
nnum = 0;
9276
for(i = 0; i < snum; i++){
9277
if(nnum >= limit){
9278
*restp = snum - i;
9279
break;
9280
}
9281
scores[i].value = NULL;
9282
if(ci >= 0){
9283
if((cbuf = cbmapget(db->spacc, (char *)&(scores[i].id), sizeof(int), &csiz)) != NULL)
9284
cbmapmove(db->spacc, (char *)&(scores[i].id), sizeof(int), FALSE);
9285
} else {
9286
cbuf = NULL;
9287
csiz = 0;
9288
}
9289
mbuf = NULL;
9290
if(scores[i].id >= ESTPDOCIDMIN){
9291
scores[nnum++] = scores[i];
9292
} else if((cbuf && anum == 1) ||
9293
(mbuf = est_crget(db->attrdb, db->zmode, scores[i].id, &msiz)) != NULL){
9294
miss = FALSE;
9295
for(j = 0; !miss && j < anum; j++){
9296
if(list[j].nsiz < 1) continue;
9297
if(list[j].nlist){
9298
hit = FALSE;
9299
for(k = 0; k < CB_LISTNUM(list[j].nlist); k++){
9300
lbuf = CB_LISTVAL2(list[j].nlist, k, lsiz);
9301
if(lsiz < 1) continue;
9302
if(!(vbuf = cbmaploadone(mbuf, msiz, lbuf, lsiz, &vsiz))) continue;
9303
if(est_match_attr(vbuf, vsiz, list[j].cop, list[j].sign, list[j].val, list[j].vsiz,
9304
list[j].sval, list[j].ssiz, list[j].regex, list[j].num)){
9305
hit = TRUE;
9306
free(vbuf);
9307
break;
9308
}
9309
free(vbuf);
9310
}
9311
if(!hit) miss = TRUE;
9312
vbuf = NULL;
9313
} else {
9314
if(mbuf){
9315
vbuf = cbmaploadone(mbuf, msiz, list[j].name, list[j].nsiz, &vsiz);
9316
} else if(csiz != 1 || cbuf[0] != '\0'){
9317
vbuf = cbmemdup(cbuf, csiz);
9318
vsiz = csiz;
9319
} else {
9320
vbuf = NULL;
9321
}
9322
if(list[j].oper[0] == '\0'){
9323
if(!vbuf) miss = TRUE;
9324
} else {
9325
if(!vbuf){
9326
vbuf = cbmemdup("", 0);
9327
vsiz = 0;
9328
}
9329
if(!est_match_attr(vbuf, vsiz, list[j].cop, list[j].sign,
9330
list[j].val, list[j].vsiz, list[j].sval, list[j].ssiz,
9331
list[j].regex, list[j].num)) miss = TRUE;
9332
}
9333
}
9334
if(j == ci && !cbuf){
9335
if(vbuf){
9336
cbmapput(db->spacc, (char *)&(scores[i].id), sizeof(int), vbuf, vsiz, FALSE);
9337
} else {
9338
cbmapput(db->spacc, (char *)&(scores[i].id), sizeof(int), "", 1, FALSE);
9339
}
9340
if(cbmaprnum(db->spacc) > db->scmnum){
9341
num = db->scmnum * 0.1 + 1;
9342
cbmapiterinit(db->spacc);
9343
for(k = 0; k < num && (ibuf = cbmapiternext(db->spacc, &isiz)) != NULL; k++){
9344
cbmapout(db->spacc, ibuf, isiz);
9345
}
9346
}
9347
}
9348
if(j == oi){
9349
scores[i].value = vbuf;
9350
} else {
9351
free(vbuf);
9352
}
9353
}
9354
if(miss){
9355
free(scores[i].value);
9356
} else {
9357
scores[nnum++] = scores[i];
9358
}
9359
}
9360
free(mbuf);
9361
}
9362
snum = nnum;
9363
} else {
9364
for(i = 0; i < snum; i++){
9365
scores[i].value = NULL;
9366
}
9367
}
9368
est_free_cattr_list(list, anum);
9369
} else {
4613
9370
for(i = 0; i < snum; i++){
4614
if(nnum >= limit){
4615
*restp = snum - i;
4616
break;
4617
}
4618
9371
scores[i].value = NULL;
4619
if(ci >= 0){
4620
if((cbuf = cbmapget(db->spacc, (char *)&(scores[i].id), sizeof(int), &csiz)) != NULL)
9372
}
9373
}
9374
if(oname){
9375
if(!cbstricmp(oname, ESTORDIDA)){
9376
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
9377
} else if(!cbstricmp(oname, ESTORDIDD)){
9378
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id_desc);
9379
} else if(!cbstricmp(oname, ESTORDSCA)){
9380
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score_asc);
9381
} else if(!cbstricmp(oname, ESTORDSCD)){
9382
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score_desc);
9383
} else {
9384
ci = db->spacc && !strcmp(oname, db->scname);
9385
onlen = strlen(oname);
9386
attridx = (ESTATTRIDX *)cbmapget(db->aidxs, oname, onlen, NULL);
9387
if(attridx && attridx->type != ESTIDXATTRSEQ) attridx = NULL;
9388
for(i = 0; i < snum; i++){
9389
if(scores[i].value) continue;
9390
if(ci &&
9391
(cbuf = cbmapget(db->spacc, (char *)&(scores[i].id), sizeof(int), &csiz)) != NULL){
4621
9392
cbmapmove(db->spacc, (char *)&(scores[i].id), sizeof(int), FALSE);
4622
} else {
4623
cbuf = NULL;
4624
csiz = 0;
4625
}
4626
mbuf = NULL;
4627
if((cbuf && anum == 1) ||
4628
(mbuf = crget(db->attrdb, (char *)&(scores[i].id), sizeof(int), 0, -1, &msiz)) != NULL){
4629
miss = FALSE;
4630
for(j = 0; !miss && j < anum; j++){
4631
if(list[j].nsiz < 1) continue;
4632
if(mbuf){
4633
vbuf = cbmaploadone(mbuf, msiz, list[j].name, list[j].nsiz, &vsiz);
4634
} else if(csiz != 1 || cbuf[0] != '\0'){
4635
vbuf = cbmemdup(cbuf, csiz);
4636
vsiz = csiz;
4637
} else {
4638
vbuf = NULL;
4639
}
4640
if(list[j].oper[0] == '\0'){
4641
if(!vbuf) miss = TRUE;
4642
} else {
4643
if(!vbuf){
4644
vbuf = cbmemdup("", 0);
4645
vsiz = 0;
4646
}
4647
if(!est_match_attr(vbuf, vsiz, list[j].cop, list[j].sign, list[j].val, list[j].vsiz,
4648
list[j].sval, list[j].ssiz, list[j].regex, list[j].num))
4649
miss = TRUE;
4650
}
4651
if(j == ci && !cbuf){
4652
if(vbuf){
4653
cbmapput(db->spacc, (char *)&(scores[i].id), sizeof(int), vbuf, vsiz, FALSE);
4654
} else {
4655
cbmapput(db->spacc, (char *)&(scores[i].id), sizeof(int), "", 1, FALSE);
4656
}
4657
if(cbmaprnum(db->spacc) > db->scmnum){
4658
num = db->scmnum * 0.1 + 1;
4659
cbmapiterinit(db->spacc);
4660
for(k = 0; k < num && (ibuf = cbmapiternext(db->spacc, &isiz)) != NULL; k++){
4661
cbmapout(db->spacc, ibuf, isiz);
4662
}
4663
}
4664
}
4665
if(j == oi){
4666
scores[i].value = vbuf;
4667
} else {
4668
free(vbuf);
4669
}
4670
}
4671
if(miss){
4672
free(scores[i].value);
4673
} else {
4674
scores[nnum++] = scores[i];
4675
}
4676
}
4677
free(mbuf);
4678
}
4679
snum = nnum;
4680
for(i = 0; i < anum; i++){
4681
if(list[i].regex) est_regex_delete(list[i].regex);
4682
free(list[i].sval);
4683
free(list[i].val);
4684
free(list[i].oper);
4685
free(list[i].name);
4686
}
4687
free(list);
4688
} else {
4689
for(i = 0; i < snum; i++){
4690
scores[i].value = NULL;
4691
}
4692
}
4693
if(oname){
4694
ci = db->spacc && !strcmp(oname, db->scname);
4695
onlen = strlen(oname);
4696
for(i = 0; i < snum; i++){
4697
if(scores[i].value) continue;
4698
if(ci && (cbuf = cbmapget(db->spacc, (char *)&(scores[i].id), sizeof(int), &csiz)) != NULL){
4699
cbmapmove(db->spacc, (char *)&(scores[i].id), sizeof(int), FALSE);
4700
if(csiz == 1 && cbuf[0] == '\0'){
4701
scores[i].value = cbmemdup("", 0);
4702
} else {
9393
if(csiz == 1 && cbuf[0] == '\0'){
9394
scores[i].value = cbmemdup("", 0);
9395
} else {
9396
scores[i].value = cbmemdup(cbuf, csiz);
9397
}
9398
continue;
9399
}
9400
if((cbuf = cbmapget(ordattrs, (char *)&(scores[i].id), sizeof(int), &csiz)) != NULL){
4703
9401
scores[i].value = cbmemdup(cbuf, csiz);
9402
continue;
4704
9403
}
4705
continue;
4706
}
4707
if((mbuf = crget(db->attrdb, (char *)&(scores[i].id), sizeof(int), 0, -1, &msiz)) != NULL){
4708
if((vbuf = cbmaploadone(mbuf, msiz, oname, onlen, &vsiz)) != NULL){
4709
if(ci) cbmapput(db->spacc, (char *)&(scores[i].id), sizeof(int), vbuf, vsiz, FALSE);
9404
if(attridx){
9405
if(!(vbuf = est_aidx_seq_get(attridx->db, scores[i].id, &vsiz))) vbuf = cbmemdup("", 0);
4710
9406
scores[i].value = vbuf;
9407
continue;
9408
}
9409
if((mbuf = est_crget(db->attrdb, db->zmode, scores[i].id, &msiz)) != NULL){
9410
if((vbuf = cbmaploadone(mbuf, msiz, oname, onlen, &vsiz)) != NULL){
9411
if(ci) cbmapput(db->spacc, (char *)&(scores[i].id), sizeof(int), vbuf, vsiz, FALSE);
9412
scores[i].value = vbuf;
9413
} else {
9414
if(ci) cbmapput(db->spacc, (char *)&(scores[i].id), sizeof(int), "", 1, FALSE);
9415
scores[i].value = cbmemdup("", 0);
9416
}
9417
if(ci && cbmaprnum(db->spacc) > db->scmnum){
9418
num = db->scmnum * 0.1 + 1;
9419
cbmapiterinit(db->spacc);
9420
for(j = 0; j < num && (ibuf = cbmapiternext(db->spacc, &isiz)) != NULL; j++){
9421
cbmapout(db->spacc, ibuf, isiz);
9422
}
9423
}
9424
free(mbuf);
4711
9425
} else {
4712
if(ci) cbmapput(db->spacc, (char *)&(scores[i].id), sizeof(int), "", 1, FALSE);
4713
9426
scores[i].value = cbmemdup("", 0);
4714
9427
}
4715
if(ci && cbmaprnum(db->spacc) > db->scmnum){
4716
num = db->scmnum * 0.1 + 1;
4717
cbmapiterinit(db->spacc);
4718
for(j = 0; j < num && (ibuf = cbmapiternext(db->spacc, &isiz)) != NULL; j++){
4719
cbmapout(db->spacc, ibuf, isiz);
4720
}
4721
}
4722
free(mbuf);
9428
}
9429
if(!cbstricmp(otype, ESTORDSTRA)){
9430
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_str_asc);
9431
} else if(!cbstricmp(otype, ESTORDSTRD)){
9432
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_str_desc);
9433
} else if(!cbstricmp(otype, ESTORDNUMA)){
9434
for(i = 0; i < snum; i++){
9435
tval = cbstrmktime(scores[i].value);
9436
free(scores[i].value);
9437
scores[i].value = (void *)tval;
9438
}
9439
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_num_asc);
9440
for(i = 0; i < snum; i++){
9441
scores[i].value = NULL;
9442
}
9443
} else if(!cbstricmp(otype, ESTORDNUMD)){
9444
for(i = 0; i < snum; i++){
9445
tval = cbstrmktime(scores[i].value);
9446
free(scores[i].value);
9447
scores[i].value = (void *)tval;
9448
}
9449
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_num_desc);
9450
for(i = 0; i < snum; i++){
9451
scores[i].value = NULL;
9452
}
9453
}
9454
for(i = 0; i < snum; i++){
9455
free(scores[i].value);
9456
}
9457
}
9458
free(oname);
9459
}
9460
if(distinct){
9461
dnlen = strlen(distinct);
9462
umap = cbmapopenex(snum + 1);
9463
attridx = (ESTATTRIDX *)cbmapget(db->aidxs, distinct, dnlen, NULL);
9464
if(attridx && attridx->type != ESTIDXATTRSEQ) attridx = NULL;
9465
nnum = 0;
9466
for(i = 0; i < snum; i++){
9467
if(scores[i].id >= ESTPDOCIDMIN){
9468
if(!(vbuf = est_db_get_doc_attr(db, scores[i].id, distinct))) vbuf = cbmemdup("", 0);
9469
vsiz = strlen(vbuf);
9470
} else if(attridx){
9471
if(!(vbuf = est_aidx_seq_get(attridx->db, scores[i].id, &vsiz))){
9472
vbuf = cbmemdup("", 0);
9473
vsiz = 0;
9474
}
4723
9475
} else {
4724
scores[i].value = cbmemdup("", 0);
4725
}
4726
}
4727
if(!cbstricmp(otype, ESTORDSTRA)){
4728
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_str_asc);
4729
} else if(!cbstricmp(otype, ESTORDSTRD)){
4730
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_str_desc);
4731
} else if(!cbstricmp(otype, ESTORDNUMA)){
4732
for(i = 0; i < snum; i++){
4733
tval = cbstrmktime(scores[i].value);
4734
free(scores[i].value);
4735
scores[i].value = (void *)tval;
4736
}
4737
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_num_asc);
4738
for(i = 0; i < snum; i++){
4739
scores[i].value = NULL;
4740
}
4741
} else if(!cbstricmp(otype, ESTORDNUMD)){
4742
for(i = 0; i < snum; i++){
4743
tval = cbstrmktime(scores[i].value);
4744
free(scores[i].value);
4745
scores[i].value = (void *)tval;
4746
}
4747
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_num_desc);
4748
for(i = 0; i < snum; i++){
4749
scores[i].value = NULL;
4750
}
4751
}
4752
for(i = 0; i < snum; i++){
4753
free(scores[i].value);
4754
}
4755
free(oname);
9476
if((mbuf = est_crget(db->attrdb, db->zmode, scores[i].id, &msiz)) != NULL){
9477
if(!(vbuf = cbmaploadone(mbuf, msiz, distinct, dnlen, &vsiz))){
9478
vbuf = cbmemdup("", 0);
9479
vsiz = 0;
9480
}
9481
free(mbuf);
9482
} else {
9483
vbuf = cbmemdup("", 0);
9484
vsiz = 0;
9485
}
9486
}
9487
if(cbmapput(umap, vbuf, vsiz, "", 0, FALSE)) scores[nnum++] = scores[i];
9488
free(vbuf);
9489
}
9490
snum = nnum;
9491
cbmapclose(umap);
4756
9492
}
4757
9493
return snum;
4758
9494
}
4759
9495
4760
9496
9497
/* Make a list of condition attributes.
9498
`attrs' specifies a list object of attribute expressions.
9499
`nump' specifies the pointer to which the number of elements in the result is assigned.
9500
The return value is a list of condition attributes. */
9501
static ESTCATTR *est_make_cattr_list(const CBLIST *attrs, int *nump){
9502
ESTCATTR *list;
9503
const char *rp, *pv;
9504
unsigned char *utmp;
9505
int i, anum, tsiz;
9506
assert(attrs && nump);
9507
anum = CB_LISTNUM(attrs);
9508
CB_MALLOC(list, sizeof(ESTCATTR) * anum + 1);
9509
for(i = 0; i < anum; i++){
9510
list[i].name = NULL;
9511
list[i].oper = NULL;
9512
list[i].val = NULL;
9513
rp = CB_LISTVAL(attrs, i);
9514
while(*rp > 0 && *rp <= ' '){
9515
rp++;
9516
}
9517
if((pv = strchr(rp, ' ')) != NULL){
9518
list[i].nsiz = pv - rp;
9519
list[i].name = cbmemdup(rp, list[i].nsiz);
9520
rp = pv;
9521
while(*rp > 0 && *rp <= ' '){
9522
rp++;
9523
}
9524
if((pv = strchr(rp, ' ')) != NULL){
9525
list[i].oper = cbmemdup(rp, pv - rp);
9526
rp = pv;
9527
while(*rp > 0 && *rp <= ' '){
9528
rp++;
9529
}
9530
list[i].vsiz = strlen(rp);
9531
list[i].val = cbmemdup(rp, list[i].vsiz);
9532
} else {
9533
list[i].oper = cbmemdup(rp, -1);
9534
}
9535
} else {
9536
list[i].nsiz = strlen(rp);
9537
list[i].name = cbmemdup(rp, list[i].nsiz);
9538
}
9539
if(strchr(list[i].name, ',')){
9540
list[i].nlist = cbsplit(list[i].name, list[i].nsiz, ",");
9541
} else {
9542
list[i].nlist = NULL;
9543
}
9544
if(!list[i].oper){
9545
list[i].oper = cbmemdup("", 0);
9546
}
9547
if(!list[i].val){
9548
list[i].vsiz = 0;
9549
list[i].val = cbmemdup("", 0);
9550
}
9551
}
9552
for(i = 0; i < anum; i++){
9553
rp = list[i].oper;
9554
if(*rp == '!'){
9555
list[i].sign = FALSE;
9556
rp++;
9557
} else {
9558
list[i].sign = TRUE;
9559
}
9560
if(*rp == 'I' || *rp == 'i'){
9561
if(est_check_cjk_only(list[i].val)){
9562
list[i].sval = NULL;
9563
list[i].ssiz = 0;
9564
} else {
9565
utmp = (unsigned char *)est_uconv_in(list[i].val, list[i].vsiz, &tsiz);
9566
est_normalize_text(utmp, tsiz, &tsiz);
9567
est_canonicalize_text(utmp, tsiz, FALSE);
9568
list[i].sval = (char *)est_uconv_out((char *)utmp, tsiz, &(list[i].ssiz));
9569
free(utmp);
9570
}
9571
rp++;
9572
} else {
9573
list[i].sval = NULL;
9574
list[i].ssiz = 0;
9575
}
9576
list[i].regex = NULL;
9577
list[i].num = cbstrmktime(list[i].val);
9578
if(!cbstricmp(rp, ESTOPSTREQ)){
9579
list[i].cop = ESTOPSTREQ;
9580
} else if(!cbstricmp(rp, ESTOPSTRNE)){
9581
list[i].cop = ESTOPSTRNE;
9582
} else if(!cbstricmp(rp, ESTOPSTRINC)){
9583
list[i].cop = ESTOPSTRINC;
9584
} else if(!cbstricmp(rp, ESTOPSTRBW)){
9585
list[i].cop = ESTOPSTRBW;
9586
} else if(!cbstricmp(rp, ESTOPSTREW)){
9587
list[i].cop = ESTOPSTREW;
9588
} else if(!cbstricmp(rp, ESTOPSTRAND)){
9589
list[i].cop = ESTOPSTRAND;
9590
} else if(!cbstricmp(rp, ESTOPSTROR)){
9591
list[i].cop = ESTOPSTROR;
9592
} else if(!cbstricmp(rp, ESTOPSTROREQ)){
9593
list[i].cop = ESTOPSTROREQ;
9594
} else if(!cbstricmp(rp, ESTOPSTRRX)){
9595
list[i].cop = ESTOPSTRRX;
9596
list[i].regex = list[i].sval ? est_regex_new(list[i].sval) : est_regex_new(list[i].val);
9597
} else if(!cbstricmp(rp, ESTOPNUMEQ)){
9598
list[i].cop = ESTOPNUMEQ;
9599
} else if(!cbstricmp(rp, ESTOPNUMNE)){
9600
list[i].cop = ESTOPNUMNE;
9601
} else if(!cbstricmp(rp, ESTOPNUMGT)){
9602
list[i].cop = ESTOPNUMGT;
9603
} else if(!cbstricmp(rp, ESTOPNUMGE)){
9604
list[i].cop = ESTOPNUMGE;
9605
} else if(!cbstricmp(rp, ESTOPNUMLT)){
9606
list[i].cop = ESTOPNUMLT;
9607
} else if(!cbstricmp(rp, ESTOPNUMLE)){
9608
list[i].cop = ESTOPNUMLE;
9609
} else if(!cbstricmp(rp, ESTOPNUMBT)){
9610
list[i].cop = ESTOPNUMBT;
9611
} else {
9612
list[i].cop = ESTOPSTRINC;
9613
list[i].val[0] = '\0';
9614
list[i].vsiz = 0;
9615
if(list[i].sval){
9616
list[i].sval[0] = '\0';
9617
list[i].ssiz = 0;
9618
}
9619
}
9620
}
9621
*nump = anum;
9622
return list;
9623
}
9624
9625
9626
/* Release resources of a list of condition attributes.
9627
`list' specifies a list of condition attributes.
9628
`anum' specifies the number of elements of the list. */
9629
static void est_free_cattr_list(ESTCATTR *list, int anum){
9630
int i;
9631
assert(list && anum >= 0);
9632
for(i = 0; i < anum; i++){
9633
if(list[i].regex) est_regex_delete(list[i].regex);
9634
free(list[i].sval);
9635
free(list[i].val);
9636
free(list[i].oper);
9637
if(list[i].nlist) CB_LISTCLOSE(list[i].nlist);
9638
free(list[i].name);
9639
}
9640
free(list);
9641
}
9642
9643
4761
9644
/* Narrow and sort scores of search candidates.
4762
9645
`db' specifies a database object.
4763
9646
`scores' specifies an array of scores of search candidates.
4767
9650
`vnum' specifies the number of dimensions of the vector.
4768
9651
`tfidf' specifies whether to perform TF-IDF tuning.
4769
9652
`limit' specifies the upper limit of similarity for documents to survive.
9653
`opts' specifies optoins for eclipse.
4770
9654
`shadows' specifies a map object to store shadow document information.
4771
9655
The return value is the new number of the array. */
4772
9656
static int est_eclipse_scores(ESTDB *db, ESTSCORE *scores, int snum, int num,
4773
9657
int vnum, int tfidf, double limit, CBMAP *shadows){
4774
9658
CBMAP *svmap, *tvmap;
4775
int i, j, max, *svec, *tvec, pair[2], nnum;
9659
const char *suri, *turi;
9660
char *tmp;
9661
int i, j, ubase, simurl, max, *svec, *tvec, pair[2], nnum;
4776
9662
double dval;
4777
9663
assert(db && scores && snum >= 0 && num >= 0 && vnum > 0 && limit > 0.0 && shadows);
4778
max = limit < 0.1 ? snum : num * ((2.0 / limit) + 0.5);
4779
if(max > snum) max = snum;
4780
CB_MALLOC(svec, vnum * sizeof(int));
4781
CB_MALLOC(tvec, vnum * sizeof(int));
4782
for(i = 0; i < max; i++){
4783
svmap = est_get_tvmap(db, scores[i].id, vnum, tfidf);
4784
scores[i].value = (char *)svmap;
4785
}
4786
for(i = 0; i < max; i++){
4787
svmap = (CBMAP *)(scores[i].value);
4788
if(!svmap || cbmaprnum(svmap) < 1) continue;
4789
if(num-- < 1) continue;
4790
est_vector_set_seed(svmap, svec, vnum);
4791
for(j = i + 1; j < max; j++){
4792
tvmap = (CBMAP *)(scores[j].value);
4793
if(!tvmap || cbmaprnum(tvmap) < 1) continue;
4794
est_vector_set_target(svmap, tvmap, tvec, vnum);
4795
dval = est_vector_cosine(svec, tvec, vnum);
4796
if(dval > limit){
4797
cbmapclose(tvmap);
4798
scores[j].value = NULL;
4799
pair[0] = scores[j].id;
4800
pair[1] = (int)(dval * 10000.0);
4801
cbmapputcat(shadows, (char *)&(scores[i].id), sizeof(int),
4802
(char *)pair, sizeof(int) * 2);
4803
}
4804
}
9664
ubase = FALSE;
9665
simurl = FALSE;
9666
if(limit == ESTECLSERV || limit == ESTECLDIR || limit == ESTECLFILE){
9667
ubase = TRUE;
9668
} else if(limit >= ESTECLSIMURL){
9669
simurl = TRUE;
9670
limit -= ESTECLSIMURL;
9671
if(limit < 0.01) limit = 0.01;
9672
if(limit > 1.0) limit = 1.0;
4805
9673
}
4806
9674
nnum = 0;
4807
for(i = 0; i < max; i++){
4808
if(scores[i].value){
4809
cbmapclose((CBMAP *)(scores[i].value));
4810
scores[nnum++] = scores[i];
4811
}
4812
}
4813
for(i = max; i < snum; i++){
4814
scores[nnum++] = scores[i];
4815
}
4816
free(tvec);
4817
free(svec);
9675
if(ubase){
9676
if(limit == ESTECLSERV){
9677
max = num * 14.8 + 8;
9678
} else if(limit == ESTECLDIR){
9679
max = num * 6.8 + 8;
9680
} else {
9681
max = num * 4.8 + 8;
9682
}
9683
if(max > snum) max = snum;
9684
for(i = 0; i < max; i++){
9685
scores[i].value = est_db_get_doc_attr(db, scores[i].id, ESTDATTRURI);
9686
}
9687
for(i = 0; i < max; i++){
9688
if(!scores[i].value) continue;
9689
for(j = i + 1; j < max; j++){
9690
dval = 0.0;
9691
if(scores[j].value){
9692
switch(est_url_sameness(scores[i].value, scores[j].value)){
9693
case 1:
9694
dval = ESTECLSERV;
9695
break;
9696
case 2:
9697
dval = ESTECLDIR;
9698
break;
9699
case 3:
9700
dval = ESTECLFILE;
9701
break;
9702
}
9703
}
9704
if(dval >= limit){
9705
free(scores[j].value);
9706
scores[j].value = NULL;
9707
pair[0] = scores[j].id;
9708
pair[1] = 0;
9709
cbmapputcat(shadows, (char *)&(scores[i].id), sizeof(int),
9710
(char *)pair, sizeof(int) * 2);
9711
}
9712
}
9713
}
9714
for(i = 0; i < max; i++){
9715
if(scores[i].value){
9716
free(scores[i].value);
9717
scores[nnum++] = scores[i];
9718
}
9719
}
9720
for(i = max; i < snum; i++){
9721
scores[nnum++] = scores[i];
9722
}
9723
} else {
9724
max = limit < 0.1 ? snum : num * ((2.4 / (limit - 0.05)) + 0.8) + 8;
9725
if(simurl) max *= 1.4;
9726
if(max > snum) max = snum;
9727
CB_MALLOC(svec, vnum * sizeof(int));
9728
CB_MALLOC(tvec, vnum * sizeof(int));
9729
for(i = 0; i < max; i++){
9730
if((svmap = est_get_tvmap(db, scores[i].id, vnum, tfidf)) != NULL){
9731
scores[i].value = (char *)svmap;
9732
if(simurl && (tmp = est_db_get_doc_attr(db, scores[i].id, ESTDATTRURI)) != NULL){
9733
cbmapput(svmap, "", 0, tmp, -1, TRUE);
9734
free(tmp);
9735
}
9736
} else {
9737
scores[i].value = NULL;
9738
}
9739
}
9740
for(i = 0; i < max; i++){
9741
svmap = (CBMAP *)(scores[i].value);
9742
if(!svmap || cbmaprnum(svmap) < 1) continue;
9743
suri = cbmapget((CBMAP *)scores[i].value, "", -1, NULL);
9744
if(num-- < 1) continue;
9745
est_vector_set_seed(svmap, svec, vnum);
9746
for(j = i + 1; j < max; j++){
9747
tvmap = (CBMAP *)(scores[j].value);
9748
if(!tvmap || cbmaprnum(tvmap) < 1) continue;
9749
est_vector_set_target(svmap, tvmap, tvec, vnum);
9750
dval = est_vector_cosine(svec, tvec, vnum);
9751
if(dval > 0.01 && suri &&
9752
(turi = cbmapget((CBMAP *)scores[j].value, "", -1, NULL)) != NULL){
9753
switch(est_url_sameness(suri, turi)){
9754
default:
9755
dval = pow(cos(acos(dval) * (1.0 - pow(dval, 9.9))), 1.07);
9756
break;
9757
case 1:
9758
dval = pow(cos(acos(dval) * (1.0 - pow(dval, 4.1))), 1.05);
9759
break;
9760
case 2:
9761
dval = pow(cos(acos(dval) * (1.0 - pow(dval, 2.9))), 1.03);
9762
break;
9763
case 3:
9764
dval = pow(cos(acos(dval) * (1.0 - pow(dval, 2.1))), 1.01);
9765
break;
9766
}
9767
}
9768
if(dval > limit){
9769
cbmapclose(tvmap);
9770
scores[j].value = NULL;
9771
pair[0] = scores[j].id;
9772
pair[1] = (int)(dval * 10000.0);
9773
cbmapputcat(shadows, (char *)&(scores[i].id), sizeof(int),
9774
(char *)pair, sizeof(int) * 2);
9775
}
9776
}
9777
}
9778
for(i = 0; i < max; i++){
9779
if(scores[i].value){
9780
cbmapclose((CBMAP *)(scores[i].value));
9781
scores[nnum++] = scores[i];
9782
}
9783
}
9784
for(i = max; i < snum; i++){
9785
scores[nnum++] = scores[i];
9786
}
9787
free(tvec);
9788
free(svec);
9789
}
4818
9790
return nnum;
4819
9791
}
4820
9792
4824
9796
`tsiz' specifies the size of the target value
4825
9797
`cop' specifies the pointer to the operator.
4826
9798
`sign' specifies the sign of operation.
4827
`oval' specifies the operation value;
9799
`oval' specifies the operation value.
4828
9800
`osiz' specifies the size of the operation value
4829
`sval' specifies the operation value of small cases;
9801
`sval' specifies the operation value of small cases.
4830
9802
`ssiz' specifies the size of the operation value of small cases.
4831
9803
`regex' specifies the regular expressions.
4832
9804
`onum' specifies the numeric value.
4864
9836
hit = est_check_strand(tval, oval);
4865
9837
} else if(cop == ESTOPSTROR){
4866
9838
hit = est_check_stror(tval, oval);
9839
} else if(cop == ESTOPSTROREQ){
9840
hit = est_check_stroreq(tval, oval);
4867
9841
} else if(cop == ESTOPSTRRX){
4868
9842
hit = regex ? est_regex_match(regex, tval) : FALSE;
4869
9843
} else if(cop == ESTOPNUMEQ){
4880
9854
hit = cbstrmktime(tval) <= onum;
4881
9855
} else if(cop == ESTOPNUMBT){
4882
9856
hit = est_check_numbt(tval, oval);
9857
} else if(cop == ESTOPDUMMY){
9858
hit = TRUE;
4883
9859
} else {
4884
9860
hit = FALSE;
4885
9861
}
4898
9874
assert(tval && oval);
4899
9875
sp = oval;
4900
9876
while(*sp != '\0'){
4901
while(*sp == ' '){
9877
while(*sp == ' ' || *sp == ','){
4902
9878
sp++;
4903
9879
}
4904
9880
ep = sp;
4905
while(*ep != '\0' && *ep != ' '){
9881
while(*ep != '\0' && *ep != ' ' && *ep != ','){
4906
9882
ep++;
4907
9883
}
4908
9884
if(ep > sp){
4911
9887
for(pp = sp, qp = rp; pp < ep; pp++, qp++){
4912
9888
if(*pp != *qp) break;
4913
9889
}
4914
if(pp == ep){
9890
if(pp == ep && (*qp == '\0' || *qp == ' ' || *qp == ',')){
4915
9891
hit = TRUE;
4916
9892
break;
4917
9893
}
4934
9910
assert(tval && oval);
4935
9911
sp = oval;
4936
9912
while(*sp != '\0'){
4937
while(*sp == ' '){
9913
while(*sp == ' ' || *sp == ','){
4938
9914
sp++;
4939
9915
}
4940
9916
ep = sp;
4941
while(*ep != '\0' && *ep != ' '){
9917
while(*ep != '\0' && *ep != ' ' && *ep != ','){
4942
9918
ep++;
4943
9919
}
4944
9920
if(ep > sp){
4947
9923
for(pp = sp, qp = rp; pp < ep; pp++, qp++){
4948
9924
if(*pp != *qp) break;
4949
9925
}
4950
if(pp == ep){
9926
if(pp == ep && (*qp == '\0' || *qp == ' ' || *qp == ',')){
4951
9927
hit = TRUE;
4952
9928
break;
4953
9929
}
4960
9936
}
4961
9937
4962
9938
9939
/* Check whether a string is equal to at least one token in another string.
9940
`tval' specifies the target value;
9941
`oval' specifies the operation value;
9942
The return value is the result of the check. */
9943
static int est_check_stroreq(const char *tval, const char *oval){
9944
const char *sp, *ep, *rp;
9945
assert(tval && oval);
9946
sp = oval;
9947
while(*sp != '\0'){
9948
while(*sp == ' ' || *sp == ','){
9949
sp++;
9950
}
9951
ep = sp;
9952
while(*ep != '\0' && *ep != ' ' && *ep != ','){
9953
ep++;
9954
}
9955
if(ep > sp){
9956
for(rp = tval; *rp != '\0'; rp++){
9957
if(*sp != *rp || sp >= ep) break;
9958
sp++;
9959
}
9960
if(*rp == '\0' && sp == ep) return TRUE;
9961
}
9962
sp = ep;
9963
}
9964
return FALSE;
9965
}
9966
9967
4963
9968
/* Check whether a decimal string is between two tokens in another string.
4964
9969
`tval' specifies the target value;
4965
9970
`oval' specifies the operation value;
5007
10012
`unum' specifies the number of adopted documents for a keyword.
5008
10013
`tfidf' specifies whether to perform TF-IDF tuning.
5009
10014
`nmin' specifies the minimum value for narrowing.
5010
The return value is an array whose elements are ID numbers of similar documents. */
10015
`auxmin' specifies the minimum hits to adopt the auxiliary index. If it is not more than 0,
10016
the auxiliary index is not used.
10017
`auxwords' specifies a map object where keywords used with the auxiliary index are stored. If
10018
it is `NULL', it is not used.
10019
The return value is an array of score structures of corresponding documents. */
5011
10020
static ESTSCORE *est_search_similar(ESTDB *db, CBMAP *svmap, int *nump,
5012
int knum, int unum, int tfidf, double nmin){
10021
int knum, int unum, int mnum, int tfidf,
10022
double nmin, int auxmin, CBMAP *auxwords){
5013
10023
ESTSCORE *scores, *tscores;
5014
10024
CBMAP *tvmap;
5015
10025
const char *word;
5016
10026
int i, j, vnum, snum, tmax, tsnum, nnum, lid, *svec, *tvec;
5017
10027
double dval;
5018
10028
assert(db && svmap && nump && knum >= 0 && unum >= 0 && nmin >= 0.0);
5019
CB_MALLOC(scores, sizeof(ESTSCORE) * unum * knum);
10029
CB_MALLOC(scores, sizeof(ESTSCORE) * (unum * knum + CB_LISTNUM(db->pdocs)) + 1);
5020
10030
snum = 0;
5021
10031
if((vnum = cbmaprnum(svmap)) < 1) vnum = 1;
5022
10032
cbmapiterinit(svmap);
5023
10033
tmax = unum;
5024
for(i = 0; i < knum && (word = cbmapiternext(svmap, NULL)) != NULL; i++){
5025
tscores = est_search_union(db, word, 1, &tsnum, NULL, TRUE);
5026
qsort(tscores, tsnum, sizeof(ESTSCORE), est_score_compare_by_score);
10034
for(i = 0; (i < knum || (i < knum * 2 && snum < unum * 2)) &&
10035
(word = cbmapiternext(svmap, NULL)) != NULL; i++){
10036
while(*word > '\0' && *word <= ' '){
10037
word++;
10038
}
10039
tscores = est_search_union(db, word, 1, NULL, &tsnum, NULL, TRUE, auxmin, auxwords);
10040
qsort(tscores, tsnum, sizeof(ESTSCORE), est_score_compare_by_score_desc);
5027
10041
for(j = 0; j < tmax && j < tsnum; j++){
5028
10042
scores[snum].id = tscores[j].id;
5029
scores[snum].score = tscores[j].score;
10043
scores[snum].score = tscores[j].score * (knum * 2.2 - i);
5030
10044
snum++;
5031
10045
}
5032
10046
free(tscores);
5033
10047
tmax -= unum / knum / 1.25;
5034
}
5035
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id);
10048
if(tmax < unum / 4) tmax = unum / 4;
10049
}
10050
for(i = 0; i < CB_LISTNUM(db->pdocs); i++){
10051
scores[snum].id = ESTPDOCIDMIN + i;
10052
scores[snum].score = 1;
10053
snum++;
10054
}
10055
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_id_asc);
5036
10056
nnum = 0;
5037
10057
lid = -1;
10058
for(i = 0; i < snum; i++){
10059
if(nnum > 0 && scores[i].id == lid){
10060
scores[nnum-1].score += scores[i].score;
10061
continue;
10062
}
10063
scores[nnum].id = scores[i].id;
10064
scores[nnum].score = scores[i].score;
10065
nnum++;
10066
lid = scores[i].id;
10067
}
10068
snum = nnum;
10069
qsort(scores, snum, sizeof(ESTSCORE), est_score_compare_by_score_desc);
10070
nnum = 0;
5038
10071
CB_MALLOC(svec, vnum * sizeof(int));
5039
10072
CB_MALLOC(tvec, vnum * sizeof(int));
5040
10073
est_vector_set_seed(svmap, svec, vnum);
5041
for(i = 0; i < snum; i++){
5042
if(scores[i].id != lid){
5043
tvmap = est_get_tvmap(db, scores[i].id, vnum, tfidf);
5044
if(tvmap){
5045
est_vector_set_target(svmap, tvmap, tvec, vnum);
5046
if((dval = est_vector_cosine(svec, tvec, vnum)) >= nmin){
5047
scores[nnum].id = scores[i].id;
5048
scores[nnum].score = (int)(dval * 10000);
5049
if(scores[nnum].score == 9999) scores[nnum].score = 10000;
5050
nnum++;
5051
}
5052
cbmapclose(tvmap);
10074
for(i = 0; i < snum && nnum < mnum; i++){
10075
tvmap = est_get_tvmap(db, scores[i].id, vnum, tfidf);
10076
if(tvmap){
10077
est_vector_set_target(svmap, tvmap, tvec, vnum);
10078
if((dval = est_vector_cosine(svec, tvec, vnum)) >= nmin){
10079
scores[nnum].id = scores[i].id;
10080
scores[nnum].score = (int)(dval * 10000);
10081
if(scores[nnum].score == 9999) scores[nnum].score = 10000;
10082
scores[nnum].value = NULL;
10083
nnum++;
5053
10084
}
10085
cbmapclose(tvmap);
5054
10086
}
5055
lid = scores[i].id;
5056
10087
}
5057
10088
free(tvec);
5058
10089
free(svec);
5066
10097
`phrase' specifies a search phrase for similar search.
5067
10098
The return value is a map object of the seed vector. */
5068
10099
static CBMAP *est_phrase_vector(const char *phrase){
10100
ESTKEYSC *scores;
5069
10101
CBMAP *svmap;
5070
10102
CBLIST *list;
5071
10103
const char *pv, *rp;
5072
10104
char *utext, *rtext;
5073
10105
int i, num, len, size;
5074
10106
svmap = cbmapopenex(ESTMINIBNUM);
5075
list = cblistopen();
10107
CB_LISTOPEN(list);
5076
10108
while(*phrase != '\0'){
5077
10109
if(*phrase == ESTOPWITH[0] && cbstrfwmatch(phrase, ESTOPWITH)){
5078
10110
phrase += strlen(ESTOPWITH);
5084
10116
}
5085
10117
phrase++;
5086
10118
}
5087
cblistpush(list, pv, phrase - pv);
10119
CB_LISTPUSH(list, pv, phrase - pv);
5088
10120
} else {
5089
10121
phrase++;
5090
10122
}
5091
10123
}
5092
10124
for(i = 0; i < CB_LISTNUM(list); i++){
5093
pv = CB_LISTVAL(list, i, NULL);
10125
pv = CB_LISTVAL(list, i);
5094
10126
while(*pv > '\0' && *pv <= ' '){
5095
10127
pv++;
5096
10128
}
5106
10138
free(utext);
5107
10139
}
5108
10140
}
5109
cblistclose(list);
10141
CB_LISTCLOSE(list);
10142
CB_MALLOC(scores, cbmaprnum(svmap) * sizeof(ESTKEYSC) + 1);
10143
cbmapiterinit(svmap);
10144
for(i = 0; (rp = cbmapiternext(svmap, &len)) != NULL; i++){
10145
scores[i].word = rp;
10146
scores[i].wsiz = len;
10147
scores[i].pt = atoi(cbmapiterval(rp, NULL));
10148
}
10149
qsort(scores, i, sizeof(ESTKEYSC), est_keysc_compare);
10150
for(i--; i >= 0; i--){
10151
cbmapmove(svmap, scores[i].word, scores[i].wsiz, TRUE);
10152
}
10153
free(scores);
5110
10154
return svmap;
5111
10155
}
5112
10156
5125
10169
if(!(doc = est_db_get_doc(db, id, 0))) return NULL;
5126
10170
tvmap = est_db_etch_doc(tfidf ? db : NULL, doc, vnum);
5127
10171
est_doc_delete(doc);
5128
if(dpwritable(db->metadb)) est_db_put_keywords(db, id, tvmap);
10172
if(dpwritable(db->metadb)) est_db_put_keywords(db, id, tvmap, 1.0);
5129
10173
return tvmap;
5130
10174
}
5131
10175
5132
10176
10177
/* Calculate sameness of two URLs.
10178
The return value is 0 if the both have different servers, 1 if the both have the same server,
10179
2 if the both have the same parent directory, 3 if the both have the same file. */
10180
static int est_url_sameness(const char *aurl, const char *burl){
10181
const char *apv, *bpv;
10182
int i, alen, blen;
10183
assert(aurl && burl);
10184
if((apv = strstr(aurl, "://")) != NULL){
10185
aurl = apv + 3;
10186
} else {
10187
return 0;
10188
}
10189
if((bpv = strstr(burl, "://")) != NULL){
10190
burl = bpv + 3;
10191
} else {
10192
return 0;
10193
}
10194
if(!(apv = strchr(aurl, '/'))) apv = aurl + strlen(aurl);
10195
if(!(bpv = strchr(burl, '/'))) bpv = burl + strlen(burl);
10196
alen = apv - aurl;
10197
blen = bpv - burl;
10198
if(alen != blen || memcmp(aurl, burl, alen)) return 0;
10199
aurl = *apv == '\0' ? "/" : apv;
10200
burl = *bpv == '\0' ? "/" : bpv;
10201
if(!(apv = strchr(aurl, '?'))) apv = aurl + strlen(aurl);
10202
if(!(bpv = strchr(burl, '?'))) bpv = burl + strlen(burl);
10203
alen = apv - aurl;
10204
blen = bpv - burl;
10205
if(alen == blen && !memcmp(aurl, burl, alen)) return 3;
10206
apv = aurl;
10207
for(i = 0; i < alen; i++){
10208
if(aurl[i] == '/') apv = aurl + i;
10209
}
10210
bpv = burl;
10211
for(i = 0; i < blen; i++){
10212
if(burl[i] == '/') bpv = burl + i;
10213
}
10214
alen = apv - aurl;
10215
blen = bpv - burl;
10216
if(alen == blen && !memcmp(aurl, burl, alen)) return 2;
10217
return 1;
10218
}
10219
10220
5133
10221
/* Close the handle to the file of random number generator. */
5134
10222
static void est_random_fclose(void){
5135
10223
if(est_random_ifp) fclose(est_random_ifp);
Loggerhead is a web-based interface for Breezy
Version: 2.0.1