4
** The author disclaims copyright to this source code. In place of
5
** a legal notice, here is a blessing:
7
** May you do good and not evil.
8
** May you find forgiveness for yourself and forgive others.
9
** May you share freely, never taking more than you give.
11
******************************************************************************
18
#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
23
#include "fts3_tokenizer.h"
24
#include "fts3_hash.h"
27
** This constant controls how often segments are merged. Once there are
28
** FTS3_MERGE_COUNT segments of level N, they are merged into a single
29
** segment of level N+1.
31
#define FTS3_MERGE_COUNT 16
34
** This is the maximum amount of data (in bytes) to store in the
35
** Fts3Table.pendingTerms hash table. Normally, the hash table is
36
** populated as documents are inserted/updated/deleted in a transaction
37
** and used to create a new segment when the transaction is committed.
38
** However if this limit is reached midway through a transaction, a new
39
** segment is created and the hash table cleared immediately.
41
#define FTS3_MAX_PENDING_DATA (1*1024*1024)
44
** Macro to return the number of elements in an array. SQLite has a
45
** similar macro called ArraySize(). Use a different name to avoid
46
** a collision when building an amalgamation with built-in FTS3.
48
#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
51
** Maximum length of a varint encoded integer. The varint format is different
52
** from that used by SQLite, so the maximum length is 10, not 9.
54
#define FTS3_VARINT_MAX 10
57
** This section provides definitions to allow the
58
** FTS3 extension to be compiled outside of the
61
#ifndef SQLITE_AMALGAMATION
63
** Macros indicating that conditional expressions are always true or
66
# define ALWAYS(x) (x)
69
** Internal types used by SQLite.
71
typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */
72
typedef short int i16; /* 2-byte (or larger) signed integer */
74
** Macro used to suppress compiler warnings for unused parameters.
76
#define UNUSED_PARAMETER(x) (void)(x)
79
typedef struct Fts3Table Fts3Table;
80
typedef struct Fts3Cursor Fts3Cursor;
81
typedef struct Fts3Expr Fts3Expr;
82
typedef struct Fts3Phrase Fts3Phrase;
83
typedef struct Fts3SegReader Fts3SegReader;
84
typedef struct Fts3SegFilter Fts3SegFilter;
87
** A connection to a fulltext index is an instance of the following
88
** structure. The xCreate and xConnect methods create an instance
89
** of this structure and xDestroy and xDisconnect free that instance.
90
** All other methods receive a pointer to the structure as one of their
94
sqlite3_vtab base; /* Base class used by SQLite core */
95
sqlite3 *db; /* The database connection */
96
const char *zDb; /* logical database name */
97
const char *zName; /* virtual table name */
98
int nColumn; /* number of named columns in virtual table */
99
char **azColumn; /* column names. malloced */
100
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
102
/* Precompiled statements used by the implementation. Each of these
103
** statements is run and reset within a single virtual table API call.
105
sqlite3_stmt *aStmt[18];
107
/* Pointer to string containing the SQL:
109
** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ?
113
int nLeavesStmt; /* Valid statements in aLeavesStmt */
114
int nLeavesTotal; /* Total number of prepared leaves stmts */
115
int nLeavesAlloc; /* Allocated size of aLeavesStmt */
116
sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */
118
int nNodeSize; /* Soft limit for node size */
120
/* The following hash table is used to buffer pending index updates during
121
** transactions. Variable nPendingData estimates the memory size of the
122
** pending data, including hash table overhead, but not malloc overhead.
123
** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed
124
** automatically. Variable iPrevDocid is the docid of the most recently
128
sqlite_int64 iPrevDocid;
129
Fts3Hash pendingTerms;
133
** When the core wants to read from the virtual table, it creates a
134
** virtual table cursor (an instance of the following structure) using
135
** the xOpen method. Cursors are destroyed using the xClose method.
138
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
139
i16 eSearch; /* Search strategy (see below) */
140
u8 isEof; /* True if at End Of Results */
141
u8 isRequireSeek; /* True if must seek pStmt to %_content row */
142
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
143
Fts3Expr *pExpr; /* Parsed MATCH query string */
144
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
145
char *pNextId; /* Pointer into the body of aDoclist */
146
char *aDoclist; /* List of docids for full-text queries */
147
int nDoclist; /* Size of buffer at aDoclist */
151
** The Fts3Cursor.eSearch member is always set to one of the following.
152
** Actualy, Fts3Cursor.eSearch can be greater than or equal to
153
** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index
154
** of the column to be searched. For example, in
156
** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d);
157
** SELECT docid FROM ex1 WHERE b MATCH 'one two three';
159
** Because the LHS of the MATCH operator is 2nd column "b",
160
** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a,
161
** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1"
162
** indicating that all columns should be searched,
163
** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
165
#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
166
#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
167
#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
170
** A "phrase" is a sequence of one or more tokens that must match in
171
** sequence. A single token is the base case and the most common case.
172
** For a sequence of tokens contained in "...", nToken will be the number
173
** of tokens in the string.
176
int nToken; /* Number of tokens in the phrase */
177
int iColumn; /* Index of column this phrase must match */
178
int isNot; /* Phrase prefixed by unary not (-) operator */
180
char *z; /* Text of the token */
181
int n; /* Number of bytes in buffer pointed to by z */
182
int isPrefix; /* True if token ends in with a "*" character */
183
} aToken[1]; /* One entry for each token in the phrase */
187
** A tree of these objects forms the RHS of a MATCH operator.
190
int eType; /* One of the FTSQUERY_XXX values defined below */
191
int nNear; /* Valid if eType==FTSQUERY_NEAR */
192
Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
193
Fts3Expr *pLeft; /* Left operand */
194
Fts3Expr *pRight; /* Right operand */
195
Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
199
** Candidate values for Fts3Query.eType. Note that the order of the first
200
** four values is in order of precedence when parsing expressions. For
201
** example, the following:
203
** "a OR b AND c NOT d NEAR e"
207
** "a OR (b AND (c NOT (d NEAR e)))"
209
#define FTSQUERY_NEAR 1
210
#define FTSQUERY_NOT 2
211
#define FTSQUERY_AND 3
212
#define FTSQUERY_OR 4
213
#define FTSQUERY_PHRASE 5
217
int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *);
218
int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*,
219
sqlite3_vtab **, char **);
222
int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
223
int sqlite3Fts3PendingTermsFlush(Fts3Table *);
224
void sqlite3Fts3PendingTermsClear(Fts3Table *);
225
int sqlite3Fts3Optimize(Fts3Table *);
226
int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
227
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
228
void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *);
229
int sqlite3Fts3SegReaderIterate(
230
Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
231
int (*)(Fts3Table *, void *, char *, int, char *, int), void *
233
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*);
234
int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
236
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
237
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
238
#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
239
#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
240
#define FTS3_SEGMENT_PREFIX 0x00000008
242
/* Type passed as 4th argument to SegmentReaderIterate() */
243
struct Fts3SegFilter {
251
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
252
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
253
int sqlite3Fts3GetVarint32(const char *, int *);
254
int sqlite3Fts3VarintLen(sqlite3_uint64);
255
void sqlite3Fts3Dequote(char *);
257
/* fts3_tokenizer.c */
258
const char *sqlite3Fts3NextToken(const char *, int *);
259
int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
260
int sqlite3Fts3InitTokenizer(Fts3Hash *pHash,
261
const char *, sqlite3_tokenizer **, const char **, char **
265
void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
266
void sqlite3Fts3Snippet(sqlite3_context*, Fts3Cursor*,
267
const char *, const char *, const char *
271
int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
272
char **, int, int, const char *, int, Fts3Expr **
274
void sqlite3Fts3ExprFree(Fts3Expr *);
276
void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
279
#endif /* _FTSINT_H */
added
removed
ext/fts3/fts3Int.h
