4
** The author disclaims copyright to this source code.
6
*************************************************************************
7
** Defines the interface to tokenizers used by fulltext-search. There
8
** are three basic components:
10
** sqlite3_tokenizer_module is a singleton defining the tokenizer
11
** interface functions. This is essentially the class structure for
14
** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
15
** including customization information defined at creation time.
17
** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
18
** tokens from a particular input.
23
/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
24
** If tokenizers are to be allowed to call sqlite3_*() functions, then
25
** we will need a way to register the API consistently.
30
** Structures used by the tokenizer interface.
32
typedef struct sqlite3_tokenizer sqlite3_tokenizer;
33
typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
34
typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
36
struct sqlite3_tokenizer_module {
37
int iVersion; /* currently 0 */
40
** Create and destroy a tokenizer. argc/argv are passed down from
41
** the fulltext virtual table creation to allow customization.
43
int (*xCreate)(int argc, const char **argv,
44
sqlite3_tokenizer **ppTokenizer);
45
int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
48
** Tokenize a particular input. Call xOpen() to prepare to
49
** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
50
** xClose() to free any internal state. The pInput passed to
51
** xOpen() must exist until the cursor is closed. The ppToken
52
** result from xNext() is only valid until the next call to xNext()
53
** or until xClose() is called.
55
/* TODO(shess) current implementation requires pInput to be
56
** nul-terminated. This should either be fixed, or pInput/nBytes
57
** should be converted to zInput.
59
int (*xOpen)(sqlite3_tokenizer *pTokenizer,
60
const char *pInput, int nBytes,
61
sqlite3_tokenizer_cursor **ppCursor);
62
int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
63
int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
64
const char **ppToken, int *pnBytes,
65
int *piStartOffset, int *piEndOffset, int *piPosition);
68
struct sqlite3_tokenizer {
69
sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
70
/* Tokenizer implementations will typically add additional fields */
73
struct sqlite3_tokenizer_cursor {
74
sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
75
/* Tokenizer implementations will typically add additional fields */
79
** Get the module for a tokenizer which generates tokens based on a
80
** set of non-token characters. The default is to break tokens at any
81
** non-alnum character, though the set of delimiters can also be
82
** specified by the first argv argument to xCreate().
84
/* TODO(shess) This doesn't belong here. Need some sort of
85
** registration process.
87
void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
89
#endif /* _TOKENIZER_H_ */
added
removed
ext/fts1/tokenizer.h
