1
#ifndef FTS_TOKENIZER_PRIVATE_H
2
#define FTS_TOKENIZER_PRIVATE_H
4
#include "fts-tokenizer.h"
6
#define FTS_TOKENIZER_CLASSES_NR 2
8
struct fts_tokenizer_vfuncs {
9
int (*create)(const char *const *settings,
10
struct fts_tokenizer **tokenizer_r, const char **error_r);
11
void (*destroy)(struct fts_tokenizer *tok);
13
void (*reset)(struct fts_tokenizer *tok);
14
int (*next)(struct fts_tokenizer *tok, const unsigned char *data,
15
size_t size, size_t *skip_r, const char **token_r,
16
const char **error_r);
19
enum fts_tokenizer_parent_state {
20
FTS_TOKENIZER_PARENT_STATE_ADD_DATA = 0,
21
FTS_TOKENIZER_PARENT_STATE_NEXT_OUTPUT,
22
FTS_TOKENIZER_PARENT_STATE_FINALIZE
25
struct fts_tokenizer {
27
const struct fts_tokenizer_vfuncs *v;
30
struct fts_tokenizer *parent;
31
buffer_t *parent_input;
32
enum fts_tokenizer_parent_state parent_state;
34
const unsigned char *prev_data;
37
bool prev_reply_finished;
38
bool skip_parents; /* Return token as is, do not hand to parents. */
41
void fts_tokenizer_register(const struct fts_tokenizer *tok_class);
42
void fts_tokenizer_unregister(const struct fts_tokenizer *tok_class);
added
removed
src/lib-fts/fts-tokenizer-private.h
