1
package org.apache.lucene.queryParser;
4
* Licensed to the Apache Software Foundation (ASF) under one or more
5
* contributor license agreements. See the NOTICE file distributed with
6
* this work for additional information regarding copyright ownership.
7
* The ASF licenses this file to You under the Apache License, Version 2.0
8
* (the "License"); you may not use this file except in compliance with
9
* the License. You may obtain a copy of the License at
11
* http://www.apache.org/licenses/LICENSE-2.0
13
* Unless required by applicable law or agreed to in writing, software
14
* distributed under the License is distributed on an "AS IS" BASIS,
15
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
* See the License for the specific language governing permissions and
17
* limitations under the License.
20
import java.io.IOException;
21
import java.io.Reader;
23
import org.apache.lucene.analysis.Analyzer;
24
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
25
import org.apache.lucene.analysis.LowerCaseFilter;
26
import org.apache.lucene.analysis.TokenFilter;
27
import org.apache.lucene.analysis.TokenStream;
28
import org.apache.lucene.analysis.standard.StandardTokenizer;
29
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
30
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
31
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
32
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
33
import org.apache.lucene.search.Query;
34
import org.apache.lucene.util.Version;
37
* Test QueryParser's ability to deal with Analyzers that return more
38
* than one token per position or that return tokens with a position
42
public class TestMultiAnalyzer extends BaseTokenStreamTestCase {
44
private static int multiToken = 0;
46
public void testMultiAnalyzer() throws ParseException {
48
QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "", new MultiAnalyzer());
50
// trivial, no multiple tokens:
51
assertEquals("foo", qp.parse("foo").toString());
52
assertEquals("foo", qp.parse("\"foo\"").toString());
53
assertEquals("foo foobar", qp.parse("foo foobar").toString());
54
assertEquals("\"foo foobar\"", qp.parse("\"foo foobar\"").toString());
55
assertEquals("\"foo foobar blah\"", qp.parse("\"foo foobar blah\"").toString());
57
// two tokens at the same position:
58
assertEquals("(multi multi2) foo", qp.parse("multi foo").toString());
59
assertEquals("foo (multi multi2)", qp.parse("foo multi").toString());
60
assertEquals("(multi multi2) (multi multi2)", qp.parse("multi multi").toString());
61
assertEquals("+(foo (multi multi2)) +(bar (multi multi2))",
62
qp.parse("+(foo multi) +(bar multi)").toString());
63
assertEquals("+(foo (multi multi2)) field:\"bar (multi multi2)\"",
64
qp.parse("+(foo multi) field:\"bar multi\"").toString());
67
assertEquals("\"(multi multi2) foo\"", qp.parse("\"multi foo\"").toString());
68
assertEquals("\"foo (multi multi2)\"", qp.parse("\"foo multi\"").toString());
69
assertEquals("\"foo (multi multi2) foobar (multi multi2)\"",
70
qp.parse("\"foo multi foobar multi\"").toString());
73
assertEquals("(field:multi field:multi2) field:foo", qp.parse("field:multi field:foo").toString());
74
assertEquals("field:\"(multi multi2) foo\"", qp.parse("field:\"multi foo\"").toString());
76
// three tokens at one position:
77
assertEquals("triplemulti multi3 multi2", qp.parse("triplemulti").toString());
78
assertEquals("foo (triplemulti multi3 multi2) foobar",
79
qp.parse("foo triplemulti foobar").toString());
81
// phrase with non-default slop:
82
assertEquals("\"(multi multi2) foo\"~10", qp.parse("\"multi foo\"~10").toString());
84
// phrase with non-default boost:
85
assertEquals("\"(multi multi2) foo\"^2.0", qp.parse("\"multi foo\"^2").toString());
87
// phrase after changing default slop
89
assertEquals("\"(multi multi2) foo\"~99 bar",
90
qp.parse("\"multi foo\" bar").toString());
91
assertEquals("\"(multi multi2) foo\"~99 \"foo bar\"~2",
92
qp.parse("\"multi foo\" \"foo bar\"~2").toString());
95
// non-default operator:
96
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
97
assertEquals("+(multi multi2) +foo", qp.parse("multi foo").toString());
101
public void testMultiAnalyzerWithSubclassOfQueryParser() throws ParseException {
103
DumbQueryParser qp = new DumbQueryParser("", new MultiAnalyzer());
104
qp.setPhraseSlop(99); // modified default slop
106
// direct call to (super's) getFieldQuery to demonstrate differnce
107
// between phrase and multiphrase with modified default slop
108
assertEquals("\"foo bar\"~99",
109
qp.getSuperFieldQuery("","foo bar", true).toString());
110
assertEquals("\"(multi multi2) bar\"~99",
111
qp.getSuperFieldQuery("","multi bar", true).toString());
114
// ask sublcass to parse phrase with modified default slop
115
assertEquals("\"(multi multi2) foo\"~99 bar",
116
qp.parse("\"multi foo\" bar").toString());
120
public void testPosIncrementAnalyzer() throws ParseException {
121
QueryParser qp = new QueryParser(Version.LUCENE_24, "", new PosIncrementAnalyzer());
122
assertEquals("quick brown", qp.parse("the quick brown").toString());
123
assertEquals("\"quick brown\"", qp.parse("\"the quick brown\"").toString());
124
assertEquals("quick brown fox", qp.parse("the quick brown fox").toString());
125
assertEquals("\"quick brown fox\"", qp.parse("\"the quick brown fox\"").toString());
129
* Expands "multi" to "multi" and "multi2", both at the same position,
130
* and expands "triplemulti" to "triplemulti", "multi3", and "multi2".
132
private class MultiAnalyzer extends Analyzer {
134
public MultiAnalyzer() {
138
public TokenStream tokenStream(String fieldName, Reader reader) {
139
TokenStream result = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
140
result = new TestFilter(result);
141
result = new LowerCaseFilter(TEST_VERSION_CURRENT, result);
146
private final class TestFilter extends TokenFilter {
148
private String prevType;
149
private int prevStartOffset;
150
private int prevEndOffset;
152
private final CharTermAttribute termAtt;
153
private final PositionIncrementAttribute posIncrAtt;
154
private final OffsetAttribute offsetAtt;
155
private final TypeAttribute typeAtt;
157
public TestFilter(TokenStream in) {
159
termAtt = addAttribute(CharTermAttribute.class);
160
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
161
offsetAtt = addAttribute(OffsetAttribute.class);
162
typeAtt = addAttribute(TypeAttribute.class);
166
public final boolean incrementToken() throws java.io.IOException {
167
if (multiToken > 0) {
168
termAtt.setEmpty().append("multi"+(multiToken+1));
169
offsetAtt.setOffset(prevStartOffset, prevEndOffset);
170
typeAtt.setType(prevType);
171
posIncrAtt.setPositionIncrement(0);
175
boolean next = input.incrementToken();
179
prevType = typeAtt.type();
180
prevStartOffset = offsetAtt.startOffset();
181
prevEndOffset = offsetAtt.endOffset();
182
String text = termAtt.toString();
183
if (text.equals("triplemulti")) {
186
} else if (text.equals("multi")) {
195
public void reset() throws IOException {
197
this.prevType = null;
198
this.prevStartOffset = 0;
199
this.prevEndOffset = 0;
204
* Analyzes "the quick brown" as: quick(incr=2) brown(incr=1).
205
* Does not work correctly for input other than "the quick brown ...".
207
private class PosIncrementAnalyzer extends Analyzer {
209
public PosIncrementAnalyzer() {
213
public TokenStream tokenStream(String fieldName, Reader reader) {
214
TokenStream result = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
215
result = new TestPosIncrementFilter(result);
216
result = new LowerCaseFilter(TEST_VERSION_CURRENT, result);
221
private final class TestPosIncrementFilter extends TokenFilter {
223
CharTermAttribute termAtt;
224
PositionIncrementAttribute posIncrAtt;
226
public TestPosIncrementFilter(TokenStream in) {
228
termAtt = addAttribute(CharTermAttribute.class);
229
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
233
public final boolean incrementToken () throws java.io.IOException {
234
while(input.incrementToken()) {
235
if (termAtt.toString().equals("the")) {
236
// stopword, do nothing
237
} else if (termAtt.toString().equals("quick")) {
238
posIncrAtt.setPositionIncrement(2);
241
posIncrAtt.setPositionIncrement(1);
249
/** a very simple subclass of QueryParser */
250
private final static class DumbQueryParser extends QueryParser {
252
public DumbQueryParser(String f, Analyzer a) {
253
super(TEST_VERSION_CURRENT, f, a);
256
/** expose super's version */
257
public Query getSuperFieldQuery(String f, String t, boolean quoted)
258
throws ParseException {
259
return super.getFieldQuery(f,t,quoted);
261
/** wrap super's version */
263
protected Query getFieldQuery(String f, String t, boolean quoted)
264
throws ParseException {
265
return new DumbQueryWrapper(getSuperFieldQuery(f,t,quoted));
270
* A very simple wrapper to prevent instanceof checks but uses
271
* the toString of the query it wraps.
273
private final static class DumbQueryWrapper extends Query {
276
public DumbQueryWrapper(Query q) {
281
public String toString(String f) {
282
return q.toString(f);