2
* Licensed to the Apache Software Foundation (ASF) under one or more
3
* contributor license agreements. See the NOTICE file distributed with
4
* this work for additional information regarding copyright ownership.
5
* The ASF licenses this file to You under the Apache License, Version 2.0
6
* (the "License"); you may not use this file except in compliance with
7
* the License. You may obtain a copy of the License at
9
* http://www.apache.org/licenses/LICENSE-2.0
11
* Unless required by applicable law or agreed to in writing, software
12
* distributed under the License is distributed on an "AS IS" BASIS,
13
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
* See the License for the specific language governing permissions and
15
* limitations under the License.
18
package org.apache.solr.analysis;
20
import org.apache.lucene.analysis.TokenFilter;
21
import org.apache.lucene.analysis.TokenStream;
22
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
23
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
25
import java.io.IOException;
28
* Trims leading and trailing whitespace from Tokens in the stream.
32
public final class TrimFilter extends TokenFilter {
34
final boolean updateOffsets;
35
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
36
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
39
public TrimFilter(TokenStream in, boolean updateOffsets) {
41
this.updateOffsets = updateOffsets;
45
public boolean incrementToken() throws IOException {
46
if (!input.incrementToken()) return false;
48
char[] termBuffer = termAtt.buffer();
49
int len = termAtt.length();
50
//TODO: Is this the right behavior or should we return false? Currently, " ", returns true, so I think this should
59
// eat the first characters
60
//QUESTION: Should we use Character.isWhitespace() instead?
61
for (start = 0; start < len && termBuffer[start] <= ' '; start++) {
63
// eat the end characters
64
for (end = len; end >= start && termBuffer[end - 1] <= ' '; end--) {
67
if (start > 0 || end < len) {
69
termAtt.copyBuffer(termBuffer, start, (end - start));
74
int newStart = offsetAtt.startOffset()+start;
75
int newEnd = offsetAtt.endOffset() - (start<end ? endOff:0);
76
offsetAtt.setOffset(newStart, newEnd);