1
package org.apache.lucene.index;
4
* Licensed to the Apache Software Foundation (ASF) under one or more
5
* contributor license agreements. See the NOTICE file distributed with
6
* this work for additional information regarding copyright ownership.
7
* The ASF licenses this file to You under the Apache License, Version 2.0
8
* (the "License"); you may not use this file except in compliance with
9
* the License. You may obtain a copy of the License at
11
* http://www.apache.org/licenses/LICENSE-2.0
13
* Unless required by applicable law or agreed to in writing, software
14
* distributed under the License is distributed on an "AS IS" BASIS,
15
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
* See the License for the specific language governing permissions and
17
* limitations under the License.
20
import java.io.IOException;
22
import org.apache.lucene.analysis.MockAnalyzer;
23
import org.apache.lucene.document.Document;
24
import org.apache.lucene.document.Field;
25
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
26
import org.apache.lucene.store.Directory;
28
import org.apache.lucene.util.LuceneTestCase;
30
public class TestIndexWriterMergePolicy extends LuceneTestCase {
32
// Test the normal case
33
public void testNormalCase() throws IOException {
34
Directory dir = newDirectory();
36
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
37
TEST_VERSION_CURRENT, new MockAnalyzer(random))
38
.setMaxBufferedDocs(10).setMergePolicy(new LogDocMergePolicy()));
40
for (int i = 0; i < 100; i++) {
42
checkInvariants(writer);
49
// Test to see if there is over merge
50
public void testNoOverMerge() throws IOException {
51
Directory dir = newDirectory();
53
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
54
TEST_VERSION_CURRENT, new MockAnalyzer(random))
55
.setMaxBufferedDocs(10).setMergePolicy(new LogDocMergePolicy()));
57
boolean noOverMerge = false;
58
for (int i = 0; i < 100; i++) {
60
checkInvariants(writer);
61
if (writer.getNumBufferedDocuments() + writer.getSegmentCount() >= 18) {
65
assertTrue(noOverMerge);
71
// Test the case where flush is forced after every addDoc
72
public void testForceFlush() throws IOException {
73
Directory dir = newDirectory();
75
LogDocMergePolicy mp = new LogDocMergePolicy();
76
mp.setMinMergeDocs(100);
77
mp.setMergeFactor(10);
78
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
79
TEST_VERSION_CURRENT, new MockAnalyzer(random))
80
.setMaxBufferedDocs(10).setMergePolicy(mp));
82
for (int i = 0; i < 100; i++) {
86
mp = new LogDocMergePolicy();
87
mp.setMergeFactor(10);
88
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
89
new MockAnalyzer(random)).setOpenMode(
90
OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(mp));
91
mp.setMinMergeDocs(100);
92
checkInvariants(writer);
99
// Test the case where mergeFactor changes
100
public void testMergeFactorChange() throws IOException {
101
Directory dir = newDirectory();
103
IndexWriter writer = new IndexWriter(
105
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
106
setMaxBufferedDocs(10).
107
setMergePolicy(newLogMergePolicy()).
108
setMergeScheduler(new SerialMergeScheduler())
111
writer.setInfoStream(VERBOSE ? System.out : null);
113
for (int i = 0; i < 250; i++) {
115
checkInvariants(writer);
118
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5);
120
// merge policy only fixes segments on levels where merges
121
// have been triggered, so check invariants after all adds
122
for (int i = 0; i < 10; i++) {
125
checkInvariants(writer);
131
// Test the case where both mergeFactor and maxBufferedDocs change
132
public void testMaxBufferedDocsChange() throws IOException {
133
Directory dir = newDirectory();
135
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
136
TEST_VERSION_CURRENT, new MockAnalyzer(random))
137
.setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())
138
.setMergeScheduler(new SerialMergeScheduler()));
140
// leftmost* segment has 1 doc
141
// rightmost* segment has 100 docs
142
for (int i = 1; i <= 100; i++) {
143
for (int j = 0; j < i; j++) {
145
checkInvariants(writer);
149
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
150
new MockAnalyzer(random)).setOpenMode(
151
OpenMode.APPEND).setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())
152
.setMergeScheduler(new SerialMergeScheduler()));
156
LogDocMergePolicy ldmp = new LogDocMergePolicy();
157
ldmp.setMergeFactor(10);
158
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
159
new MockAnalyzer(random)).setOpenMode(
160
OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new SerialMergeScheduler()));
162
// merge policy only fixes segments on levels where merges
163
// have been triggered, so check invariants after all adds
164
for (int i = 0; i < 100; i++) {
167
checkInvariants(writer);
169
for (int i = 100; i < 1000; i++) {
173
writer.waitForMerges();
175
checkInvariants(writer);
181
// Test the case where a merge results in no doc at all
182
public void testMergeDocCount0() throws IOException {
183
Directory dir = newDirectory();
185
LogDocMergePolicy ldmp = new LogDocMergePolicy();
186
ldmp.setMergeFactor(100);
187
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
188
TEST_VERSION_CURRENT, new MockAnalyzer(random))
189
.setMaxBufferedDocs(10).setMergePolicy(ldmp));
191
for (int i = 0; i < 250; i++) {
193
checkInvariants(writer);
197
IndexReader reader = IndexReader.open(dir, false);
198
reader.deleteDocuments(new Term("content", "aaa"));
201
ldmp = new LogDocMergePolicy();
202
ldmp.setMergeFactor(5);
203
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
204
new MockAnalyzer(random)).setOpenMode(
205
OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler()));
207
// merge factor is changed, so check invariants after all adds
208
for (int i = 0; i < 10; i++) {
212
writer.waitForMerges();
214
checkInvariants(writer);
215
assertEquals(10, writer.maxDoc());
221
private void addDoc(IndexWriter writer) throws IOException {
222
Document doc = new Document();
223
doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
224
writer.addDocument(doc);
227
private void checkInvariants(IndexWriter writer) throws IOException {
228
writer.waitForMerges();
229
int maxBufferedDocs = writer.getConfig().getMaxBufferedDocs();
230
int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor();
231
int maxMergeDocs = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMaxMergeDocs();
233
int ramSegmentCount = writer.getNumBufferedDocuments();
234
assertTrue(ramSegmentCount < maxBufferedDocs);
237
int upperBound = maxBufferedDocs;
240
int segmentCount = writer.getSegmentCount();
241
for (int i = segmentCount - 1; i >= 0; i--) {
242
int docCount = writer.getDocCount(i);
243
assertTrue("docCount=" + docCount + " lowerBound=" + lowerBound + " upperBound=" + upperBound + " i=" + i + " segmentCount=" + segmentCount + " index=" + writer.segString() + " config=" + writer.getConfig(), docCount > lowerBound);
245
if (docCount <= upperBound) {
248
if (upperBound * mergeFactor <= maxMergeDocs) {
249
assertTrue("maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor + "; segs=" + writer.segString() + " config=" + writer.getConfig(), numSegments < mergeFactor);
253
lowerBound = upperBound;
254
upperBound *= mergeFactor;
255
} while (docCount > upperBound);
259
if (upperBound * mergeFactor <= maxMergeDocs) {
260
assertTrue(numSegments < mergeFactor);