1
package org.apache.lucene.util;
4
* Licensed to the Apache Software Foundation (ASF) under one or more
5
* contributor license agreements. See the NOTICE file distributed with
6
* this work for additional information regarding copyright ownership.
7
* The ASF licenses this file to You under the Apache License, Version 2.0
8
* (the "License"); you may not use this file except in compliance with
9
* the License. You may obtain a copy of the License at
11
* http://www.apache.org/licenses/LICENSE-2.0
13
* Unless required by applicable law or agreed to in writing, software
14
* distributed under the License is distributed on an "AS IS" BASIS,
15
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
* See the License for the specific language governing permissions and
17
* limitations under the License.
20
import java.io.BufferedOutputStream;
21
import java.io.ByteArrayOutputStream;
23
import java.io.FileOutputStream;
24
import java.io.IOException;
25
import java.io.InputStream;
26
import java.io.OutputStream;
27
import java.io.PrintStream;
28
import java.lang.reflect.Method;
29
import java.util.Enumeration;
30
import java.util.HashMap;
32
import java.util.Random;
33
import java.util.zip.ZipEntry;
34
import java.util.zip.ZipFile;
36
import org.junit.Assert;
38
import org.apache.lucene.document.Document;
39
import org.apache.lucene.document.Field;
40
import org.apache.lucene.document.Fieldable;
41
import org.apache.lucene.index.CheckIndex;
42
import org.apache.lucene.index.ConcurrentMergeScheduler;
43
import org.apache.lucene.index.IndexWriter;
44
import org.apache.lucene.index.LogMergePolicy;
45
import org.apache.lucene.index.MergePolicy;
46
import org.apache.lucene.index.MergeScheduler;
47
import org.apache.lucene.index.TieredMergePolicy;
48
import org.apache.lucene.search.FieldDoc;
49
import org.apache.lucene.search.ScoreDoc;
50
import org.apache.lucene.search.TopDocs;
51
import org.apache.lucene.store.Directory;
53
public class _TestUtil {
55
/** Returns temp dir, based on String arg in its name;
56
* does not create the directory. */
57
public static File getTempDir(String desc) {
59
File f = createTempFile(desc, "tmp", LuceneTestCase.TEMP_DIR);
61
LuceneTestCase.registerTempFile(f);
63
} catch (IOException e) {
64
throw new RuntimeException(e);
69
* Deletes a directory and everything underneath it.
71
public static void rmDir(File dir) throws IOException {
73
for (File f : dir.listFiles()) {
74
if (f.isDirectory()) {
78
throw new IOException("could not delete " + f);
83
throw new IOException("could not delete " + dir);
89
* Convenience method: Unzip zipName + ".zip" under destDir, removing destDir first
91
public static void unzip(File zipName, File destDir) throws IOException {
93
ZipFile zipFile = new ZipFile(zipName);
95
Enumeration<? extends ZipEntry> entries = zipFile.entries();
100
LuceneTestCase.registerTempFile(destDir);
102
while (entries.hasMoreElements()) {
103
ZipEntry entry = entries.nextElement();
105
InputStream in = zipFile.getInputStream(entry);
106
File targetFile = new File(destDir, entry.getName());
107
if (entry.isDirectory()) {
108
// allow unzipping with directory structure
111
if (targetFile.getParentFile()!=null) {
112
// be on the safe side: do not rely on that directories are always extracted
113
// before their children (although this makes sense, but is it guaranteed?)
114
targetFile.getParentFile().mkdirs();
116
OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
118
byte[] buffer = new byte[8192];
120
while((len = in.read(buffer)) >= 0) {
121
out.write(buffer, 0, len);
132
public static void syncConcurrentMerges(IndexWriter writer) {
133
syncConcurrentMerges(writer.getConfig().getMergeScheduler());
136
public static void syncConcurrentMerges(MergeScheduler ms) {
137
if (ms instanceof ConcurrentMergeScheduler)
138
((ConcurrentMergeScheduler) ms).sync();
141
/** This runs the CheckIndex tool on the index in. If any
142
* issues are hit, a RuntimeException is thrown; else,
143
* true is returned. */
144
public static CheckIndex.Status checkIndex(Directory dir) throws IOException {
145
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
147
CheckIndex checker = new CheckIndex(dir);
148
checker.setInfoStream(new PrintStream(bos));
149
CheckIndex.Status indexStatus = checker.checkIndex();
150
if (indexStatus == null || indexStatus.clean == false) {
151
System.out.println("CheckIndex failed");
152
System.out.println(bos.toString());
153
throw new RuntimeException("CheckIndex failed");
159
/** Use only for testing.
160
* @deprecated -- in 3.0 we can use Arrays.toString
163
public static String arrayToString(int[] array) {
164
StringBuilder buf = new StringBuilder();
166
for(int i=0;i<array.length;i++) {
170
buf.append(array[i]);
173
return buf.toString();
176
/** Use only for testing.
177
* @deprecated -- in 3.0 we can use Arrays.toString
180
public static String arrayToString(Object[] array) {
181
StringBuilder buf = new StringBuilder();
183
for(int i=0;i<array.length;i++) {
187
buf.append(array[i]);
190
return buf.toString();
193
public static String randomSimpleString(Random r) {
194
final int end = r.nextInt(10);
199
final char[] buffer = new char[end];
200
for (int i = 0; i < end; i++) {
201
buffer[i] = (char) _TestUtil.nextInt(r, 97, 102);
203
return new String(buffer, 0, end);
206
/** Returns random string, including full unicode range. */
207
public static String randomUnicodeString(Random r) {
208
return randomUnicodeString(r, 20);
212
* Returns a random string up to a certain length.
214
public static String randomUnicodeString(Random r, int maxLength) {
215
final int end = r.nextInt(maxLength);
220
final char[] buffer = new char[end];
221
randomFixedLengthUnicodeString(r, buffer, 0, buffer.length);
222
return new String(buffer, 0, end);
226
* Fills provided char[] with valid random unicode code
229
public static void randomFixedLengthUnicodeString(Random random, char[] chars, int offset, int length) {
231
final int end = offset + length;
233
final int t = random.nextInt(5);
234
if (0 == t && i < length - 1) {
235
// Make a surrogate pair
237
chars[i++] = (char) nextInt(random, 0xd800, 0xdbff);
239
chars[i++] = (char) nextInt(random, 0xdc00, 0xdfff);
241
chars[i++] = (char) random.nextInt(0x80);
243
chars[i++] = (char) nextInt(random, 0x80, 0x7ff);
245
chars[i++] = (char) nextInt(random, 0x800, 0xd7ff);
247
chars[i++] = (char) nextInt(random, 0xe000, 0xfffe);
252
private static final int[] blockStarts = {
253
0x0000, 0x0080, 0x0100, 0x0180, 0x0250, 0x02B0, 0x0300, 0x0370, 0x0400,
254
0x0500, 0x0530, 0x0590, 0x0600, 0x0700, 0x0750, 0x0780, 0x07C0, 0x0800,
255
0x0900, 0x0980, 0x0A00, 0x0A80, 0x0B00, 0x0B80, 0x0C00, 0x0C80, 0x0D00,
256
0x0D80, 0x0E00, 0x0E80, 0x0F00, 0x1000, 0x10A0, 0x1100, 0x1200, 0x1380,
257
0x13A0, 0x1400, 0x1680, 0x16A0, 0x1700, 0x1720, 0x1740, 0x1760, 0x1780,
258
0x1800, 0x18B0, 0x1900, 0x1950, 0x1980, 0x19E0, 0x1A00, 0x1A20, 0x1B00,
259
0x1B80, 0x1C00, 0x1C50, 0x1CD0, 0x1D00, 0x1D80, 0x1DC0, 0x1E00, 0x1F00,
260
0x2000, 0x2070, 0x20A0, 0x20D0, 0x2100, 0x2150, 0x2190, 0x2200, 0x2300,
261
0x2400, 0x2440, 0x2460, 0x2500, 0x2580, 0x25A0, 0x2600, 0x2700, 0x27C0,
262
0x27F0, 0x2800, 0x2900, 0x2980, 0x2A00, 0x2B00, 0x2C00, 0x2C60, 0x2C80,
263
0x2D00, 0x2D30, 0x2D80, 0x2DE0, 0x2E00, 0x2E80, 0x2F00, 0x2FF0, 0x3000,
264
0x3040, 0x30A0, 0x3100, 0x3130, 0x3190, 0x31A0, 0x31C0, 0x31F0, 0x3200,
265
0x3300, 0x3400, 0x4DC0, 0x4E00, 0xA000, 0xA490, 0xA4D0, 0xA500, 0xA640,
266
0xA6A0, 0xA700, 0xA720, 0xA800, 0xA830, 0xA840, 0xA880, 0xA8E0, 0xA900,
267
0xA930, 0xA960, 0xA980, 0xAA00, 0xAA60, 0xAA80, 0xABC0, 0xAC00, 0xD7B0,
268
0xE000, 0xF900, 0xFB00, 0xFB50, 0xFE00, 0xFE10,
269
0xFE20, 0xFE30, 0xFE50, 0xFE70, 0xFF00, 0xFFF0,
270
0x10000, 0x10080, 0x10100, 0x10140, 0x10190, 0x101D0, 0x10280, 0x102A0,
271
0x10300, 0x10330, 0x10380, 0x103A0, 0x10400, 0x10450, 0x10480, 0x10800,
272
0x10840, 0x10900, 0x10920, 0x10A00, 0x10A60, 0x10B00, 0x10B40, 0x10B60,
273
0x10C00, 0x10E60, 0x11080, 0x12000, 0x12400, 0x13000, 0x1D000, 0x1D100,
274
0x1D200, 0x1D300, 0x1D360, 0x1D400, 0x1F000, 0x1F030, 0x1F100, 0x1F200,
275
0x20000, 0x2A700, 0x2F800, 0xE0000, 0xE0100, 0xF0000, 0x100000
278
private static final int[] blockEnds = {
279
0x007F, 0x00FF, 0x017F, 0x024F, 0x02AF, 0x02FF, 0x036F, 0x03FF, 0x04FF,
280
0x052F, 0x058F, 0x05FF, 0x06FF, 0x074F, 0x077F, 0x07BF, 0x07FF, 0x083F,
281
0x097F, 0x09FF, 0x0A7F, 0x0AFF, 0x0B7F, 0x0BFF, 0x0C7F, 0x0CFF, 0x0D7F,
282
0x0DFF, 0x0E7F, 0x0EFF, 0x0FFF, 0x109F, 0x10FF, 0x11FF, 0x137F, 0x139F,
283
0x13FF, 0x167F, 0x169F, 0x16FF, 0x171F, 0x173F, 0x175F, 0x177F, 0x17FF,
284
0x18AF, 0x18FF, 0x194F, 0x197F, 0x19DF, 0x19FF, 0x1A1F, 0x1AAF, 0x1B7F,
285
0x1BBF, 0x1C4F, 0x1C7F, 0x1CFF, 0x1D7F, 0x1DBF, 0x1DFF, 0x1EFF, 0x1FFF,
286
0x206F, 0x209F, 0x20CF, 0x20FF, 0x214F, 0x218F, 0x21FF, 0x22FF, 0x23FF,
287
0x243F, 0x245F, 0x24FF, 0x257F, 0x259F, 0x25FF, 0x26FF, 0x27BF, 0x27EF,
288
0x27FF, 0x28FF, 0x297F, 0x29FF, 0x2AFF, 0x2BFF, 0x2C5F, 0x2C7F, 0x2CFF,
289
0x2D2F, 0x2D7F, 0x2DDF, 0x2DFF, 0x2E7F, 0x2EFF, 0x2FDF, 0x2FFF, 0x303F,
290
0x309F, 0x30FF, 0x312F, 0x318F, 0x319F, 0x31BF, 0x31EF, 0x31FF, 0x32FF,
291
0x33FF, 0x4DBF, 0x4DFF, 0x9FFF, 0xA48F, 0xA4CF, 0xA4FF, 0xA63F, 0xA69F,
292
0xA6FF, 0xA71F, 0xA7FF, 0xA82F, 0xA83F, 0xA87F, 0xA8DF, 0xA8FF, 0xA92F,
293
0xA95F, 0xA97F, 0xA9DF, 0xAA5F, 0xAA7F, 0xAADF, 0xABFF, 0xD7AF, 0xD7FF,
294
0xF8FF, 0xFAFF, 0xFB4F, 0xFDFF, 0xFE0F, 0xFE1F,
295
0xFE2F, 0xFE4F, 0xFE6F, 0xFEFF, 0xFFEF, 0xFFFE, /* avoid 0xFFFF on 3.x */
296
0x1007F, 0x100FF, 0x1013F, 0x1018F, 0x101CF, 0x101FF, 0x1029F, 0x102DF,
297
0x1032F, 0x1034F, 0x1039F, 0x103DF, 0x1044F, 0x1047F, 0x104AF, 0x1083F,
298
0x1085F, 0x1091F, 0x1093F, 0x10A5F, 0x10A7F, 0x10B3F, 0x10B5F, 0x10B7F,
299
0x10C4F, 0x10E7F, 0x110CF, 0x123FF, 0x1247F, 0x1342F, 0x1D0FF, 0x1D1FF,
300
0x1D24F, 0x1D35F, 0x1D37F, 0x1D7FF, 0x1F02F, 0x1F09F, 0x1F1FF, 0x1F2FF,
301
0x2A6DF, 0x2B73F, 0x2FA1F, 0xE007F, 0xE01EF, 0xFFFFF, 0x10FFFF
304
/** Returns random string of length between 0-20 codepoints, all codepoints within the same unicode block. */
305
public static String randomRealisticUnicodeString(Random r) {
306
return randomRealisticUnicodeString(r, 20);
309
/** Returns random string of length up to maxLength codepoints , all codepoints within the same unicode block. */
310
public static String randomRealisticUnicodeString(Random r, int maxLength) {
311
return randomRealisticUnicodeString(r, 0, 20);
314
/** Returns random string of length between min and max codepoints, all codepoints within the same unicode block. */
315
public static String randomRealisticUnicodeString(Random r, int minLength, int maxLength) {
316
final int end = minLength + r.nextInt(maxLength);
317
final int block = r.nextInt(blockStarts.length);
318
StringBuilder sb = new StringBuilder();
319
for (int i = 0; i < end; i++)
320
sb.appendCodePoint(nextInt(r, blockStarts[block], blockEnds[block]));
321
return sb.toString();
324
/** Returns random string, with a given UTF-8 byte length*/
325
public static String randomFixedByteLengthUnicodeString(Random r, int length) {
327
final char[] buffer = new char[length*3];
330
for (; i < buffer.length && bytes != 0; i++) {
334
} else if (bytes >= 3) {
336
} else if (bytes >= 2) {
342
buffer[i] = (char) r.nextInt(0x80);
345
buffer[i] = (char) nextInt(r, 0x80, 0x7ff);
348
buffer[i] = (char) nextInt(r, 0x800, 0xd7ff);
351
buffer[i] = (char) nextInt(r, 0xe000, 0xfffe);
354
// Make a surrogate pair
356
buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff);
358
buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff);
363
return new String(buffer, 0, i);
366
/** start and end are BOTH inclusive */
367
public static int nextInt(Random r, int start, int end) {
368
return start + r.nextInt(end-start+1);
371
public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException {
372
String[] files = dir.listAll();
373
if (files.length > 1 || (files.length == 1 && !files[0].equals("write.lock"))) {
380
/** just tries to configure things to keep the open file
382
public static void reduceOpenFiles(IndexWriter w) {
383
// keep number of open files lowish
384
MergePolicy mp = w.getConfig().getMergePolicy();
385
if (mp instanceof LogMergePolicy) {
386
LogMergePolicy lmp = (LogMergePolicy) mp;
387
lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
388
} else if (mp instanceof TieredMergePolicy) {
389
TieredMergePolicy tmp = (TieredMergePolicy) mp;
390
tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
391
tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
394
MergeScheduler ms = w.getConfig().getMergeScheduler();
395
if (ms instanceof ConcurrentMergeScheduler) {
396
((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);
397
((ConcurrentMergeScheduler) ms).setMaxMergeCount(3);
401
/** Checks some basic behaviour of an AttributeImpl
402
* @param reflectedValues contains a map with "AttributeClass#key" as values
404
public static <T> void assertAttributeReflection(final AttributeImpl att, Map<String,T> reflectedValues) {
405
final Map<String,Object> map = new HashMap<String,Object>();
406
att.reflectWith(new AttributeReflector() {
407
public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
408
map.put(attClass.getName() + '#' + key, value);
411
Assert.assertEquals("Reflection does not produce same map", reflectedValues, map);
414
public static void keepFullyDeletedSegments(IndexWriter w) {
416
// Carefully invoke what is a package-private (test
417
// only, internal) method on IndexWriter:
418
Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments");
419
m.setAccessible(true);
421
} catch (Exception e) {
422
// Should not happen?
423
throw new RuntimeException(e);
428
* insecure, fast version of File.createTempFile
429
* uses Random instead of SecureRandom.
431
public static File createTempFile(String prefix, String suffix, File directory)
433
// Force a prefix null check first
434
if (prefix.length() < 3) {
435
throw new IllegalArgumentException("prefix must be 3");
437
String newSuffix = suffix == null ? ".tmp" : suffix;
440
result = genTempFile(prefix, newSuffix, directory);
441
} while (!result.createNewFile());
445
/* Temp file counter */
446
private static int counter = 0;
448
/* identify for differnt VM processes */
449
private static int counterBase = 0;
451
private static class TempFileLocker {};
452
private static TempFileLocker tempFileLocker = new TempFileLocker();
454
private static File genTempFile(String prefix, String suffix, File directory) {
457
synchronized (tempFileLocker) {
459
int newInt = new Random().nextInt();
460
counter = ((newInt / 65535) & 0xFFFF) + 0x2710;
461
counterBase = counter;
463
identify = counter++;
466
StringBuilder newName = new StringBuilder();
467
newName.append(prefix);
468
newName.append(counterBase);
469
newName.append(identify);
470
newName.append(suffix);
471
return new File(directory, newName.toString());
474
public static void assertEquals(TopDocs expected, TopDocs actual) {
475
Assert.assertEquals("wrong total hits", expected.totalHits, actual.totalHits);
476
Assert.assertEquals("wrong maxScore", expected.getMaxScore(), actual.getMaxScore(), 0.0);
477
Assert.assertEquals("wrong hit count", expected.scoreDocs.length, actual.scoreDocs.length);
478
for(int hitIDX=0;hitIDX<expected.scoreDocs.length;hitIDX++) {
479
final ScoreDoc expectedSD = expected.scoreDocs[hitIDX];
480
final ScoreDoc actualSD = actual.scoreDocs[hitIDX];
481
Assert.assertEquals("wrong hit docID", expectedSD.doc, actualSD.doc);
482
Assert.assertEquals("wrong hit score", expectedSD.score, actualSD.score, 0.0);
483
if (expectedSD instanceof FieldDoc) {
484
Assert.assertTrue(actualSD instanceof FieldDoc);
485
Assert.assertEquals("wrong sort field values",
486
((FieldDoc) expectedSD).fields,
487
((FieldDoc) actualSD).fields);
489
Assert.assertFalse(actualSD instanceof FieldDoc);
494
// NOTE: this is likely buggy, and cannot clone fields
495
// with tokenStreamValues, etc. Use at your own risk!!
497
// TODO: is there a pre-existing way to do this!!!
498
public static Document cloneDocument(Document doc1) {
499
final Document doc2 = new Document();
500
for(Fieldable f : doc1.getFields()) {
501
Field field1 = (Field) f;
503
Field field2 = new Field(field1.name(),
504
field1.stringValue(),
505
field1.isStored() ? Field.Store.YES : Field.Store.NO,
506
field1.isIndexed() ? (field1.isTokenized() ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED) : Field.Index.NO);
507
field2.setOmitNorms(field1.getOmitNorms());
508
field2.setIndexOptions(field1.getIndexOptions());