~slub.team/goobi-indexserver/3.x

IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());

865

((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(100);

866

IndexWriter w = new IndexWriter(startDir, conf);

867

for(int i=0;i<27;i++)

868

addDoc(w);

869

w.close();

870

871

int iter = TEST_NIGHTLY ? 200 : 20;

872

for(int i=0;i<iter;i++) {

873

if (VERBOSE) {

874

System.out.println("TEST: iter " + i);

875

}

876

MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));

877

conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler());

878

((ConcurrentMergeScheduler) conf.getMergeScheduler()).setSuppressExceptions();

879

w = new IndexWriter(dir, conf);

880

w.setInfoStream(VERBOSE ? System.out : null);

881

dir.setRandomIOExceptionRate(0.5);

882

try {

883

w.optimize();

884

} catch (IOException ioe) {

885

if (ioe.getCause() == null)

886

fail("optimize threw IOException without root cause");

887

}

888

dir.setRandomIOExceptionRate(0);

889

w.close();

890

dir.close();

891

}

892

startDir.close();

893

}

894

895

// LUCENE-1429

896

public void testOutOfMemoryErrorCausesCloseToFail() throws Exception {

897

898

final List<Throwable> thrown = new ArrayList<Throwable>();

899

final Directory dir = newDirectory();

900

final IndexWriter writer = new IndexWriter(dir,

901

newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))) {

902

@Override

903

public void message(final String message) {

904

if (message.startsWith("now flush at close") && 0 == thrown.size()) {

905

thrown.add(null);

906

throw new OutOfMemoryError("fake OOME at " + message);

907

}

908

}

909

};

910

911

// need to set an info stream so message is called

912

writer.setInfoStream(new PrintStream(new ByteArrayOutputStream()));

913

try {

914

writer.close();

915

fail("OutOfMemoryError expected");

916

}

917

catch (final OutOfMemoryError expected) {}

918

919

// throws IllegalStateEx w/o bug fix

920

writer.close();

921

dir.close();

922

}

923

924

// LUCENE-1347

925

private static final class MockIndexWriter4 extends IndexWriter {

926

927

public MockIndexWriter4(Directory dir, IndexWriterConfig conf) throws IOException {

928

super(dir, conf);

929

}

930

931

boolean doFail;

932

933

@Override

934

boolean testPoint(String name) {

935

if (doFail && name.equals("rollback before checkpoint"))

936

throw new RuntimeException("intentionally failing");

937

return true;

938

}

939

}

940

941

// LUCENE-1347

942

public void testRollbackExceptionHang() throws Throwable {

943

Directory dir = newDirectory();

944

MockIndexWriter4 w = new MockIndexWriter4(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

945

946

addDoc(w);

947

w.doFail = true;

948

try {

949

w.rollback();

950

fail("did not hit intentional RuntimeException");

951

} catch (RuntimeException re) {

952

// expected

953

}

954

955

w.doFail = false;

956

w.rollback();

957

dir.close();

958

}

959

960

// LUCENE-1044: Simulate checksum error in segments_N

961

public void testSegmentsChecksumError() throws IOException {

962

Directory dir = newDirectory();

963

964

IndexWriter writer = null;

965

966

writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

967

968

// add 100 documents

969

for (int i = 0; i < 100; i++) {

970

addDoc(writer);

971

}

972

973

// close

974

writer.close();

975

976

long gen = SegmentInfos.getCurrentSegmentGeneration(dir);

977

assertTrue("segment generation should be > 0 but got " + gen, gen > 0);

978

979

final String segmentsFileName = SegmentInfos.getCurrentSegmentFileName(dir);

980

IndexInput in = dir.openInput(segmentsFileName);

981

IndexOutput out = dir.createOutput(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1+gen));

982

out.copyBytes(in, in.length()-1);

983

byte b = in.readByte();

984

out.writeByte((byte) (1+b));

985

out.close();

986

in.close();

987

988

IndexReader reader = null;

989

try {

990

reader = IndexReader.open(dir, true);

991

} catch (IOException e) {

992

e.printStackTrace(System.out);

993

fail("segmentInfos failed to retry fallback to correct segments_N file");

994

}

995

reader.close();

996

dir.close();

997

}

998

999

// Simulate a corrupt index by removing last byte of

1000

// latest segments file and make sure we get an

1001

// IOException trying to open the index:

1002

public void testSimulatedCorruptIndex1() throws IOException {

1003

MockDirectoryWrapper dir = newDirectory();

1004

dir.setCheckIndexOnClose(false); // we are corrupting it!

1005

1006

IndexWriter writer = null;

1007

1008

writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1009

1010

// add 100 documents

1011

for (int i = 0; i < 100; i++) {

1012

addDoc(writer);

1013

}

1014

1015

// close

1016

writer.close();

1017

1018

long gen = SegmentInfos.getCurrentSegmentGeneration(dir);

1019

assertTrue("segment generation should be > 0 but got " + gen, gen > 0);

1020

1021

String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);

1022

String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,

1023

"",

1024

1+gen);

1025

IndexInput in = dir.openInput(fileNameIn);

1026

IndexOutput out = dir.createOutput(fileNameOut);

1027

long length = in.length();

1028

for(int i=0;i<length-1;i++) {

1029

out.writeByte(in.readByte());

1030

}

1031

in.close();

1032

out.close();

1033

dir.deleteFile(fileNameIn);

1034

1035

IndexReader reader = null;

1036

try {

1037

reader = IndexReader.open(dir, true);

1038

fail("reader did not hit IOException on opening a corrupt index");

1039

} catch (Exception e) {

1040

}

1041

if (reader != null) {

1042

reader.close();

1043

}

1044

dir.close();

1045

}

1046

1047

// Simulate a corrupt index by removing one of the cfs

1048

// files and make sure we get an IOException trying to

1049

// open the index:

1050

public void testSimulatedCorruptIndex2() throws IOException {

1051

MockDirectoryWrapper dir = newDirectory();

1052

dir.setCheckIndexOnClose(false); // we are corrupting it!

1053

IndexWriter writer = null;

1054

1055

writer = new IndexWriter(

1056

dir,

1057

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

1058

setMergePolicy(newLogMergePolicy(true))

1059

);

1060

((LogMergePolicy) writer.getConfig().getMergePolicy()).setNoCFSRatio(1.0);

1061

1062

// add 100 documents

1063

for (int i = 0; i < 100; i++) {

1064

addDoc(writer);

1065

}

1066

1067

// close

1068

writer.close();

1069

1070

long gen = SegmentInfos.getCurrentSegmentGeneration(dir);

1071

assertTrue("segment generation should be > 0 but got " + gen, gen > 0);

1072

1073

String[] files = dir.listAll();

1074

boolean corrupted = false;

1075

for(int i=0;i<files.length;i++) {

1076

if (files[i].endsWith(".cfs")) {

1077

dir.deleteFile(files[i]);

1078

corrupted = true;

1079

break;

1080

}

1081

}

1082

assertTrue("failed to find cfs file to remove", corrupted);

1083

1084

IndexReader reader = null;

1085

try {

1086

reader = IndexReader.open(dir, true);

1087

fail("reader did not hit IOException on opening a corrupt index");

1088

} catch (Exception e) {

1089

}

1090

if (reader != null) {

1091

reader.close();

1092

}

1093

dir.close();

1094

}

1095

1096

// Simulate a writer that crashed while writing segments

1097

// file: make sure we can still open the index (ie,

1098

// gracefully fallback to the previous segments file),

1099

// and that we can add to the index:

1100

public void testSimulatedCrashedWriter() throws IOException {

1101

MockDirectoryWrapper dir = newDirectory();

1102

dir.setPreventDoubleWrite(false);

1103

1104

IndexWriter writer = null;

1105

1106

writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1107

1108

// add 100 documents

1109

for (int i = 0; i < 100; i++) {

1110

addDoc(writer);

1111

}

1112

1113

// close

1114

writer.close();

1115

1116

long gen = SegmentInfos.getCurrentSegmentGeneration(dir);

1117

assertTrue("segment generation should be > 0 but got " + gen, gen > 0);

1118

1119

// Make the next segments file, with last byte

1120

// missing, to simulate a writer that crashed while

1121

// writing segments file:

1122

String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);

1123

String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,

1124

"",

1125

1+gen);

1126

IndexInput in = dir.openInput(fileNameIn);

1127

IndexOutput out = dir.createOutput(fileNameOut);

1128

long length = in.length();

1129

for(int i=0;i<length-1;i++) {

1130

out.writeByte(in.readByte());

1131

}

1132

in.close();

1133

out.close();

1134

1135

IndexReader reader = null;

1136

try {

1137

reader = IndexReader.open(dir, true);

1138

} catch (Exception e) {

1139

fail("reader failed to open on a crashed index");

1140

}

1141

reader.close();

1142

1143

try {

1144

writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));

1145

} catch (Exception e) {

1146

e.printStackTrace(System.out);

1147

fail("writer failed to open on a crashed index");

1148

}

1149

1150

// add 100 documents

1151

for (int i = 0; i < 100; i++) {

1152

addDoc(writer);

1153

}

1154

1155

// close

1156

writer.close();

1157

dir.close();

1158

}

1159

1160

public void testAddDocsNonAbortingException() throws Exception {

1161

final Directory dir = newDirectory();

1162

final RandomIndexWriter w = new RandomIndexWriter(random, dir);

1163

final int numDocs1 = random.nextInt(25);

1164

for(int docCount=0;docCount<numDocs1;docCount++) {

1165

Document doc = new Document();

1166

doc.add(newField("content", "good content", Field.Index.ANALYZED));

1167

w.addDocument(doc);

1168

}

1169

1170

final List<Document> docs = new ArrayList<Document>();

1171

for(int docCount=0;docCount<7;docCount++) {

1172

Document doc = new Document();

1173

docs.add(doc);

1174

doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));

1175

doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));

1176

if (docCount == 4) {

1177

Field f = newField("crash", "", Field.Index.ANALYZED);

1178

doc.add(f);

1179

MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);

1180

tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.

1181

f.setTokenStream(new CrashingFilter("crash", tokenizer));

1182

}

1183

}

1184

try {

1185

w.addDocuments(docs);

1186

// BUG: CrashingFilter didn't

1187

fail("did not hit expected exception");

1188

} catch (IOException ioe) {

1189

// expected

1190

assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());

1191

}

1192

1193

final int numDocs2 = random.nextInt(25);

1194

for(int docCount=0;docCount<numDocs2;docCount++) {

1195

Document doc = new Document();

1196

doc.add(newField("content", "good content", Field.Index.ANALYZED));

1197

w.addDocument(doc);

1198

}

1199

1200

final IndexReader r = w.getReader();

1201

w.close();

1202

1203

final IndexSearcher s = new IndexSearcher(r);

1204

PhraseQuery pq = new PhraseQuery();

1205

pq.add(new Term("content", "silly"));

1206

pq.add(new Term("content", "content"));

1207

assertEquals(0, s.search(pq, 1).totalHits);

1208

1209

pq = new PhraseQuery();

1210

pq.add(new Term("content", "good"));

1211

pq.add(new Term("content", "content"));

1212

assertEquals(numDocs1+numDocs2, s.search(pq, 1).totalHits);

1213

r.close();

1214

dir.close();

1215

}

1216

1217

1218

public void testUpdateDocsNonAbortingException() throws Exception {

1219

final Directory dir = newDirectory();

1220

final RandomIndexWriter w = new RandomIndexWriter(random, dir);

1221

final int numDocs1 = random.nextInt(25);

1222

for(int docCount=0;docCount<numDocs1;docCount++) {

1223

Document doc = new Document();

1224

doc.add(newField("content", "good content", Field.Index.ANALYZED));

1225

w.addDocument(doc);

1226

}

1227

1228

// Use addDocs (no exception) to get docs in the index:

1229

final List<Document> docs = new ArrayList<Document>();

1230

final int numDocs2 = random.nextInt(25);

1231

for(int docCount=0;docCount<numDocs2;docCount++) {

1232

Document doc = new Document();

1233

docs.add(doc);

1234

doc.add(newField("subid", "subs", Field.Index.NOT_ANALYZED));

1235

doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));

1236

doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));

1237

}

1238

w.addDocuments(docs);

1239

1240

final int numDocs3 = random.nextInt(25);

1241

for(int docCount=0;docCount<numDocs3;docCount++) {

1242

Document doc = new Document();

1243

doc.add(newField("content", "good content", Field.Index.ANALYZED));

1244

w.addDocument(doc);

1245

}

1246

1247

docs.clear();

1248

final int limit = _TestUtil.nextInt(random, 2, 25);

1249

final int crashAt = random.nextInt(limit);

1250

for(int docCount=0;docCount<limit;docCount++) {

1251

Document doc = new Document();

1252

docs.add(doc);

1253

doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));

1254

doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));

1255

if (docCount == crashAt) {

1256

Field f = newField("crash", "", Field.Index.ANALYZED);

1257

doc.add(f);

1258

MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);

1259

tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.

1260

f.setTokenStream(new CrashingFilter("crash", tokenizer));

1261

}

1262

}

1263

1264

try {

1265

w.updateDocuments(new Term("subid", "subs"), docs);

1266

// BUG: CrashingFilter didn't

1267

fail("did not hit expected exception");

1268

} catch (IOException ioe) {

1269

// expected

1270

assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());

1271

}

1272

1273

final int numDocs4 = random.nextInt(25);

1274

for(int docCount=0;docCount<numDocs4;docCount++) {

1275

Document doc = new Document();

1276

doc.add(newField("content", "good content", Field.Index.ANALYZED));

1277

w.addDocument(doc);

1278

}

1279

1280

final IndexReader r = w.getReader();

1281

w.close();

1282

1283

final IndexSearcher s = new IndexSearcher(r);

1284

PhraseQuery pq = new PhraseQuery();

1285

pq.add(new Term("content", "silly"));

1286

pq.add(new Term("content", "content"));

1287

assertEquals(numDocs2, s.search(pq, 1).totalHits);

1288

1289

pq = new PhraseQuery();

1290

pq.add(new Term("content", "good"));

1291

pq.add(new Term("content", "content"));

1292

assertEquals(numDocs1+numDocs3+numDocs4, s.search(pq, 1).totalHits);

1293

r.close();

1294

dir.close();

1295

}

1296

}

Older »