~slub.team/goobi-indexserver/3.x

assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage*150),

252

midDiskUsage < 150*startDiskUsage);

253

assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage*150),

254

endDiskUsage < 150*startDiskUsage);

255

dir.close();

256

}

257

258

259

260

* Verify that calling forceMerge when writer is open for

261

* "commit on close" works correctly both for rollback()

262

* and close().

263

264

public void testCommitOnCloseForceMerge() throws IOException {

265

MockDirectoryWrapper dir = newDirectory();

266

// Must disable throwing exc on double-write: this

267

// test uses IW.rollback which easily results in

268

// writing to same file more than once

269

dir.setPreventDoubleWrite(false);

270

IndexWriter writer = new IndexWriter(

271

dir,

272

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

273

setMaxBufferedDocs(10).

274

setMergePolicy(newLogMergePolicy(10))

275

);

276

for(int j=0;j<17;j++) {

277

TestIndexWriter.addDocWithIndex(writer, j);

278

}

279

writer.close();

280

281

writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));

282

writer.forceMerge(1);

283

284

if (VERBOSE) {

285

writer.setInfoStream(System.out);

286

}

287

288

// Open a reader before closing (commiting) the writer:

289

IndexReader reader = IndexReader.open(dir, true);

290

291

// Reader should see index as multi-seg at this

292

// point:

293

assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().length > 1);

294

reader.close();

295

296

// Abort the writer:

297

writer.rollback();

298

TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after forceMerge");

299

300

// Open a reader after aborting writer:

301

reader = IndexReader.open(dir, true);

302

303

// Reader should still see index as multi-segment

304

assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().length > 1);

305

reader.close();

306

307

if (VERBOSE) {

308

System.out.println("TEST: do real full merge");

309

}

310

writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));

311

if (VERBOSE) {

312

writer.setInfoStream(System.out);

313

}

314

writer.forceMerge(1);

315

writer.close();

316

317

if (VERBOSE) {

318

System.out.println("TEST: writer closed");

319

}

320

TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after forceMerge");

321

322

// Open a reader after aborting writer:

323

reader = IndexReader.open(dir, true);

324

325

// Reader should see index as one segment

326

assertEquals("Reader incorrectly sees more than one segment", 1, reader.getSequentialSubReaders().length);

327

reader.close();

328

dir.close();

329

}

330

331

// LUCENE-2095: make sure with multiple threads commit

332

// doesn't return until all changes are in fact in the

333

// index

334

public void testCommitThreadSafety() throws Throwable {

335

final int NUM_THREADS = 5;

336

final double RUN_SEC = 0.5;

337

final Directory dir = newDirectory();

338

final RandomIndexWriter w = new RandomIndexWriter(random, dir, newIndexWriterConfig(

339

TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));

340

_TestUtil.reduceOpenFiles(w.w);

341

w.commit();

342

final AtomicBoolean failed = new AtomicBoolean();

343

Thread[] threads = new Thread[NUM_THREADS];

344

final long endTime = System.currentTimeMillis()+((long) (RUN_SEC*1000));

345

for(int i=0;i<NUM_THREADS;i++) {

346

final int finalI = i;

347

threads[i] = new Thread() {

348

@Override

349

public void run() {

350

try {

351

final Document doc = new Document();

352

IndexReader r = IndexReader.open(dir);

353

Field f = newField("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED);

354

doc.add(f);

355

int count = 0;

356

do {

357

if (failed.get()) break;

358

for(int j=0;j<10;j++) {

359

final String s = finalI + "_" + String.valueOf(count++);

360

f.setValue(s);

361

w.addDocument(doc);

362

w.commit();

363

IndexReader r2 = IndexReader.openIfChanged(r);

364

assertNotNull(r2);

365

assertTrue(r2 != r);

366

r.close();

367

r = r2;

368

assertEquals("term=f:" + s + "; r=" + r, 1, r.docFreq(new Term("f", s)));

369

}

370

} while(System.currentTimeMillis() < endTime);

371

r.close();

372

} catch (Throwable t) {

373

failed.set(true);

374

throw new RuntimeException(t);

375

}

376

}

377

};

378

threads[i].start();

379

}

380

for(int i=0;i<NUM_THREADS;i++) {

381

threads[i].join();

382

}

383

assertFalse(failed.get());

384

w.close();

385

dir.close();

386

}

387

388

// LUCENE-1044: test writer.commit() when ac=false

389

public void testForceCommit() throws IOException {

390

Directory dir = newDirectory();

391

392

IndexWriter writer = new IndexWriter(

393

dir,

394

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

395

setMaxBufferedDocs(2).

396

setMergePolicy(newLogMergePolicy(5))

397

);

398

writer.commit();

399

400

for (int i = 0; i < 23; i++)

401

TestIndexWriter.addDoc(writer);

402

403

IndexReader reader = IndexReader.open(dir, true);

404

assertEquals(0, reader.numDocs());

405

writer.commit();

406

IndexReader reader2 = IndexReader.openIfChanged(reader);

407

assertNotNull(reader2);

408

assertEquals(0, reader.numDocs());

409

assertEquals(23, reader2.numDocs());

410

reader.close();

411

412

for (int i = 0; i < 17; i++)

413

TestIndexWriter.addDoc(writer);

414

assertEquals(23, reader2.numDocs());

415

reader2.close();

416

reader = IndexReader.open(dir, true);

417

assertEquals(23, reader.numDocs());

418

reader.close();

419

writer.commit();

420

421

reader = IndexReader.open(dir, true);

422

assertEquals(40, reader.numDocs());

423

reader.close();

424

writer.close();

425

dir.close();

426

}

427

428

public void testFutureCommit() throws Exception {

429

Directory dir = newDirectory();

430

431

IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));

432

Document doc = new Document();

433

w.addDocument(doc);

434

435

// commit to "first"

436

Map<String,String> commitData = new HashMap<String,String>();

437

commitData.put("tag", "first");

438

w.commit(commitData);

439

440

// commit to "second"

441

w.addDocument(doc);

442

commitData.put("tag", "second");

443

w.commit(commitData);

444

w.close();

445

446

// open "first" with IndexWriter

447

IndexCommit commit = null;

448

for(IndexCommit c : IndexReader.listCommits(dir)) {

449

if (c.getUserData().get("tag").equals("first")) {

450

commit = c;

451

break;

452

}

453

}

454

455

assertNotNull(commit);

456

457

w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE).setIndexCommit(commit));

458

459

assertEquals(1, w.numDocs());

460

461

// commit IndexWriter to "third"

462

w.addDocument(doc);

463

commitData.put("tag", "third");

464

w.commit(commitData);

465

w.close();

466

467

// make sure "second" commit is still there

468

commit = null;

469

for(IndexCommit c : IndexReader.listCommits(dir)) {

470

if (c.getUserData().get("tag").equals("second")) {

471

commit = c;

472

break;

473

}

474

}

475

476

assertNotNull(commit);

477

478

IndexReader r = IndexReader.open(commit, true);

479

assertEquals(2, r.numDocs());

480

r.close();

481

482

// open "second", w/ writeable IndexReader & commit

483

r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false);

484

assertEquals(2, r.numDocs());

485

r.deleteDocument(0);

486

r.deleteDocument(1);

487

commitData.put("tag", "fourth");

488

r.commit(commitData);

489

r.close();

490

491

// make sure "third" commit is still there

492

commit = null;

493

for(IndexCommit c : IndexReader.listCommits(dir)) {

494

if (c.getUserData().get("tag").equals("third")) {

495

commit = c;

496

break;

497

}

498

}

499

assertNotNull(commit);

500

501

dir.close();

502

}

503

504

public void testNoCommits() throws Exception {

505

// Tests that if we don't call commit(), the directory has 0 commits. This has

506

// changed since LUCENE-2386, where before IW would always commit on a fresh

507

// new index.

508

Directory dir = newDirectory();

509

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

510

try {

511

IndexReader.listCommits(dir);

512

fail("listCommits should have thrown an exception over empty index");

513

} catch (IndexNotFoundException e) {

514

// that's expected !

515

}

516

// No changes still should generate a commit, because it's a new index.

517

writer.close();

518

assertEquals("expected 1 commits!", 1, IndexReader.listCommits(dir).size());

519

dir.close();

520

}

521

522

// LUCENE-1274: test writer.prepareCommit()

523

public void testPrepareCommit() throws IOException {

524

Directory dir = newDirectory();

525

526

IndexWriter writer = new IndexWriter(

527

dir,

528

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

529

setMaxBufferedDocs(2).

530

setMergePolicy(newLogMergePolicy(5))

531

);

532

writer.commit();

533

534

for (int i = 0; i < 23; i++)

535

TestIndexWriter.addDoc(writer);

536

537

IndexReader reader = IndexReader.open(dir, true);

538

assertEquals(0, reader.numDocs());

539

540

writer.prepareCommit();

541

542

IndexReader reader2 = IndexReader.open(dir, true);

543

assertEquals(0, reader2.numDocs());

544

545

writer.commit();

546

547

IndexReader reader3 = IndexReader.openIfChanged(reader);

548

assertNotNull(reader3);

549

assertEquals(0, reader.numDocs());

550

assertEquals(0, reader2.numDocs());

551

assertEquals(23, reader3.numDocs());

552

reader.close();

553

reader2.close();

554

555

for (int i = 0; i < 17; i++)

556

TestIndexWriter.addDoc(writer);

557

558

assertEquals(23, reader3.numDocs());

559

reader3.close();

560

reader = IndexReader.open(dir, true);

561

assertEquals(23, reader.numDocs());

562

reader.close();

563

564

writer.prepareCommit();

565

566

reader = IndexReader.open(dir, true);

567

assertEquals(23, reader.numDocs());

568

reader.close();

569

570

writer.commit();

571

reader = IndexReader.open(dir, true);

572

assertEquals(40, reader.numDocs());

573

reader.close();

574

writer.close();

575

dir.close();

576

}

577

578

// LUCENE-1274: test writer.prepareCommit()

579

public void testPrepareCommitRollback() throws IOException {

580

MockDirectoryWrapper dir = newDirectory();

581

dir.setPreventDoubleWrite(false);

582

583

IndexWriter writer = new IndexWriter(

584

dir,

585

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

586

setMaxBufferedDocs(2).

587

setMergePolicy(newLogMergePolicy(5))

588

);

589

writer.commit();

590

591

for (int i = 0; i < 23; i++)

592

TestIndexWriter.addDoc(writer);

593

594

IndexReader reader = IndexReader.open(dir, true);

595

assertEquals(0, reader.numDocs());

596

597

writer.prepareCommit();

598

599

IndexReader reader2 = IndexReader.open(dir, true);

600

assertEquals(0, reader2.numDocs());

601

602

writer.rollback();

603

604

IndexReader reader3 = IndexReader.openIfChanged(reader);

605

assertNull(reader3);

606

assertEquals(0, reader.numDocs());

607

assertEquals(0, reader2.numDocs());

608

reader.close();

609

reader2.close();

610

611

writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

612

for (int i = 0; i < 17; i++)

613

TestIndexWriter.addDoc(writer);

614

615

reader = IndexReader.open(dir, true);

616

assertEquals(0, reader.numDocs());

617

reader.close();

618

619

writer.prepareCommit();

620

621

reader = IndexReader.open(dir, true);

622

assertEquals(0, reader.numDocs());

623

reader.close();

624

625

writer.commit();

626

reader = IndexReader.open(dir, true);

627

assertEquals(17, reader.numDocs());

628

reader.close();

629

writer.close();

630

dir.close();

631

}

632

633

// LUCENE-1274

634

public void testPrepareCommitNoChanges() throws IOException {

635

Directory dir = newDirectory();

636

637

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

638

writer.prepareCommit();

639

writer.commit();

640

writer.close();

641

642

IndexReader reader = IndexReader.open(dir, true);

643

assertEquals(0, reader.numDocs());

644

reader.close();

645

dir.close();

646

}

647

648

// LUCENE-1382

649

public void testCommitUserData() throws IOException {

650

Directory dir = newDirectory();

651

IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));

652

for(int j=0;j<17;j++)

653

TestIndexWriter.addDoc(w);

654

w.close();

655

656

assertEquals(0, IndexReader.getCommitUserData(dir).size());

657

658

IndexReader r = IndexReader.open(dir, true);

659

// commit(Map) never called for this index

660

assertEquals(0, r.getCommitUserData().size());

661

r.close();

662

663

w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));

664

for(int j=0;j<17;j++)

665

TestIndexWriter.addDoc(w);

666

Map<String,String> data = new HashMap<String,String>();

667

data.put("label", "test1");

668

w.commit(data);

669

w.close();

670

671

assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));

672

673

r = IndexReader.open(dir, true);

674

assertEquals("test1", r.getCommitUserData().get("label"));

675

r.close();

676

677

w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

678

w.forceMerge(1);

679

w.close();

680

681

assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));

682

683

dir.close();

684

}

685

}

Older »