~slub.team/goobi-indexserver/3.x

IndexWriter mainWriter = new IndexWriter(mainDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));

364

_TestUtil.reduceOpenFiles(mainWriter);

365

366

mainWriter.setInfoStream(infoStream);

367

AddDirectoriesThreads addDirThreads = new AddDirectoriesThreads(numIter, mainWriter);

368

addDirThreads.launchThreads(numDirs);

369

addDirThreads.joinThreads();

370

371

//assertEquals(100 + numDirs * (3 * numIter / 4) * addDirThreads.NUM_THREADS

372

// * addDirThreads.NUM_INIT_DOCS, addDirThreads.mainWriter.numDocs());

373

assertEquals(addDirThreads.count.intValue(), addDirThreads.mainWriter.numDocs());

374

375

addDirThreads.close(true);

376

377

assertTrue(addDirThreads.failures.size() == 0);

378

379

_TestUtil.checkIndex(mainDir);

380

381

IndexReader reader = IndexReader.open(mainDir, true);

382

assertEquals(addDirThreads.count.intValue(), reader.numDocs());

383

//assertEquals(100 + numDirs * (3 * numIter / 4) * addDirThreads.NUM_THREADS

384

// * addDirThreads.NUM_INIT_DOCS, reader.numDocs());

385

reader.close();

386

387

addDirThreads.closeDir();

388

mainDir.close();

389

}

390

391

private class AddDirectoriesThreads {

392

Directory addDir;

393

final static int NUM_THREADS = 5;

394

final static int NUM_INIT_DOCS = 100;

395

int numDirs;

396

final Thread[] threads = new Thread[NUM_THREADS];

397

IndexWriter mainWriter;

398

final List<Throwable> failures = new ArrayList<Throwable>();

399

IndexReader[] readers;

400

boolean didClose = false;

401

AtomicInteger count = new AtomicInteger(0);

402

AtomicInteger numaddIndexes = new AtomicInteger(0);

403

404

public AddDirectoriesThreads(int numDirs, IndexWriter mainWriter) throws Throwable {

405

this.numDirs = numDirs;

406

this.mainWriter = mainWriter;

407

addDir = newDirectory();

408

IndexWriter writer = new IndexWriter(addDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));

409

for (int i = 0; i < NUM_INIT_DOCS; i++) {

410

Document doc = DocHelper.createDocument(i, "addindex", 4);

411

writer.addDocument(doc);

412

}

413

414

writer.close();

415

416

readers = new IndexReader[numDirs];

417

for (int i = 0; i < numDirs; i++)

418

readers[i] = IndexReader.open(addDir, false);

419

}

420

421

void joinThreads() {

422

for (int i = 0; i < NUM_THREADS; i++)

423

try {

424

threads[i].join();

425

} catch (InterruptedException ie) {

426

throw new ThreadInterruptedException(ie);

427

}

428

}

429

430

void close(boolean doWait) throws Throwable {

431

didClose = true;

432

if (doWait) {

433

mainWriter.waitForMerges();

434

}

435

mainWriter.close(doWait);

436

}

437

438

void closeDir() throws Throwable {

439

for (int i = 0; i < numDirs; i++)

440

readers[i].close();

441

addDir.close();

442

}

443

444

void handle(Throwable t) {

445

t.printStackTrace(System.out);

446

synchronized (failures) {

447

failures.add(t);

448

}

449

}

450

451

void launchThreads(final int numIter) {

452

for (int i = 0; i < NUM_THREADS; i++) {

453

threads[i] = new Thread() {

454

@Override

455

public void run() {

456

try {

457

final Directory[] dirs = new Directory[numDirs];

458

for (int k = 0; k < numDirs; k++)

459

dirs[k] = new MockDirectoryWrapper(random, new RAMDirectory(addDir));

460

//int j = 0;

461

//while (true) {

462

// System.out.println(Thread.currentThread().getName() + ": iter

463

// j=" + j);

464

for (int x=0; x < numIter; x++) {

465

// only do addIndexes

466

doBody(x, dirs);

467

}

468

//if (numIter > 0 && j == numIter)

469

// break;

470

//doBody(j++, dirs);

471

//doBody(5, dirs);

472

//}

473

} catch (Throwable t) {

474

handle(t);

475

}

476

}

477

};

478

}

479

for (int i = 0; i < NUM_THREADS; i++)

480

threads[i].start();

481

}

482

483

void doBody(int j, Directory[] dirs) throws Throwable {

484

switch (j % 4) {

485

case 0:

486

mainWriter.addIndexes(dirs);

487

mainWriter.forceMerge(1);

488

break;

489

case 1:

490

mainWriter.addIndexes(dirs);

491

numaddIndexes.incrementAndGet();

492

break;

493

case 2:

494

mainWriter.addIndexes(readers);

495

break;

496

case 3:

497

mainWriter.commit();

498

}

499

count.addAndGet(dirs.length*NUM_INIT_DOCS);

500

}

501

}

502

503

public void testIndexWriterReopenSegmentFullMerge() throws Exception {

504

doTestIndexWriterReopenSegment(true);

505

}

506

507

public void testIndexWriterReopenSegment() throws Exception {

508

doTestIndexWriterReopenSegment(false);

509

}

510

511

/**

512

* Tests creating a segment, then check to insure the segment can be seen via

513

* IW.getReader

514

515

public void doTestIndexWriterReopenSegment(boolean doFullMerge) throws Exception {

516

Directory dir1 = newDirectory();

517

IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

518

writer.setInfoStream(infoStream);

519

IndexReader r1 = writer.getReader();

520

assertEquals(0, r1.maxDoc());

521

createIndexNoClose(false, "index1", writer);

522

writer.flush(!doFullMerge, true);

523

524

IndexReader iwr1 = writer.getReader();

525

assertEquals(100, iwr1.maxDoc());

526

527

IndexReader r2 = writer.getReader();

528

assertEquals(r2.maxDoc(), 100);

529

// add 100 documents

530

for (int x = 10000; x < 10000 + 100; x++) {

531

Document d = DocHelper.createDocument(x, "index1", 5);

532

writer.addDocument(d);

533

}

534

writer.flush(false, true);

535

// verify the reader was reopened internally

536

IndexReader iwr2 = writer.getReader();

537

assertTrue(iwr2 != r1);

538

assertEquals(200, iwr2.maxDoc());

539

// should have flushed out a segment

540

IndexReader r3 = writer.getReader();

541

assertTrue(r2 != r3);

542

assertEquals(200, r3.maxDoc());

543

544

// dec ref the readers rather than close them because

545

// closing flushes changes to the writer

546

r1.close();

547

iwr1.close();

548

r2.close();

549

r3.close();

550

iwr2.close();

551

writer.close();

552

553

// test whether the changes made it to the directory

554

writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

555

IndexReader w2r1 = writer.getReader();

556

// insure the deletes were actually flushed to the directory

557

assertEquals(200, w2r1.maxDoc());

558

w2r1.close();

559

writer.close();

560

561

dir1.close();

562

}

563

564

565

* Delete a document by term and return the doc id

566

567

* public static int deleteDocument(Term term, IndexWriter writer) throws

568

* IOException { IndexReader reader = writer.getReader(); TermDocs td =

569

* reader.termDocs(term); int doc = -1; //if (td.next()) { // doc = td.doc();

570

* //} //writer.deleteDocuments(term); td.close(); return doc; }

571

572

573

public static void createIndex(Random random, Directory dir1, String indexName,

574

boolean multiSegment) throws IOException {

575

IndexWriter w = new IndexWriter(dir1, LuceneTestCase.newIndexWriterConfig(random,

576

TEST_VERSION_CURRENT, new MockAnalyzer(random))

577

.setMergePolicy(new LogDocMergePolicy()));

578

for (int i = 0; i < 100; i++) {

579

w.addDocument(DocHelper.createDocument(i, indexName, 4));

580

if (multiSegment && (i % 10) == 0) {

581

}

582

}

583

if (!multiSegment) {

584

w.forceMerge(1);

585

}

586

w.close();

587

}

588

589

public static void createIndexNoClose(boolean multiSegment, String indexName,

590

IndexWriter w) throws IOException {

591

for (int i = 0; i < 100; i++) {

592

w.addDocument(DocHelper.createDocument(i, indexName, 4));

593

}

594

if (!multiSegment) {

595

w.forceMerge(1);

596

}

597

}

598

599

private static class MyWarmer extends IndexWriter.IndexReaderWarmer {

600

int warmCount;

601

@Override

602

public void warm(IndexReader reader) throws IOException {

603

warmCount++;

604

}

605

}

606

607

public void testMergeWarmer() throws Exception {

608

609

Directory dir1 = newDirectory();

610

// Enroll warmer

611

MyWarmer warmer = new MyWarmer();

612

IndexWriter writer = new IndexWriter(

613

dir1,

614

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

615

setMaxBufferedDocs(2).

616

setMergedSegmentWarmer(warmer).

617

setMergeScheduler(new ConcurrentMergeScheduler()).

618

setMergePolicy(newLogMergePolicy())

619

);

620

writer.setInfoStream(infoStream);

621

622

// create the index

623

createIndexNoClose(false, "test", writer);

624

625

// get a reader to put writer into near real-time mode

626

IndexReader r1 = writer.getReader();

627

628

((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);

629

630

int num = atLeast(100);

631

for (int i = 0; i < num; i++) {

632

writer.addDocument(DocHelper.createDocument(i, "test", 4));

633

}

634

((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();

635

636

assertTrue(warmer.warmCount > 0);

637

final int count = warmer.warmCount;

638

639

writer.addDocument(DocHelper.createDocument(17, "test", 4));

640

writer.forceMerge(1);

641

assertTrue(warmer.warmCount > count);

642

643

writer.close();

644

r1.close();

645

dir1.close();

646

}

647

648

public void testAfterCommit() throws Exception {

649

Directory dir1 = newDirectory();

650

IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler()));

651

writer.commit();

652

writer.setInfoStream(infoStream);

653

654

// create the index

655

createIndexNoClose(false, "test", writer);

656

657

// get a reader to put writer into near real-time mode

658

IndexReader r1 = writer.getReader();

659

_TestUtil.checkIndex(dir1);

660

writer.commit();

661

_TestUtil.checkIndex(dir1);

662

assertEquals(100, r1.numDocs());

663

664

for (int i = 0; i < 10; i++) {

665

writer.addDocument(DocHelper.createDocument(i, "test", 4));

666

}

667

((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();

668

669

IndexReader r2 = IndexReader.openIfChanged(r1);

670

if (r2 != null) {

671

r1.close();

672

r1 = r2;

673

}

674

assertEquals(110, r1.numDocs());

675

writer.close();

676

r1.close();

677

dir1.close();

678

}

679

680

// Make sure reader remains usable even if IndexWriter closes

681

public void testAfterClose() throws Exception {

682

Directory dir1 = newDirectory();

683

IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

684

writer.setInfoStream(infoStream);

685

686

// create the index

687

createIndexNoClose(false, "test", writer);

688

689

IndexReader r = writer.getReader();

690

writer.close();

691

692

_TestUtil.checkIndex(dir1);

693

694

// reader should remain usable even after IndexWriter is closed:

695

assertEquals(100, r.numDocs());

696

Query q = new TermQuery(new Term("indexname", "test"));

697

IndexSearcher searcher = newSearcher(r);

698

assertEquals(100, searcher.search(q, 10).totalHits);

699

searcher.close();

700

try {

701

IndexReader.openIfChanged(r);

702

fail("failed to hit AlreadyClosedException");

703

} catch (AlreadyClosedException ace) {

704

// expected

705

}

706

r.close();

707

dir1.close();

708

}

709

710

// Stress test reopen during addIndexes

711

public void testDuringAddIndexes() throws Exception {

712

MockDirectoryWrapper dir1 = newDirectory();

713

final IndexWriter writer = new IndexWriter(

714

dir1,

715

newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).

716

setMergePolicy(newLogMergePolicy(2))

717

);

718

writer.setInfoStream(infoStream);

719

((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);

720

721

// create the index

722

createIndexNoClose(false, "test", writer);

723

writer.commit();

724

725

final Directory[] dirs = new Directory[10];

726

for (int i=0;i<10;i++) {

727

dirs[i] = new MockDirectoryWrapper(random, new RAMDirectory(dir1));

728

}

729

730

IndexReader r = writer.getReader();

731

732

final int NUM_THREAD = 5;

733

final float SECONDS = 0.5f;

734

735

final long endTime = (long) (System.currentTimeMillis() + 1000.*SECONDS);

736

final List<Throwable> excs = Collections.synchronizedList(new ArrayList<Throwable>());

737

738

final Thread[] threads = new Thread[NUM_THREAD];

739

for(int i=0;i<NUM_THREAD;i++) {

740

threads[i] = new Thread() {

741

@Override

742

public void run() {

743

do {

744

try {

745

writer.addIndexes(dirs);

746

writer.maybeMerge();

747

} catch (Throwable t) {

748

excs.add(t);

749

throw new RuntimeException(t);

750

}

751

} while(System.currentTimeMillis() < endTime);

752

}

753

};

754

threads[i].setDaemon(true);

755

threads[i].start();

756

}

757

758

int lastCount = 0;

759

while(System.currentTimeMillis() < endTime) {

760

IndexReader r2 = IndexReader.openIfChanged(r);

761

if (r2 != null) {

762

r.close();

763

r = r2;

764

}

765

Query q = new TermQuery(new Term("indexname", "test"));

766

IndexSearcher searcher = newSearcher(r);

767

final int count = searcher.search(q, 10).totalHits;

768

searcher.close();

769

assertTrue(count >= lastCount);

770

lastCount = count;

771

}

772

773

for(int i=0;i<NUM_THREAD;i++) {

774

threads[i].join();

775

}

776

// final check

777

IndexReader r2 = IndexReader.openIfChanged(r);

778

if (r2 != null) {

779

r.close();

780

r = r2;

781

}

782

Query q = new TermQuery(new Term("indexname", "test"));

783

IndexSearcher searcher = newSearcher(r);

784

final int count = searcher.search(q, 10).totalHits;

785

searcher.close();

786

assertTrue(count >= lastCount);

787

788

assertEquals(0, excs.size());

789

r.close();

790

assertEquals(0, dir1.getOpenDeletedFiles().size());

791

792

writer.close();

793

794

dir1.close();

795

}

796

797

// Stress test reopen during add/delete

798

public void testDuringAddDelete() throws Exception {

799

Directory dir1 = newDirectory();

800

final IndexWriter writer = new IndexWriter(

801

dir1,

802

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

803

setMergePolicy(newLogMergePolicy(2))

804

);

805

writer.setInfoStream(infoStream);

806

((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);

807

808

// create the index

809

createIndexNoClose(false, "test", writer);

810

writer.commit();

811

812

IndexReader r = writer.getReader();

813

814

final int NUM_THREAD = 5;

815

final float SECONDS = 0.5f;

816

817

final long endTime = (long) (System.currentTimeMillis() + 1000.*SECONDS);

818

final List<Throwable> excs = Collections.synchronizedList(new ArrayList<Throwable>());

819

820

final Thread[] threads = new Thread[NUM_THREAD];

821

for(int i=0;i<NUM_THREAD;i++) {

822

threads[i] = new Thread() {

823

final Random r = new Random(random.nextLong());

824

825

@Override

826

public void run() {

827

int count = 0;

828

do {

829

try {

830

for(int docUpto=0;docUpto<10;docUpto++) {

831

writer.addDocument(DocHelper.createDocument(10*count+docUpto, "test", 4));

832

}

833

count++;

834

final int limit = count*10;

835

for(int delUpto=0;delUpto<5;delUpto++) {

836

int x = r.nextInt(limit);

837

writer.deleteDocuments(new Term("field3", "b"+x));

838

}

839

} catch (Throwable t) {

840

excs.add(t);

841

throw new RuntimeException(t);

842

}

843

} while(System.currentTimeMillis() < endTime);

844

}

845

};

846

threads[i].setDaemon(true);

847

threads[i].start();

848

}

849

850

int sum = 0;

851

while(System.currentTimeMillis() < endTime) {

852

IndexReader r2 = IndexReader.openIfChanged(r);

853

if (r2 != null) {

854

r.close();

855

r = r2;

856

}

857

Query q = new TermQuery(new Term("indexname", "test"));

858

IndexSearcher searcher = newSearcher(r);

859

sum += searcher.search(q, 10).totalHits;

860

searcher.close();

861

}

862

863

for(int i=0;i<NUM_THREAD;i++) {

864

threads[i].join();

865

}

866

// at least search once

867

IndexReader r2 = IndexReader.openIfChanged(r);

868

if (r2 != null) {

869

r.close();

870

r = r2;

871

}

872

Query q = new TermQuery(new Term("indexname", "test"));

873

IndexSearcher searcher = newSearcher(r);

874

sum += searcher.search(q, 10).totalHits;

875

searcher.close();

876

assertTrue("no documents found at all", sum > 0);

877

878

assertEquals(0, excs.size());

879

writer.close();

880

881

r.close();

882

dir1.close();

883

}

884

885

public void testForceMergeDeletes() throws Throwable {

886

Directory dir = newDirectory();

887

final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));

888

Document doc = new Document();

889

doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED));

890

Field id = newField("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED);

891

doc.add(id);

892

id.setValue("0");

893

w.addDocument(doc);

894

id.setValue("1");

895

w.addDocument(doc);

896

w.deleteDocuments(new Term("id", "0"));

897

898

IndexReader r = w.getReader();

899

w.forceMergeDeletes();

900

w.close();

901

r.close();

902

r = IndexReader.open(dir, true);

903

assertEquals(1, r.numDocs());

904

assertFalse(r.hasDeletions());

905

r.close();

906

dir.close();

907

}

908

909

public void testDeletesNumDocs() throws Throwable {

910

Directory dir = newDirectory();

911

final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

912

Document doc = new Document();

913

doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED));

914

Field id = newField("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED);

915

doc.add(id);

916

id.setValue("0");

917

w.addDocument(doc);

918

id.setValue("1");

919

w.addDocument(doc);

920

IndexReader r = w.getReader();

921

assertEquals(2, r.numDocs());

922

r.close();

923

924

w.deleteDocuments(new Term("id", "0"));

925

r = w.getReader();

926

assertEquals(1, r.numDocs());

927

r.close();

928

929

w.deleteDocuments(new Term("id", "1"));

930

r = w.getReader();

931

assertEquals(0, r.numDocs());

932

r.close();

933

934

w.close();

935

dir.close();

936

}

937

938

public void testEmptyIndex() throws Exception {

939

// Ensures that getReader works on an empty index, which hasn't been committed yet.

940

Directory dir = newDirectory();

941

IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));

942

IndexReader r = w.getReader();

943

assertEquals(0, r.numDocs());

944

r.close();

945

w.close();

946

dir.close();

947

}

948

949

public void testSegmentWarmer() throws Exception {

950

Directory dir = newDirectory();

951

final AtomicBoolean didWarm = new AtomicBoolean();

952

IndexWriter w = new IndexWriter(

953

dir,

954

newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).

955

setMaxBufferedDocs(2).

956

setReaderPooling(true).

957

setMergedSegmentWarmer(new IndexWriter.IndexReaderWarmer() {

958

@Override

959

public void warm(IndexReader r) throws IOException {

960

IndexSearcher s = newSearcher(r);

961

TopDocs hits = s.search(new TermQuery(new Term("foo", "bar")), 10);

962

assertEquals(20, hits.totalHits);

963

didWarm.set(true);

964

s.close();

965

}

966

}).

967

setMergePolicy(newLogMergePolicy(10))

968

);

969

970

Document doc = new Document();

971

doc.add(newField("foo", "bar", Field.Store.YES, Field.Index.NOT_ANALYZED));

972

for(int i=0;i<20;i++) {

973

w.addDocument(doc);

974

}

975

w.waitForMerges();

976

w.close();

977

dir.close();

978

assertTrue(didWarm.get());

979

}

980

981

public void testNoTermsIndex() throws Exception {

982

Directory dir = newDirectory();

983

IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(

984

TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))

985

.setReaderTermsIndexDivisor(-1));

986

Document doc = new Document();

987

doc.add(new Field("f", "val", Store.NO, Index.ANALYZED));

988

w.addDocument(doc);

989

IndexReader r = IndexReader.open(w, true);

990

try {

991

r.termDocs(new Term("f", "val"));

992

fail("should have failed to seek since terms index was not loaded");

993

} catch (IllegalStateException e) {

994

// expected - we didn't load the term index

995

} finally {

996

r.close();

997

w.close();

998

dir.close();

999

}

1000

}

1001

1002

public void testReopenAfterNoRealChange() throws Exception {

1003

Directory d = newDirectory();

1004

IndexWriter w = new IndexWriter(

1005

1006

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1007

w.setInfoStream(VERBOSE ? System.out : null);

1008

1009

IndexReader r = w.getReader(); // start pooling readers

1010

1011

IndexReader r2 = IndexReader.openIfChanged(r);

1012

assertNull(r2);

1013

1014

w.addDocument(new Document());

1015

IndexReader r3 = IndexReader.openIfChanged(r);

1016

assertNotNull(r3);

1017

assertTrue(r3.getVersion() != r.getVersion());

1018

assertTrue(r3.isCurrent());

1019

1020

// Deletes nothing in reality...:

1021

w.deleteDocuments(new Term("foo", "bar"));

1022

1023

// ... but IW marks this as not current:

1024

assertFalse(r3.isCurrent());

1025

IndexReader r4 = IndexReader.openIfChanged(r3);

1026

assertNull(r4);

1027

1028

// Deletes nothing in reality...:

1029

w.deleteDocuments(new Term("foo", "bar"));

1030

IndexReader r5 = IndexReader.openIfChanged(r3, w, true);

1031

assertNull(r5);

1032

1033

r3.close();

1034

1035

w.close();

1036

d.close();

1037

}

1038

}

Older »