~slub.team/goobi-indexserver/3.x

doc.add(new Field("tvpositionoffset","one two two three three three", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));

277

278

writer.addDocument(doc);

279

}

280

writer.close();

281

IndexReader reader = IndexReader.open(d, false);

282

FieldSortedTermVectorMapper mapper = new FieldSortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());

283

reader.getTermFreqVector(0, mapper);

284

Map<String,SortedSet<TermVectorEntry>> map = mapper.getFieldToTerms();

285

assertTrue("map is null and it shouldn't be", map != null);

286

assertTrue("map Size: " + map.size() + " is not: " + 4, map.size() == 4);

287

Set<TermVectorEntry> set = map.get("termvector");

288

for (Iterator<TermVectorEntry> iterator = set.iterator(); iterator.hasNext();) {

289

TermVectorEntry entry = iterator.next();

290

assertTrue("entry is null and it shouldn't be", entry != null);

291

if (VERBOSE) System.out.println("Entry: " + entry);

292

}

293

reader.close();

294

d.close();

295

}

296

297

static void assertTermDocsCount(String msg,

298

IndexReader reader,

299

Term term,

300

int expected)

301

throws IOException

302

{

303

TermDocs tdocs = null;

304

305

try {

306

tdocs = reader.termDocs(term);

307

assertNotNull(msg + ", null TermDocs", tdocs);

308

int count = 0;

309

while(tdocs.next()) {

310

count++;

311

}

312

assertEquals(msg + ", count mismatch", expected, count);

313

314

} finally {

315

if (tdocs != null)

316

tdocs.close();

317

}

318

319

}

320

321

322

public void testBinaryFields() throws IOException {

323

Directory dir = newDirectory();

324

byte[] bin = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};

325

326

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));

327

328

for (int i = 0; i < 10; i++) {

329

addDoc(writer, "document number " + (i + 1));

330

addDocumentWithFields(writer);

331

addDocumentWithDifferentFields(writer);

332

addDocumentWithTermVectorFields(writer);

333

}

334

writer.close();

335

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));

336

Document doc = new Document();

337

doc.add(new Field("bin1", bin));

338

doc.add(new Field("junk", "junk text", Field.Store.NO, Field.Index.ANALYZED));

339

writer.addDocument(doc);

340

writer.close();

341

IndexReader reader = IndexReader.open(dir, false);

342

doc = reader.document(reader.maxDoc() - 1);

343

Field[] fields = doc.getFields("bin1");

344

assertNotNull(fields);

345

assertEquals(1, fields.length);

346

Field b1 = fields[0];

347

assertTrue(b1.isBinary());

348

byte[] data1 = b1.getBinaryValue();

349

assertEquals(bin.length, b1.getBinaryLength());

350

for (int i = 0; i < bin.length; i++) {

351

assertEquals(bin[i], data1[i + b1.getBinaryOffset()]);

352

}

353

Set<String> lazyFields = new HashSet<String>();

354

lazyFields.add("bin1");

355

FieldSelector sel = new SetBasedFieldSelector(new HashSet<String>(), lazyFields);

356

doc = reader.document(reader.maxDoc() - 1, sel);

357

Fieldable[] fieldables = doc.getFieldables("bin1");

358

assertNotNull(fieldables);

359

assertEquals(1, fieldables.length);

360

Fieldable fb1 = fieldables[0];

361

assertTrue(fb1.isBinary());

362

assertEquals(bin.length, fb1.getBinaryLength());

363

data1 = fb1.getBinaryValue();

364

assertEquals(bin.length, fb1.getBinaryLength());

365

for (int i = 0; i < bin.length; i++) {

366

assertEquals(bin[i], data1[i + fb1.getBinaryOffset()]);

367

}

368

reader.close();

369

// force merge

370

371

372

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));

373

writer.forceMerge(1);

374

writer.close();

375

reader = IndexReader.open(dir, false);

376

doc = reader.document(reader.maxDoc() - 1);

377

fields = doc.getFields("bin1");

378

assertNotNull(fields);

379

assertEquals(1, fields.length);

380

b1 = fields[0];

381

assertTrue(b1.isBinary());

382

data1 = b1.getBinaryValue();

383

assertEquals(bin.length, b1.getBinaryLength());

384

for (int i = 0; i < bin.length; i++) {

385

assertEquals(bin[i], data1[i + b1.getBinaryOffset()]);

386

}

387

reader.close();

388

dir.close();

389

}

390

391

// Make sure attempts to make changes after reader is

392

// closed throws IOException:

393

public void testChangesAfterClose() throws IOException {

394

Directory dir = newDirectory();

395

396

IndexWriter writer = null;

397

IndexReader reader = null;

398

Term searchTerm = new Term("content", "aaa");

399

400

// add 11 documents with term : aaa

401

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

402

for (int i = 0; i < 11; i++) {

403

addDoc(writer, searchTerm.text());

404

}

405

writer.close();

406

407

reader = IndexReader.open(dir, false);

408

409

// Close reader:

410

reader.close();

411

412

// Then, try to make changes:

413

try {

414

reader.deleteDocument(4);

415

fail("deleteDocument after close failed to throw IOException");

416

} catch (AlreadyClosedException e) {

417

// expected

418

}

419

420

try {

421

reader.setNorm(5, "aaa", 2.0f);

422

fail("setNorm after close failed to throw IOException");

423

} catch (AlreadyClosedException e) {

424

// expected

425

}

426

427

try {

428

reader.undeleteAll();

429

fail("undeleteAll after close failed to throw IOException");

430

} catch (AlreadyClosedException e) {

431

// expected

432

}

433

dir.close();

434

}

435

436

// Make sure we get lock obtain failed exception with 2 writers:

437

public void testLockObtainFailed() throws IOException {

438

Directory dir = newDirectory();

439

440

Term searchTerm = new Term("content", "aaa");

441

442

// add 11 documents with term : aaa

443

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

444

writer.commit();

445

for (int i = 0; i < 11; i++) {

446

addDoc(writer, searchTerm.text());

447

}

448

449

// Create reader:

450

IndexReader reader = IndexReader.open(dir, false);

451

452

// Try to make changes

453

try {

454

reader.deleteDocument(4);

455

fail("deleteDocument should have hit LockObtainFailedException");

456

} catch (LockObtainFailedException e) {

457

// expected

458

}

459

460

try {

461

reader.setNorm(5, "aaa", 2.0f);

462

fail("setNorm should have hit LockObtainFailedException");

463

} catch (LockObtainFailedException e) {

464

// expected

465

}

466

467

try {

468

reader.undeleteAll();

469

fail("undeleteAll should have hit LockObtainFailedException");

470

} catch (LockObtainFailedException e) {

471

// expected

472

}

473

writer.close();

474

reader.close();

475

dir.close();

476

}

477

478

// Make sure you can set norms & commit even if a reader

479

// is open against the index:

480

public void testWritingNorms() throws IOException {

481

Directory dir = newDirectory();

482

IndexWriter writer;

483

IndexReader reader;

484

Term searchTerm = new Term("content", "aaa");

485

486

// add 1 documents with term : aaa

487

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

488

addDoc(writer, searchTerm.text());

489

writer.close();

490

491

// now open reader & set norm for doc 0

492

reader = IndexReader.open(dir, false);

493

reader.setNorm(0, "content", (float) 2.0);

494

495

// we should be holding the write lock now:

496

assertTrue("locked", IndexWriter.isLocked(dir));

497

498

reader.commit();

499

500

// we should not be holding the write lock now:

501

assertTrue("not locked", !IndexWriter.isLocked(dir));

502

503

// open a 2nd reader:

504

IndexReader reader2 = IndexReader.open(dir, false);

505

506

// set norm again for doc 0

507

reader.setNorm(0, "content", (float) 3.0);

508

assertTrue("locked", IndexWriter.isLocked(dir));

509

510

reader.close();

511

512

// we should not be holding the write lock now:

513

assertTrue("not locked", !IndexWriter.isLocked(dir));

514

515

reader2.close();

516

dir.close();

517

}

518

519

520

// Make sure you can set norms & commit, and there are

521

// no extra norms files left:

522

public void testWritingNormsNoReader() throws IOException {

523

Directory dir = newDirectory();

524

IndexWriter writer = null;

525

IndexReader reader = null;

526

Term searchTerm = new Term("content", "aaa");

527

528

// add 1 documents with term : aaa

529

writer = new IndexWriter(

530

dir,

531

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

532

setMergePolicy(newLogMergePolicy(false))

533

);

534

addDoc(writer, searchTerm.text());

535

writer.close();

536

537

// now open reader & set norm for doc 0 (writes to

538

// _0_1.s0)

539

reader = IndexReader.open(dir, false);

540

reader.setNorm(0, "content", (float) 2.0);

541

reader.close();

542

543

// now open reader again & set norm for doc 0 (writes to _0_2.s0)

544

reader = IndexReader.open(dir, false);

545

reader.setNorm(0, "content", (float) 2.0);

546

reader.close();

547

assertFalse("failed to remove first generation norms file on writing second generation",

548

dir.fileExists("_0_1.s0"));

549

550

dir.close();

551

}

552

553

/* ??? public void testOpenEmptyDirectory() throws IOException{

554

String dirName = "test.empty";

555

File fileDirName = new File(dirName);

556

if (!fileDirName.exists()) {

557

fileDirName.mkdir();

558

}

559

try {

560

IndexReader.open(fileDirName);

561

fail("opening IndexReader on empty directory failed to produce FileNotFoundException");

562

} catch (FileNotFoundException e) {

563

// GOOD

564

}

565

rmDir(fileDirName);

566

}*/

567

568

public void testFilesOpenClose() throws IOException {

569

// Create initial data set

570

File dirFile = _TestUtil.getTempDir("TestIndexReader.testFilesOpenClose");

571

Directory dir = newFSDirectory(dirFile);

572

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

573

addDoc(writer, "test");

574

writer.close();

575

dir.close();

576

577

// Try to erase the data - this ensures that the writer closed all files

578

_TestUtil.rmDir(dirFile);

579

dir = newFSDirectory(dirFile);

580

581

// Now create the data set again, just as before

582

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));

583

addDoc(writer, "test");

584

writer.close();

585

dir.close();

586

587

// Now open existing directory and test that reader closes all files

588

dir = newFSDirectory(dirFile);

589

IndexReader reader1 = IndexReader.open(dir, false);

590

reader1.close();

591

dir.close();

592

593

// The following will fail if reader did not close

594

// all files

595

_TestUtil.rmDir(dirFile);

596

}

597

598

public void testLastModified() throws Exception {

599

for(int i=0;i<2;i++) {

600

final Directory dir = newDirectory();

601

assertFalse(IndexReader.indexExists(dir));

602

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));

603

addDocumentWithFields(writer);

604

assertTrue(IndexWriter.isLocked(dir)); // writer open, so dir is locked

605

writer.close();

606

assertTrue(IndexReader.indexExists(dir));

607

IndexReader reader = IndexReader.open(dir, false);

608

assertFalse(IndexWriter.isLocked(dir)); // reader only, no lock

609

long version = IndexReader.lastModified(dir);

610

if (i == 1) {

611

long version2 = IndexReader.lastModified(dir);

612

assertEquals(version, version2);

613

}

614

reader.close();

615

// modify index and check version has been

616

// incremented:

617

Thread.sleep(1000);

618

619

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));

620

addDocumentWithFields(writer);

621

writer.close();

622

reader = IndexReader.open(dir, false);

623

assertTrue("old lastModified is " + version + "; new lastModified is " + IndexReader.lastModified(dir), version <= IndexReader.lastModified(dir));

624

reader.close();

625

dir.close();

626

}

627

}

628

629

public void testVersion() throws IOException {

630

Directory dir = newDirectory();

631

assertFalse(IndexReader.indexExists(dir));

632

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

633

addDocumentWithFields(writer);

634

assertTrue(IndexWriter.isLocked(dir)); // writer open, so dir is locked

635

writer.close();

636

assertTrue(IndexReader.indexExists(dir));

637

IndexReader reader = IndexReader.open(dir, false);

638

assertFalse(IndexWriter.isLocked(dir)); // reader only, no lock

639

long version = IndexReader.getCurrentVersion(dir);

640

reader.close();

641

// modify index and check version has been

642

// incremented:

643

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));

644

addDocumentWithFields(writer);

645

writer.close();

646

reader = IndexReader.open(dir, false);

647

assertTrue("old version is " + version + "; new version is " + IndexReader.getCurrentVersion(dir), version < IndexReader.getCurrentVersion(dir));

648

reader.close();

649

dir.close();

650

}

651

652

public void testLock() throws IOException {

653

Directory dir = newDirectory();

654

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

655

addDocumentWithFields(writer);

656

writer.close();

657

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));

658

IndexReader reader = IndexReader.open(dir, false);

659

try {

660

reader.deleteDocument(0);

661

fail("expected lock");

662

} catch(IOException e) {

663

// expected exception

664

}

665

try {

666

IndexWriter.unlock(dir); // this should not be done in the real world!

667

} catch (LockReleaseFailedException lrfe) {

668

writer.close();

669

}

670

reader.deleteDocument(0);

671

reader.close();

672

writer.close();

673

dir.close();

674

}

675

676

public void testDocsOutOfOrderJIRA140() throws IOException {

677

Directory dir = newDirectory();

678

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

679

for(int i=0;i<11;i++) {

680

addDoc(writer, "aaa");

681

}

682

writer.close();

683

IndexReader reader = IndexReader.open(dir, false);

684

685

// Try to delete an invalid docId, yet, within range

686

// of the final bits of the BitVector:

687

688

boolean gotException = false;

689

try {

690

reader.deleteDocument(11);

691

} catch (ArrayIndexOutOfBoundsException e) {

692

gotException = true;

693

}

694

reader.close();

695

696

writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));

697

698

// We must add more docs to get a new segment written

699

for(int i=0;i<11;i++) {

700

addDoc(writer, "aaa");

701

}

702

703

// Without the fix for LUCENE-140 this call will

704

// [incorrectly] hit a "docs out of order"

705

// IllegalStateException because above out-of-bounds

706

// deleteDocument corrupted the index:

707

writer.forceMerge(1);

708

writer.close();

709

if (!gotException) {

710

fail("delete of out-of-bounds doc number failed to hit exception");

711

}

712

dir.close();

713

}

714

715

public void testExceptionReleaseWriteLockJIRA768() throws IOException {

716

717

Directory dir = newDirectory();

718

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

719

addDoc(writer, "aaa");

720

writer.close();

721

722

IndexReader reader = IndexReader.open(dir, false);

723

try {

724

reader.deleteDocument(1);

725

fail("did not hit exception when deleting an invalid doc number");

726

} catch (ArrayIndexOutOfBoundsException e) {

727

// expected

728

}

729

reader.close();

730

if (IndexWriter.isLocked(dir)) {

731

fail("write lock is still held after close");

732

}

733

734

reader = IndexReader.open(dir, false);

735

try {

736

reader.setNorm(1, "content", (float) 2.0);

737

fail("did not hit exception when calling setNorm on an invalid doc number");

738

} catch (ArrayIndexOutOfBoundsException e) {

739

// expected

740

}

741

reader.close();

742

if (IndexWriter.isLocked(dir)) {

743

fail("write lock is still held after close");

744

}

745

dir.close();

746

}

747

748

private String arrayToString(String[] l) {

749

String s = "";

750

for(int i=0;i<l.length;i++) {

751

if (i > 0) {

752

s += "\n ";

753

}

754

s += l[i];

755

}

756

return s;

757

}

758

759

public void testOpenReaderAfterDelete() throws IOException {

760

File dirFile = _TestUtil.getTempDir("deletetest");

761

Directory dir = newFSDirectory(dirFile);

762

try {

763

IndexReader.open(dir, false);

764

fail("expected FileNotFoundException");

765

} catch (FileNotFoundException e) {

766

// expected

767

}

768

769

dirFile.delete();

770

771

// Make sure we still get a CorruptIndexException (not NPE):

772

try {

773

IndexReader.open(dir, false);

774

fail("expected FileNotFoundException");

775

} catch (FileNotFoundException e) {

776

// expected

777

}

778

779

dir.close();

780

}

781

782

static void addDocumentWithFields(IndexWriter writer) throws IOException

783

{

784

Document doc = new Document();

785

doc.add(newField("keyword","test1", Field.Store.YES, Field.Index.NOT_ANALYZED));

786

doc.add(newField("text","test1", Field.Store.YES, Field.Index.ANALYZED));

787

doc.add(newField("unindexed","test1", Field.Store.YES, Field.Index.NO));

788

doc.add(newField("unstored","test1", Field.Store.NO, Field.Index.ANALYZED));

789

writer.addDocument(doc);

790

}

791

792

static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException

793

{

794

Document doc = new Document();

795

doc.add(newField("keyword2","test1", Field.Store.YES, Field.Index.NOT_ANALYZED));

796

doc.add(newField("text2","test1", Field.Store.YES, Field.Index.ANALYZED));

797

doc.add(newField("unindexed2","test1", Field.Store.YES, Field.Index.NO));

798

doc.add(newField("unstored2","test1", Field.Store.NO, Field.Index.ANALYZED));

799

writer.addDocument(doc);

800

}

801

802

static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException

803

{

804

Document doc = new Document();

805

doc.add(newField("tvnot","tvnot", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));

806

doc.add(newField("termvector","termvector", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES));

807

doc.add(newField("tvoffset","tvoffset", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_OFFSETS));

808

doc.add(newField("tvposition","tvposition", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS));

809

doc.add(newField("tvpositionoffset","tvpositionoffset", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));

810

811

writer.addDocument(doc);

812

}

813

814

static void addDoc(IndexWriter writer, String value) throws IOException {

815

Document doc = new Document();

816

doc.add(newField("content", value, Field.Store.NO, Field.Index.ANALYZED));

817

writer.addDocument(doc);

818

}

819

820

public static void assertIndexEquals(IndexReader index1, IndexReader index2) throws IOException {

821

assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs());

822

assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc());

823

assertEquals("Only one IndexReader has deletions.", index1.hasDeletions(), index2.hasDeletions());

824

if (!(index1 instanceof ParallelReader)) {

825

assertEquals("Single segment test differs.", index1.getSequentialSubReaders().length == 1, index2.getSequentialSubReaders().length == 1);

826

}

827

828

// check field names

829

Collection<String> fields1 = index1.getFieldNames(FieldOption.ALL);

830

Collection<String> fields2 = index1.getFieldNames(FieldOption.ALL);

831

assertEquals("IndexReaders have different numbers of fields.", fields1.size(), fields2.size());

832

Iterator<String> it1 = fields1.iterator();

833

Iterator<String> it2 = fields1.iterator();

834

while (it1.hasNext()) {

835

assertEquals("Different field names.", it1.next(), it2.next());

836

}

837

838

// check norms

839

it1 = fields1.iterator();

840

while (it1.hasNext()) {

841

String curField = it1.next();

842

byte[] norms1 = index1.norms(curField);

843

byte[] norms2 = index2.norms(curField);

844

if (norms1 != null && norms2 != null)

845

{

846

assertEquals(norms1.length, norms2.length);

847

for (int i = 0; i < norms1.length; i++) {

848

assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", norms1[i], norms2[i]);

849

}

850

}

851

else

852

{

853

assertSame(norms1, norms2);

854

}

855

}

856

857

// check deletions

858

for (int i = 0; i < index1.maxDoc(); i++) {

859

assertEquals("Doc " + i + " only deleted in one index.", index1.isDeleted(i), index2.isDeleted(i));

860

}

861

862

// check stored fields

863

for (int i = 0; i < index1.maxDoc(); i++) {

864

if (!index1.isDeleted(i)) {

865

Document doc1 = index1.document(i);

866

Document doc2 = index2.document(i);

867

List<Fieldable> fieldable1 = doc1.getFields();

868

List<Fieldable> fieldable2 = doc2.getFields();

869

assertEquals("Different numbers of fields for doc " + i + ".", fieldable1.size(), fieldable2.size());

870

Iterator<Fieldable> itField1 = fieldable1.iterator();

871

Iterator<Fieldable> itField2 = fieldable2.iterator();

872

while (itField1.hasNext()) {

873

Field curField1 = (Field) itField1.next();

874

Field curField2 = (Field) itField2.next();

875

assertEquals("Different fields names for doc " + i + ".", curField1.name(), curField2.name());

876

assertEquals("Different field values for doc " + i + ".", curField1.stringValue(), curField2.stringValue());

877

}

878

}

879

}

880

881

// check dictionary and posting lists

882

TermEnum enum1 = index1.terms();

883

TermEnum enum2 = index2.terms();

884

TermPositions tp1 = index1.termPositions();

885

TermPositions tp2 = index2.termPositions();

886

while(enum1.next()) {

887

assertTrue(enum2.next());

888

assertEquals("Different term in dictionary.", enum1.term(), enum2.term());

889

tp1.seek(enum1.term());

890

tp2.seek(enum1.term());

891

while(tp1.next()) {

892

assertTrue(tp2.next());

893

assertEquals("Different doc id in postinglist of term " + enum1.term() + ".", tp1.doc(), tp2.doc());

894

assertEquals("Different term frequence in postinglist of term " + enum1.term() + ".", tp1.freq(), tp2.freq());

895

for (int i = 0; i < tp1.freq(); i++) {

896

assertEquals("Different positions in postinglist of term " + enum1.term() + ".", tp1.nextPosition(), tp2.nextPosition());

897

}

898

}

899

}

900

}

901

902

public void testGetIndexCommit() throws IOException {

903

904

Directory d = newDirectory();

905

906

// set up writer

907

IndexWriter writer = new IndexWriter(

908

909

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

910

setMaxBufferedDocs(2).

911

setMergePolicy(newLogMergePolicy(10))

912

);

913

for(int i=0;i<27;i++)

914

addDocumentWithFields(writer);

915

writer.close();

916

917

SegmentInfos sis = new SegmentInfos();

918

sis.read(d);

919

IndexReader r = IndexReader.open(d, false);

920

IndexCommit c = r.getIndexCommit();

921

922

assertEquals(sis.getCurrentSegmentFileName(), c.getSegmentsFileName());

923

924

assertTrue(c.equals(r.getIndexCommit()));

925

926

// Change the index

927

writer = new IndexWriter(

928

929

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

930

setOpenMode(OpenMode.APPEND).

931

setMaxBufferedDocs(2).

932

setMergePolicy(newLogMergePolicy(10))

933

);

934

for(int i=0;i<7;i++)

935

addDocumentWithFields(writer);

936

writer.close();

937

938

IndexReader r2 = IndexReader.openIfChanged(r);

939

assertNotNull(r2);

940

assertFalse(c.equals(r2.getIndexCommit()));

941

assertFalse(r2.getIndexCommit().getSegmentCount() == 1);

942

r2.close();

943

944

writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT,

945

new MockAnalyzer(random))

946

.setOpenMode(OpenMode.APPEND));

947

writer.forceMerge(1);

948

writer.close();

949

950

r2 = IndexReader.openIfChanged(r);

951

assertNotNull(r2);

952

assertNull(IndexReader.openIfChanged(r2));

953

assertEquals(1, r2.getIndexCommit().getSegmentCount());

954

955

r.close();

956

r2.close();

957

d.close();

958

}

959

960

public void testReadOnly() throws Throwable {

961

Directory d = newDirectory();

962

IndexWriter writer = new IndexWriter(d, newIndexWriterConfig(

963

TEST_VERSION_CURRENT, new MockAnalyzer(random)));

964

addDocumentWithFields(writer);

965

writer.commit();

966

addDocumentWithFields(writer);

967

writer.close();

968

969

IndexReader r = IndexReader.open(d, true);

970

try {

971

r.deleteDocument(0);

972

fail();

973

} catch (UnsupportedOperationException uoe) {

974

// expected

975

}

976

977

writer = new IndexWriter(

978

979

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

980

setOpenMode(OpenMode.APPEND).

981

setMergePolicy(newLogMergePolicy(10))

982

);

983

addDocumentWithFields(writer);

984

writer.close();

985

986

// Make sure reopen is still readonly:

987

IndexReader r2 = IndexReader.openIfChanged(r);

988

assertNotNull(r2);

989

r.close();

990

991

assertFalse(r == r2);

992

993

try {

994

r2.deleteDocument(0);

995

fail();

996

} catch (UnsupportedOperationException uoe) {

997

// expected

998

}

999

1000

writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT,

1001

new MockAnalyzer(random))

1002

.setOpenMode(OpenMode.APPEND));

1003

writer.forceMerge(1);

1004

writer.close();

1005

1006

// Make sure reopen to a single segment is still readonly:

1007

IndexReader r3 = IndexReader.openIfChanged(r2);

1008

assertNotNull(r3);

1009

assertFalse(r3 == r2);

1010

r2.close();

1011

1012

assertFalse(r == r2);

1013

1014

try {

1015

r3.deleteDocument(0);

1016

fail();

1017

} catch (UnsupportedOperationException uoe) {

1018

// expected

1019

}

1020

1021

// Make sure write lock isn't held

1022

writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT,

1023

new MockAnalyzer(random))

1024

.setOpenMode(OpenMode.APPEND));

1025

writer.close();

1026

1027

r3.close();

1028

d.close();

1029

}

1030

1031

1032

// LUCENE-1474

1033

public void testIndexReader() throws Exception {

1034

Directory dir = newDirectory();

1035

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(

1036

TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1037

writer.addDocument(createDocument("a"));

1038

writer.addDocument(createDocument("b"));

1039

writer.addDocument(createDocument("c"));

1040

writer.close();

1041

IndexReader reader = IndexReader.open(dir, false);

1042

reader.deleteDocuments(new Term("id", "a"));

1043

reader.flush();

1044

reader.deleteDocuments(new Term("id", "b"));

1045

reader.close();

1046

IndexReader.open(dir,true).close();

1047

dir.close();

1048

}

1049

1050

static Document createDocument(String id) {

1051

Document doc = new Document();

1052

doc.add(newField("id", id, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));

1053

return doc;

1054

}

1055

1056

// LUCENE-1468 -- make sure on attempting to open an

1057

// IndexReader on a non-existent directory, you get a

1058

// good exception

1059

public void testNoDir() throws Throwable {

1060

Directory dir = newFSDirectory(_TestUtil.getTempDir("doesnotexist"));

1061

try {

1062

IndexReader.open(dir, true);

1063

fail("did not hit expected exception");

1064

} catch (NoSuchDirectoryException nsde) {

1065

// expected

1066

}

1067

dir.close();

1068

}

1069

1070

// LUCENE-1509

1071

public void testNoDupCommitFileNames() throws Throwable {

1072

1073

Directory dir = newDirectory();

1074

1075

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(

1076

TEST_VERSION_CURRENT, new MockAnalyzer(random))

1077

.setMaxBufferedDocs(2));

1078

writer.addDocument(createDocument("a"));

1079

writer.addDocument(createDocument("a"));

1080

writer.addDocument(createDocument("a"));

1081

writer.close();

1082

1083

Collection<IndexCommit> commits = IndexReader.listCommits(dir);

1084

for (final IndexCommit commit : commits) {

1085

Collection<String> files = commit.getFileNames();

1086

HashSet<String> seen = new HashSet<String>();

1087

for (final String fileName : files) {

1088

assertTrue("file " + fileName + " was duplicated", !seen.contains(fileName));

1089

seen.add(fileName);

1090

}

1091

}

1092

1093

dir.close();

1094

}

1095

1096

// LUCENE-1579: Ensure that on a cloned reader, segments

1097

// reuse the doc values arrays in FieldCache

1098

public void testFieldCacheReuseAfterClone() throws Exception {

1099

Directory dir = newDirectory();

1100

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1101

Document doc = new Document();

1102

doc.add(newField("number", "17", Field.Store.NO, Field.Index.NOT_ANALYZED));

1103

writer.addDocument(doc);

1104

writer.close();

1105

1106

// Open reader

1107

IndexReader r = SegmentReader.getOnlySegmentReader(dir);

1108

final int[] ints = FieldCache.DEFAULT.getInts(r, "number");

1109

assertEquals(1, ints.length);

1110

assertEquals(17, ints[0]);

1111

1112

// Clone reader

1113

IndexReader r2 = (IndexReader) r.clone();

1114

r.close();

1115

assertTrue(r2 != r);

1116

final int[] ints2 = FieldCache.DEFAULT.getInts(r2, "number");

1117

r2.close();

1118

1119

assertEquals(1, ints2.length);

1120

assertEquals(17, ints2[0]);

1121

assertTrue(ints == ints2);

1122

1123

dir.close();

1124

}

1125

1126

// LUCENE-1579: Ensure that on a reopened reader, that any

1127

// shared segments reuse the doc values arrays in

1128

// FieldCache

1129

public void testFieldCacheReuseAfterReopen() throws Exception {

1130

Directory dir = newDirectory();

1131

IndexWriter writer = new IndexWriter(

1132

dir,

1133

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

1134

setMergePolicy(newLogMergePolicy(10))

1135

);

1136

Document doc = new Document();

1137

doc.add(newField("number", "17", Field.Store.NO, Field.Index.NOT_ANALYZED));

1138

((LogMergePolicy) writer.getMergePolicy()).setMergeFactor(10);

1139

writer.addDocument(doc);

1140

writer.commit();

1141

1142

// Open reader1

1143

IndexReader r = IndexReader.open(dir, false);

1144

IndexReader r1 = SegmentReader.getOnlySegmentReader(r);

1145

final int[] ints = FieldCache.DEFAULT.getInts(r1, "number");

1146

assertEquals(1, ints.length);

1147

assertEquals(17, ints[0]);

1148

1149

// Add new segment

1150

writer.addDocument(doc);

1151

writer.commit();

1152

1153

// Reopen reader1 --> reader2

1154

IndexReader r2 = IndexReader.openIfChanged(r);

1155

assertNotNull(r2);

1156

r.close();

1157

IndexReader sub0 = r2.getSequentialSubReaders()[0];

1158

final int[] ints2 = FieldCache.DEFAULT.getInts(sub0, "number");

1159

r2.close();

1160

assertTrue(ints == ints2);

1161

1162

writer.close();

1163

dir.close();

1164

}

1165

1166

// LUCENE-1586: getUniqueTermCount

1167

public void testUniqueTermCount() throws Exception {

1168

Directory dir = newDirectory();

1169

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1170

Document doc = new Document();

1171

doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", Field.Store.NO, Field.Index.ANALYZED));

1172

doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", Field.Store.NO, Field.Index.ANALYZED));

1173

writer.addDocument(doc);

1174

writer.addDocument(doc);

1175

writer.commit();

1176

1177

IndexReader r = IndexReader.open(dir, false);

1178

IndexReader r1 = SegmentReader.getOnlySegmentReader(r);

1179

assertEquals(36, r1.getUniqueTermCount());

1180

writer.addDocument(doc);

1181

writer.commit();

1182

IndexReader r2 = IndexReader.openIfChanged(r);

1183

assertNotNull(r2);

1184

r.close();

1185

try {

1186

r2.getUniqueTermCount();

1187

fail("expected exception");

1188

} catch (UnsupportedOperationException uoe) {

1189

// expected

1190

}

1191

IndexReader[] subs = r2.getSequentialSubReaders();

1192

for(int i=0;i<subs.length;i++) {

1193

assertEquals(36, subs[i].getUniqueTermCount());

1194

}

1195

r2.close();

1196

writer.close();

1197

dir.close();

1198

}

1199

1200

// LUCENE-1609: don't load terms index

1201

public void testNoTermsIndex() throws Throwable {

1202

Directory dir = newDirectory();

1203

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1204

Document doc = new Document();

1205

doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", Field.Store.NO, Field.Index.ANALYZED));

1206

doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", Field.Store.NO, Field.Index.ANALYZED));

1207

writer.addDocument(doc);

1208

writer.addDocument(doc);

1209

writer.close();

1210

1211

IndexReader r = IndexReader.open(dir, null, true, -1);

1212

try {

1213

r.docFreq(new Term("field", "f"));

1214

fail("did not hit expected exception");

1215

} catch (IllegalStateException ise) {

1216

// expected

1217

}

1218

assertFalse(((SegmentReader) r.getSequentialSubReaders()[0]).termsIndexLoaded());

1219

1220

assertEquals(-1, ((SegmentReader) r.getSequentialSubReaders()[0]).getTermInfosIndexDivisor());

1221

writer = new IndexWriter(

1222

dir,

1223

newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).

1224

setMergePolicy(newLogMergePolicy(10))

1225

);

1226

writer.addDocument(doc);

1227

writer.close();

1228

1229

// LUCENE-1718: ensure re-open carries over no terms index:

1230

IndexReader r2 = IndexReader.openIfChanged(r);

1231

assertNotNull(r2);

1232

assertNull(IndexReader.openIfChanged(r2));

1233

r.close();

1234

IndexReader[] subReaders = r2.getSequentialSubReaders();

1235

assertEquals(2, subReaders.length);

1236

for(int i=0;i<2;i++) {

1237

assertFalse(((SegmentReader) subReaders[i]).termsIndexLoaded());

1238

}

1239

r2.close();

1240

dir.close();

1241

}

1242

1243

// LUCENE-2046

1244

public void testPrepareCommitIsCurrent() throws Throwable {

1245

Directory dir = newDirectory();

1246

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(

1247

TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1248

writer.commit();

1249

Document doc = new Document();

1250

writer.addDocument(doc);

1251

IndexReader r = IndexReader.open(dir, true);

1252

assertTrue(r.isCurrent());

1253

writer.addDocument(doc);

1254

writer.prepareCommit();

1255

assertTrue(r.isCurrent());

1256

IndexReader r2 = IndexReader.openIfChanged(r);

1257

assertNull(r2);

1258

writer.commit();

1259

assertFalse(r.isCurrent());

1260

writer.close();

1261

r.close();

1262

dir.close();

1263

}

1264

1265

// LUCENE-2753

1266

public void testListCommits() throws Exception {

1267

Directory dir = newDirectory();

1268

SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());

1269

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(

1270

TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setIndexDeletionPolicy(sdp));

1271

writer.addDocument(new Document());

1272

writer.commit();

1273

sdp.snapshot("c1");

1274

writer.addDocument(new Document());

1275

writer.commit();

1276

sdp.snapshot("c2");

1277

writer.addDocument(new Document());

1278

writer.commit();

1279

sdp.snapshot("c3");

1280

writer.close();

1281

long currentGen = 0;

1282

for (IndexCommit ic : IndexReader.listCommits(dir)) {

1283

assertTrue("currentGen=" + currentGen + " commitGen=" + ic.getGeneration(), currentGen < ic.getGeneration());

1284

currentGen = ic.getGeneration();

1285

}

1286

dir.close();

1287

}

1288

1289

// LUCENE-2812

1290

public void testIndexExists() throws Exception {

1291

Directory dir = newDirectory();

1292

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1293

writer.addDocument(new Document());

1294

writer.prepareCommit();

1295

assertFalse(IndexReader.indexExists(dir));

1296

writer.close();

1297

assertTrue(IndexReader.indexExists(dir));

1298

dir.close();

1299

}

1300

1301

// LUCENE-2474

1302

public void testReaderFinishedListener() throws Exception {

1303

Directory dir = newDirectory();

1304

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));

1305

((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(3);

1306

writer.setInfoStream(VERBOSE ? System.out : null);

1307

writer.addDocument(new Document());

1308

writer.commit();

1309

writer.addDocument(new Document());

1310

writer.commit();

1311

final IndexReader reader = writer.getReader();

1312

final int[] closeCount = new int[1];

1313

final IndexReader.ReaderFinishedListener listener = new IndexReader.ReaderFinishedListener() {

1314

public void finished(IndexReader reader) {

1315

closeCount[0]++;

1316

}

1317

};

1318

1319

reader.addReaderFinishedListener(listener);

1320

1321

reader.close();

1322

1323

// Just the top reader

1324

assertEquals(1, closeCount[0]);

1325

writer.close();

1326

1327

// Now also the subs

1328

assertEquals(3, closeCount[0]);

1329

1330

IndexReader reader2 = IndexReader.open(dir);

1331

reader2.addReaderFinishedListener(listener);

1332

1333

closeCount[0] = 0;

1334

reader2.close();

1335

assertEquals(3, closeCount[0]);

1336

dir.close();

1337

}

1338

1339

public void testOOBDocID() throws Exception {

1340

Directory dir = newDirectory();

1341

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1342

writer.addDocument(new Document());

1343

IndexReader r = writer.getReader();

1344

writer.close();

1345

r.document(0);

1346

try {

1347

r.document(1);

1348

fail("did not hit exception");

1349

} catch (IllegalArgumentException iae) {

1350

// expected

1351

}

1352

r.close();

1353

dir.close();

1354

}

1355

1356

public void testTryIncRef() throws CorruptIndexException, LockObtainFailedException, IOException {

1357

Directory dir = newDirectory();

1358

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1359

writer.addDocument(new Document());

1360

writer.commit();

1361

IndexReader r = IndexReader.open(dir);

1362

assertTrue(r.tryIncRef());

1363

r.decRef();

1364

r.close();

1365

assertFalse(r.tryIncRef());

1366

writer.close();

1367

dir.close();

1368

}

1369

1370

public void testStressTryIncRef() throws CorruptIndexException, LockObtainFailedException, IOException, InterruptedException {

1371

Directory dir = newDirectory();

1372

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));

1373

writer.addDocument(new Document());

1374

writer.commit();

1375

IndexReader r = IndexReader.open(dir);

1376

int numThreads = atLeast(2);

1377

1378

IncThread[] threads = new IncThread[numThreads];

1379

for (int i = 0; i < threads.length; i++) {

1380

threads[i] = new IncThread(r, random);

1381

threads[i].start();

1382

}

1383

Thread.sleep(100);

1384

1385

assertTrue(r.tryIncRef());

1386

r.decRef();

1387

r.close();

1388

1389

for (int i = 0; i < threads.length; i++) {

1390

threads[i].join();

1391

assertNull(threads[i].failed);

1392

}

1393

assertFalse(r.tryIncRef());

1394

writer.close();

1395

dir.close();

1396

}

1397

1398

static class IncThread extends Thread {

1399

final IndexReader toInc;

1400

final Random random;

1401

Throwable failed;

1402

1403

IncThread(IndexReader toInc, Random random) {

1404

this.toInc = toInc;

1405

this.random = random;

1406

}

1407

1408

@Override

1409

public void run() {

1410

try {

1411

while (toInc.tryIncRef()) {

1412

assertFalse(toInc.hasDeletions());

1413

toInc.decRef();

1414

}

1415

assertFalse(toInc.tryIncRef());

1416

} catch (Throwable e) {

1417

failed = e;

1418

}

1419

}

1420

}

1421

}

Older »