~slub.team/goobi-indexserver/3.x

IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setMergePolicy(newLogMergePolicy());

308

((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);

309

IndexWriter writer = new IndexWriter(tmpDir, conf);

310

writer.addDocument(testDoc);

311

writer.close();

312

313

assertTrue(fieldInfos != null);

314

FieldsReader reader;

315

long lazyTime = 0;

316

long regularTime = 0;

317

int length = 10;

318

Set<String> lazyFieldNames = new HashSet<String>();

319

lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);

320

SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(Collections. <String> emptySet(), lazyFieldNames);

321

322

for (int i = 0; i < length; i++) {

323

reader = new FieldsReader(tmpDir, TEST_SEGMENT_NAME, fieldInfos);

324

assertTrue(reader.size() == 1);

325

326

Document doc;

327

doc = reader.doc(0, null);//Load all of them

328

assertTrue("doc is null and it shouldn't be", doc != null);

329

Fieldable field = doc.getFieldable(DocHelper.LARGE_LAZY_FIELD_KEY);

330

assertTrue("field is null and it shouldn't be", field != null);

331

assertTrue("field is lazy", field.isLazy() == false);

332

String value;

333

long start;

334

long finish;

335

start = System.currentTimeMillis();

336

//On my machine this was always 0ms.

337

value = field.stringValue();

338

finish = System.currentTimeMillis();

339

assertTrue("value is null and it shouldn't be", value != null);

340

regularTime += (finish - start);

341

reader.close();

342

reader = null;

343

doc = null;

344

//Hmmm, are we still in cache???

345

System.gc();

346

reader = new FieldsReader(tmpDir, TEST_SEGMENT_NAME, fieldInfos);

347

doc = reader.doc(0, fieldSelector);

348

field = doc.getFieldable(DocHelper.LARGE_LAZY_FIELD_KEY);

349

assertTrue("field is not lazy", field.isLazy() == true);

350

start = System.currentTimeMillis();

351

//On my machine this took around 50 - 70ms

352

value = field.stringValue();

353

finish = System.currentTimeMillis();

354

assertTrue("value is null and it shouldn't be", value != null);

355

lazyTime += (finish - start);

356

reader.close();

357

}

358

tmpDir.close();

359

360

if (VERBOSE) {

361

System.out.println("Average Non-lazy time (should be very close to zero): " + regularTime / length + " ms for " + length + " reads");

362

System.out.println("Average Lazy Time (should be greater than zero): " + lazyTime / length + " ms for " + length + " reads");

363

}

364

}

365

366

public void testLoadSize() throws IOException {

367

FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos);

368

Document doc;

369

370

doc = reader.doc(0, new FieldSelector(){

371

public FieldSelectorResult accept(String fieldName) {

372

if (fieldName.equals(DocHelper.TEXT_FIELD_1_KEY) ||

373

fieldName.equals(DocHelper.LAZY_FIELD_BINARY_KEY))

374

return FieldSelectorResult.SIZE;

375

else if (fieldName.equals(DocHelper.TEXT_FIELD_3_KEY))

376

return FieldSelectorResult.LOAD;

377

else

378

return FieldSelectorResult.NO_LOAD;

379

}

380

});

381

Fieldable f1 = doc.getFieldable(DocHelper.TEXT_FIELD_1_KEY);

382

Fieldable f3 = doc.getFieldable(DocHelper.TEXT_FIELD_3_KEY);

383

Fieldable fb = doc.getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);

384

assertTrue(f1.isBinary());

385

assertTrue(!f3.isBinary());

386

assertTrue(fb.isBinary());

387

assertSizeEquals(2*DocHelper.FIELD_1_TEXT.length(), f1.getBinaryValue());

388

assertEquals(DocHelper.FIELD_3_TEXT, f3.stringValue());

389

assertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.length, fb.getBinaryValue());

390

391

reader.close();

392

}

393

394

private void assertSizeEquals(int size, byte[] sizebytes) {

395

assertEquals((byte) (size>>>24), sizebytes[0]);

396

assertEquals((byte) (size>>>16), sizebytes[1]);

397

assertEquals((byte) (size>>> 8), sizebytes[2]);

398

assertEquals((byte) size , sizebytes[3]);

399

}

400

401

public static class FaultyFSDirectory extends Directory {

402

403

Directory fsDir;

404

public FaultyFSDirectory(File dir) throws IOException {

405

fsDir = newFSDirectory(dir);

406

lockFactory = fsDir.getLockFactory();

407

}

408

@Override

409

public IndexInput openInput(String name) throws IOException {

410

return new FaultyIndexInput(fsDir.openInput(name));

411

}

412

@Override

413

public String[] listAll() throws IOException {

414

return fsDir.listAll();

415

}

416

@Override

417

public boolean fileExists(String name) throws IOException {

418

return fsDir.fileExists(name);

419

}

420

@Override

421

public long fileModified(String name) throws IOException {

422

return fsDir.fileModified(name);

423

}

424

@Override

425

@Deprecated

426

/* @deprecated Lucene never uses this API; it will be

427

* removed in 4.0. */

428

public void touchFile(String name) throws IOException {

429

fsDir.touchFile(name);

430

}

431

@Override

432

public void deleteFile(String name) throws IOException {

433

fsDir.deleteFile(name);

434

}

435

@Override

436

public long fileLength(String name) throws IOException {

437

return fsDir.fileLength(name);

438

}

439

@Override

440

public IndexOutput createOutput(String name) throws IOException {

441

return fsDir.createOutput(name);

442

}

443

@Override

444

public void close() throws IOException {

445

fsDir.close();

446

}

447

}

448

449

private static class FaultyIndexInput extends BufferedIndexInput {

450

IndexInput delegate;

451

static boolean doFail;

452

int count;

453

private FaultyIndexInput(IndexInput delegate) {

454

this.delegate = delegate;

455

}

456

private void simOutage() throws IOException {

457

if (doFail && count++ % 2 == 1) {

458

throw new IOException("Simulated network outage");

459

}

460

}

461

@Override

462

public void readInternal(byte[] b, int offset, int length) throws IOException {

463

simOutage();

464

delegate.readBytes(b, offset, length);

465

}

466

@Override

467

public void seekInternal(long pos) throws IOException {

468

//simOutage();

469

delegate.seek(pos);

470

}

471

@Override

472

public long length() {

473

return delegate.length();

474

}

475

@Override

476

public void close() throws IOException {

477

delegate.close();

478

}

479

@Override

480

public Object clone() {

481

return new FaultyIndexInput((IndexInput) delegate.clone());

482

}

483

}

484

485

// LUCENE-1262

486

public void testExceptions() throws Throwable {

487

File indexDir = _TestUtil.getTempDir("testfieldswriterexceptions");

488

489

try {

490

Directory dir = new FaultyFSDirectory(indexDir);

491

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(

492

TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));

493

for(int i=0;i<2;i++)

494

writer.addDocument(testDoc);

495

writer.optimize();

496

writer.close();

497

498

IndexReader reader = IndexReader.open(dir, true);

499

500

FaultyIndexInput.doFail = true;

501

502

boolean exc = false;

503

504

for(int i=0;i<2;i++) {

505

try {

506

reader.document(i);

507

} catch (IOException ioe) {

508

// expected

509

exc = true;

510

}

511

try {

512

reader.document(i);

513

} catch (IOException ioe) {

514

// expected

515

exc = true;

516

}

517

}

518

assertTrue(exc);

519

reader.close();

520

dir.close();

521

} finally {

522

_TestUtil.rmDir(indexDir);

523

}

524

525

}

526

527

public void testNumericField() throws Exception {

528

Directory dir = newDirectory();

529

RandomIndexWriter w = new RandomIndexWriter(random, dir);

530

final int numDocs = atLeast(500);

531

final Number[] answers = new Number[numDocs];

532

final NumericField.DataType[] typeAnswers = new NumericField.DataType[numDocs];

533

for(int id=0;id<numDocs;id++) {

534

Document doc = new Document();

535

NumericField nf = new NumericField("nf", Field.Store.YES, false);

536

doc.add(nf);

537

final Number answer;

538

final NumericField.DataType typeAnswer;

539

if (random.nextBoolean()) {

540

// float/double

541

if (random.nextBoolean()) {

542

final float f = random.nextFloat();

543

nf.setFloatValue(f);

544

answer = Float.valueOf(f);

545

typeAnswer = NumericField.DataType.FLOAT;

546

} else {

547

final double d = random.nextDouble();

548

nf.setDoubleValue(d);

549

answer = Double.valueOf(d);

550

typeAnswer = NumericField.DataType.DOUBLE;

551

}

552

} else {

553

// int/long

554

if (random.nextBoolean()) {

555

final int i = random.nextInt();

556

nf.setIntValue(i);

557

answer = Integer.valueOf(i);

558

typeAnswer = NumericField.DataType.INT;

559

} else {

560

final long l = random.nextLong();

561

nf.setLongValue(l);

562

answer = Long.valueOf(l);

563

typeAnswer = NumericField.DataType.LONG;

564

}

565

}

566

answers[id] = answer;

567

typeAnswers[id] = typeAnswer;

568

doc.add(new NumericField("id", Integer.MAX_VALUE, Field.Store.NO, true).setIntValue(id));

569

w.addDocument(doc);

570

}

571

final IndexReader r = w.getReader();

572

w.close();

573

574

assertEquals(numDocs, r.numDocs());

575

576

for(IndexReader sub : r.getSequentialSubReaders()) {

577

final int[] ids = FieldCache.DEFAULT.getInts(sub, "id");

578

for(int docID=0;docID<sub.numDocs();docID++) {

579

final Document doc = sub.document(docID);

580

final Fieldable f = doc.getFieldable("nf");

581

assertTrue("got f=" + f, f instanceof NumericField);

582

final NumericField nf = (NumericField) f;

583

assertEquals(answers[ids[docID]], nf.getNumericValue());

584

assertSame(typeAnswers[ids[docID]], nf.getDataType());

585

}

586

}

587

r.close();

588

dir.close();

589

}

590

591

}

Older »