~slub.team/goobi-indexserver/3.x

message("findForcedMergesSizeLimit: skip segment=" + info + ": size is > maxMergeSize (" + maxMergeSizeForForcedMerge + ") or sizeDocs is > maxMergeDocs (" + maxMergeDocs + ")");

256

}

257

// need to skip that segment + add a merge for the 'right' segments,

258

// unless there is only 1 which is merged.

259

if (last - start - 1 > 1 || (start != last - 1 && !isMerged(infos.info(start + 1)))) {

260

// there is more than 1 segment to the right of

261

// this one, or a mergeable single segment.

262

spec.add(new OneMerge(segments.subList(start + 1, last)));

263

}

264

last = start;

265

} else if (last - start == mergeFactor) {

266

// mergeFactor eligible segments were found, add them as a merge.

267

spec.add(new OneMerge(segments.subList(start, last)));

268

last = start;

269

}

270

--start;

271

}

272

273

// Add any left-over segments, unless there is just 1

274

// already fully merged

275

if (last > 0 && (++start + 1 < last || !isMerged(infos.info(start)))) {

276

spec.add(new OneMerge(segments.subList(start, last)));

277

}

278

279

return spec.merges.size() == 0 ? null : spec;

280

}

281

282

/**

283

* Returns the merges necessary to forceMerge the index. This method constraints

284

* the returned merges only by the {@code maxNumSegments} parameter, and

285

* guaranteed that exactly that number of segments will remain in the index.

286

287

private MergeSpecification findForcedMergesMaxNumSegments(SegmentInfos infos, int maxNumSegments, int last) throws IOException {

288

MergeSpecification spec = new MergeSpecification();

289

final List<SegmentInfo> segments = infos.asList();

290

291

// First, enroll all "full" merges (size

292

// mergeFactor) to potentially be run concurrently:

293

while (last - maxNumSegments + 1 >= mergeFactor) {

294

spec.add(new OneMerge(segments.subList(last - mergeFactor, last)));

295

last -= mergeFactor;

296

}

297

298

// Only if there are no full merges pending do we

299

// add a final partial (< mergeFactor segments) merge:

300

if (0 == spec.merges.size()) {

301

if (maxNumSegments == 1) {

302

303

// Since we must merge down to 1 segment, the

304

// choice is simple:

305

if (last > 1 || !isMerged(infos.info(0))) {

306

spec.add(new OneMerge(segments.subList(0, last)));

307

}

308

} else if (last > maxNumSegments) {

309

310

// Take care to pick a partial merge that is

311

// least cost, but does not make the index too

312

// lopsided. If we always just picked the

313

// partial tail then we could produce a highly

314

// lopsided index over time:

315

316

// We must merge this many segments to leave

317

// maxNumSegments in the index (from when

318

// forceMerge was first kicked off):

319

final int finalMergeSize = last - maxNumSegments + 1;

320

321

// Consider all possible starting points:

322

long bestSize = 0;

323

int bestStart = 0;

324

325

for(int i=0;i<last-finalMergeSize+1;i++) {

326

long sumSize = 0;

327

for(int j=0;j<finalMergeSize;j++)

328

sumSize += size(infos.info(j+i));

329

if (i == 0 || (sumSize < 2*size(infos.info(i-1)) && sumSize < bestSize)) {

330

bestStart = i;

331

bestSize = sumSize;

332

}

333

}

334

335

spec.add(new OneMerge(segments.subList(bestStart, bestStart + finalMergeSize)));

336

}

337

}

338

return spec.merges.size() == 0 ? null : spec;

339

}

340

341

/** Returns the merges necessary to merge the index down

342

* to a specified number of segments.

343

* This respects the {@link #maxMergeSizeForForcedMerge} setting.

344

* By default, and assuming {@code maxNumSegments=1}, only

345

* one segment will be left in the index, where that segment

346

* has no deletions pending nor separate norms, and it is in

347

* compound file format if the current useCompoundFile

348

* setting is true. This method returns multiple merges

349

* (mergeFactor at a time) so the {@link MergeScheduler}

350

* in use may make use of concurrency. */

351

@Override

352

public MergeSpecification findForcedMerges(SegmentInfos infos,

353

int maxNumSegments, Map<SegmentInfo,Boolean> segmentsToMerge) throws IOException {

354

355

assert maxNumSegments > 0;

356

if (verbose()) {

357

message("findForcedMerges: maxNumSegs=" + maxNumSegments + " segsToMerge="+ segmentsToMerge);

358

}

359

360

// If the segments are already merged (e.g. there's only 1 segment), or

361

// there are <maxNumSegements:.

362

if (isMerged(infos, maxNumSegments, segmentsToMerge)) {

363

if (verbose()) {

364

message("already merged; skip");

365

}

366

return null;

367

}

368

369

// Find the newest (rightmost) segment that needs to

370

// be merged (other segments may have been flushed

371

// since merging started):

372

int last = infos.size();

373

while (last > 0) {

374

final SegmentInfo info = infos.info(--last);

375

if (segmentsToMerge.get(info) != null) {

376

last++;

377

break;

378

}

379

}

380

381

if (last == 0) return null;

382

383

// There is only one segment already, and it is merged

384

if (maxNumSegments == 1 && last == 1 && isMerged(infos.info(0))) {

385

if (verbose()) {

386

message("already 1 seg; skip");

387

}

388

return null;

389

}

390

391

// Check if there are any segments above the threshold

392

boolean anyTooLarge = false;

393

for (int i = 0; i < last; i++) {

394

SegmentInfo info = infos.info(i);

395

if (size(info) > maxMergeSizeForForcedMerge || sizeDocs(info) > maxMergeDocs) {

396

anyTooLarge = true;

397

break;

398

}

399

}

400

401

if (anyTooLarge) {

402

return findForcedMergesSizeLimit(infos, maxNumSegments, last);

403

} else {

404

return findForcedMergesMaxNumSegments(infos, maxNumSegments, last);

405

}

406

}

407

408

/**

409

* Finds merges necessary to force-merge all deletes from the

410

* index. We simply merge adjacent segments that have

411

* deletes, up to mergeFactor at a time.

412

413

@Override

414

public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos)

415

throws CorruptIndexException, IOException {

416

final List<SegmentInfo> segments = segmentInfos.asList();

417

final int numSegments = segments.size();

418

419

if (verbose())

420

message("findForcedDeleteMerges: " + numSegments + " segments");

421

422

MergeSpecification spec = new MergeSpecification();

423

int firstSegmentWithDeletions = -1;

424

IndexWriter w = writer.get();

425

assert w != null;

426

for(int i=0;i<numSegments;i++) {

427

final SegmentInfo info = segmentInfos.info(i);

428

int delCount = w.numDeletedDocs(info);

429

if (delCount > 0) {

430

if (verbose())

431

message(" segment " + info.name + " has deletions");

432

if (firstSegmentWithDeletions == -1)

433

firstSegmentWithDeletions = i;

434

else if (i - firstSegmentWithDeletions == mergeFactor) {

435

// We've seen mergeFactor segments in a row with

436

// deletions, so force a merge now:

437

if (verbose())

438

message(" add merge " + firstSegmentWithDeletions + " to " + (i-1) + " inclusive");

439

spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, i)));

440

firstSegmentWithDeletions = i;

441

}

442

} else if (firstSegmentWithDeletions != -1) {

443

// End of a sequence of segments with deletions, so,

444

// merge those past segments even if it's fewer than

445

// mergeFactor segments

446

if (verbose())

447

message(" add merge " + firstSegmentWithDeletions + " to " + (i-1) + " inclusive");

448

spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, i)));

449

firstSegmentWithDeletions = -1;

450

}

451

}

452

453

if (firstSegmentWithDeletions != -1) {

454

if (verbose())

455

message(" add merge " + firstSegmentWithDeletions + " to " + (numSegments-1) + " inclusive");

456

spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, numSegments)));

457

}

458

459

return spec;

460

}

461

462

private static class SegmentInfoAndLevel implements Comparable<SegmentInfoAndLevel> {

463

SegmentInfo info;

464

float level;

465

int index;

466

467

public SegmentInfoAndLevel(SegmentInfo info, float level, int index) {

468

this.info = info;

469

this.level = level;

470

this.index = index;

471

}

472

473

// Sorts largest to smallest

474

public int compareTo(SegmentInfoAndLevel other) {

475

if (level < other.level)

476

return 1;

477

else if (level > other.level)

478

return -1;

479

else

480

return 0;

481

}

482

}

483

484

/** Checks if any merges are now necessary and returns a

485

* {@link MergePolicy.MergeSpecification} if so. A merge

486

* is necessary when there are more than {@link

487

* #setMergeFactor} segments at a given level. When

488

* multiple levels have too many segments, this method

489

* will return multiple merges, allowing the {@link

490

* MergeScheduler} to use concurrency. */

491

@Override

492

public MergeSpecification findMerges(SegmentInfos infos) throws IOException {

493

494

final int numSegments = infos.size();

495

if (verbose())

496

message("findMerges: " + numSegments + " segments");

497

498

// Compute levels, which is just log (base mergeFactor)

499

// of the size of each segment

500

final List<SegmentInfoAndLevel> levels = new ArrayList<SegmentInfoAndLevel>();

501

final float norm = (float) Math.log(mergeFactor);

502

503

final Collection<SegmentInfo> mergingSegments = writer.get().getMergingSegments();

504

505

for(int i=0;i<numSegments;i++) {

506

final SegmentInfo info = infos.info(i);

507

long size = size(info);

508

509

// Floor tiny segments

510

if (size < 1) {

511

size = 1;

512

}

513

514

final SegmentInfoAndLevel infoLevel = new SegmentInfoAndLevel(info, (float) Math.log(size)/norm, i);

515

levels.add(infoLevel);

516

517

if (verbose()) {

518

final long segBytes = sizeBytes(info);

519

String extra = mergingSegments.contains(info) ? " [merging]" : "";

520

if (size >= maxMergeSize) {

521

extra += " [skip: too large]";

522

}

523

message("seg=" + writer.get().segString(info) + " level=" + infoLevel.level + " size=" + String.format("%.3f MB", segBytes/1024/1024.) + extra);

524

}

525

}

526

527

final float levelFloor;

528

if (minMergeSize <= 0)

529

levelFloor = (float) 0.0;

530

else

531

levelFloor = (float) (Math.log(minMergeSize)/norm);

532

533

// Now, we quantize the log values into levels. The

534

// first level is any segment whose log size is within

535

// LEVEL_LOG_SPAN of the max size, or, who has such as

536

// segment "to the right". Then, we find the max of all

537

// other segments and use that to define the next level

538

// segment, etc.

539

540

MergeSpecification spec = null;

541

542

final int numMergeableSegments = levels.size();

543

544

int start = 0;

545

while(start < numMergeableSegments) {

546

547

// Find max level of all segments not already

548

// quantized.

549

float maxLevel = levels.get(start).level;

550

for(int i=1+start;i<numMergeableSegments;i++) {

551

final float level = levels.get(i).level;

552

if (level > maxLevel)

553

maxLevel = level;

554

}

555

556

// Now search backwards for the rightmost segment that

557

// falls into this level:

558

float levelBottom;

559

if (maxLevel <= levelFloor)

560

// All remaining segments fall into the min level

561

levelBottom = -1.0F;

562

else {

563

levelBottom = (float) (maxLevel - LEVEL_LOG_SPAN);

564

565

// Force a boundary at the level floor

566

if (levelBottom < levelFloor && maxLevel >= levelFloor)

567

levelBottom = levelFloor;

568

}

569

570

int upto = numMergeableSegments-1;

571

while(upto >= start) {

572

if (levels.get(upto).level >= levelBottom) {

573

break;

574

}

575

upto--;

576

}

577

if (verbose())

578

message(" level " + levelBottom + " to " + maxLevel + ": " + (1+upto-start) + " segments");

579

580

// Finally, record all merges that are viable at this level:

581

int end = start + mergeFactor;

582

while(end <= 1+upto) {

583

boolean anyTooLarge = false;

584

boolean anyMerging = false;

585

for(int i=start;i<end;i++) {

586

final SegmentInfo info = levels.get(i).info;

587

anyTooLarge |= (size(info) >= maxMergeSize || sizeDocs(info) >= maxMergeDocs);

588

if (mergingSegments.contains(info)) {

589

anyMerging = true;

590

break;

591

}

592

}

593

594

if (anyMerging) {

595

// skip

596

} else if (!anyTooLarge) {

597

if (spec == null)

598

spec = new MergeSpecification();

599

final List<SegmentInfo> mergeInfos = new ArrayList<SegmentInfo>();

600

for(int i=start;i<end;i++) {

601

mergeInfos.add(levels.get(i).info);

602

assert infos.contains(levels.get(i).info);

603

}

604

if (verbose()) {

605

message(" add merge=" + writer.get().segString(mergeInfos) + " start=" + start + " end=" + end);

606

}

607

spec.add(new OneMerge(mergeInfos));

608

} else if (verbose()) {

609

message(" " + start + " to " + end + ": contains segment over maxMergeSize or maxMergeDocs; skipping");

610

}

611

612

start = end;

613

end = start + mergeFactor;

614

}

615

616

start = 1+upto;

617

}

618

619

return spec;

620

}

621

622

/** Determines the largest segment (measured by

623

* document count) that may be merged with other segments.

624

* Small values (e.g., less than 10,000) are best for

625

* interactive indexing, as this limits the length of

626

* pauses while indexing to a few seconds. Larger values

627

* are best for batched indexing and speedier

628

* searches.

629

630

* The default value is {@link Integer#MAX_VALUE}.

631

632

* The default merge policy ({@link

633

* LogByteSizeMergePolicy}) also allows you to set this

634

* limit by net size (in MB) of the segment, using {@link

635

* LogByteSizeMergePolicy#setMaxMergeMB}.

636

637

public void setMaxMergeDocs(int maxMergeDocs) {

638

this.maxMergeDocs = maxMergeDocs;

639

}

640

641

/** Returns the largest segment (measured by document

642

* count) that may be merged with other segments.

643

* @see #setMaxMergeDocs */

644

public int getMaxMergeDocs() {

645

return maxMergeDocs;

646

}

647

648

@Override

649

public String toString() {

650

StringBuilder sb = new StringBuilder("[" + getClass().getSimpleName() + ": ");

651

sb.append("minMergeSize=").append(minMergeSize).append(", ");

652

sb.append("mergeFactor=").append(mergeFactor).append(", ");

653

sb.append("maxMergeSize=").append(maxMergeSize).append(", ");

654

sb.append("maxMergeSizeForForcedMerge=").append(maxMergeSizeForForcedMerge).append(", ");

655

sb.append("calibrateSizeByDeletes=").append(calibrateSizeByDeletes).append(", ");

656

sb.append("maxMergeDocs=").append(maxMergeDocs).append(", ");

657

sb.append("useCompoundFile=").append(useCompoundFile).append(", ");

658

sb.append("noCFSRatio=").append(noCFSRatio);

659

sb.append("]");

660

return sb.toString();

661

}

662

663

}

Older »