~slub.team/goobi-indexserver/3.x

IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MyAnalyzer()).setMaxBufferedDocs(-1).setMergePolicy(newLogMergePolicy(false, 10)));

Document doc = new Document();

for(int i=0;i<testFields.length;i++) {

final Field.TermVector tv;

if (testFieldsStorePos[i] && testFieldsStoreOff[i])

tv = Field.TermVector.WITH_POSITIONS_OFFSETS;

else if (testFieldsStorePos[i] && !testFieldsStoreOff[i])

tv = Field.TermVector.WITH_POSITIONS;

100

else if (!testFieldsStorePos[i] && testFieldsStoreOff[i])

101

tv = Field.TermVector.WITH_OFFSETS;

102

else

103

tv = Field.TermVector.YES;

104

doc.add(new Field(testFields[i], "", Field.Store.NO, Field.Index.ANALYZED, tv));

105

}

106

107

//Create 5 documents for testing, they all have the same

108

//terms

109

for(int j=0;j<5;j++)

110

writer.addDocument(doc);

111

writer.commit();

112

seg = writer.newestSegment().name;

113

writer.close();

114

115

fieldInfos = new FieldInfos(dir, IndexFileNames.segmentFileName(seg, IndexFileNames.FIELD_INFOS_EXTENSION));

116

}

117

118

@Override

119

public void tearDown() throws Exception {

120

dir.close();

121

super.tearDown();

122

}

123

124

private class MyTokenStream extends TokenStream {

125

private int tokenUpto;

126

127

private final CharTermAttribute termAtt;

128

private final PositionIncrementAttribute posIncrAtt;

129

private final OffsetAttribute offsetAtt;

130

131

public MyTokenStream() {

132

termAtt = addAttribute(CharTermAttribute.class);

133

posIncrAtt = addAttribute(PositionIncrementAttribute.class);

134

offsetAtt = addAttribute(OffsetAttribute.class);

135

}

136

137

@Override

138

public boolean incrementToken() {

139

if (tokenUpto >= tokens.length)

140

return false;

141

else {

142

final TestToken testToken = tokens[tokenUpto++];

143

clearAttributes();

144

termAtt.append(testToken.text);

145

offsetAtt.setOffset(testToken.startOffset, testToken.endOffset);

146

if (tokenUpto > 1) {

147

posIncrAtt.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos);

148

} else {

149

posIncrAtt.setPositionIncrement(testToken.pos+1);

150

}

151

return true;

152

}

153

}

154

155

@Override

156

public void reset() throws IOException {

157

super.reset();

158

this.tokenUpto = 0;

159

}

160

}

161

162

private class MyAnalyzer extends Analyzer {

163

@Override

164

public TokenStream tokenStream(String fieldName, Reader reader) {

165

return new MyTokenStream();

166

}

167

}

168

169

public void test() throws IOException {

170

//Check to see the files were created properly in setup

171

assertTrue(dir.fileExists(IndexFileNames.segmentFileName(seg, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)));

172

assertTrue(dir.fileExists(IndexFileNames.segmentFileName(seg, IndexFileNames.VECTORS_INDEX_EXTENSION)));

173

}

174

175

public void testReader() throws IOException {

176

TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);

177

for (int j = 0; j < 5; j++) {

178

TermFreqVector vector = reader.get(j, testFields[0]);

179

assertTrue(vector != null);

180

String[] terms = vector.getTerms();

181

assertTrue(terms != null);

182

assertTrue(terms.length == testTerms.length);

183

for (int i = 0; i < terms.length; i++) {

184

String term = terms[i];

185

//System.out.println("Term: " + term);

186

assertTrue(term.equals(testTerms[i]));

187

}

188

}

189

reader.close();

190

}

191

192

public void testPositionReader() throws IOException {

193

TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);

194

TermPositionVector vector;

195

String[] terms;

196

vector = (TermPositionVector) reader.get(0, testFields[0]);

197

assertTrue(vector != null);

198

terms = vector.getTerms();

199

assertTrue(terms != null);

200

assertTrue(terms.length == testTerms.length);

201

for (int i = 0; i < terms.length; i++) {

202

String term = terms[i];

203

//System.out.println("Term: " + term);

204

assertTrue(term.equals(testTerms[i]));

205

int[] positions = vector.getTermPositions(i);

206

assertTrue(positions != null);

207

assertTrue(positions.length == this.positions[i].length);

208

for (int j = 0; j < positions.length; j++) {

209

int position = positions[j];

210

assertTrue(position == this.positions[i][j]);

211

}

212

TermVectorOffsetInfo[] offset = vector.getOffsets(i);

213

assertTrue(offset != null);

214

assertTrue(offset.length == this.offsets[i].length);

215

for (int j = 0; j < offset.length; j++) {

216

TermVectorOffsetInfo termVectorOffsetInfo = offset[j];

217

assertTrue(termVectorOffsetInfo.equals(offsets[i][j]));

218

}

219

}

220

221

TermFreqVector freqVector = reader.get(0, testFields[1]); //no pos, no offset

222

assertTrue(freqVector != null);

223

assertTrue(freqVector instanceof TermPositionVector == false);

224

terms = freqVector.getTerms();

225

assertTrue(terms != null);

226

assertTrue(terms.length == testTerms.length);

227

for (int i = 0; i < terms.length; i++) {

228

String term = terms[i];

229

//System.out.println("Term: " + term);

230

assertTrue(term.equals(testTerms[i]));

231

}

232

reader.close();

233

}

234

235

public void testOffsetReader() throws IOException {

236

TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);

237

TermPositionVector vector = (TermPositionVector) reader.get(0, testFields[0]);

238

assertTrue(vector != null);

239

String[] terms = vector.getTerms();

240

assertTrue(terms != null);

241

assertTrue(terms.length == testTerms.length);

242

for (int i = 0; i < terms.length; i++) {

243

String term = terms[i];

244

//System.out.println("Term: " + term);

245

assertTrue(term.equals(testTerms[i]));

246

int[] positions = vector.getTermPositions(i);

247

assertTrue(positions != null);

248

assertTrue(positions.length == this.positions[i].length);

249

for (int j = 0; j < positions.length; j++) {

250

int position = positions[j];

251

assertTrue(position == this.positions[i][j]);

252

}

253

TermVectorOffsetInfo[] offset = vector.getOffsets(i);

254

assertTrue(offset != null);

255

assertTrue(offset.length == this.offsets[i].length);

256

for (int j = 0; j < offset.length; j++) {

257

TermVectorOffsetInfo termVectorOffsetInfo = offset[j];

258

assertTrue(termVectorOffsetInfo.equals(offsets[i][j]));

259

}

260

}

261

reader.close();

262

}

263

264

public void testMapper() throws IOException {

265

TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);

266

SortedTermVectorMapper mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());

267

reader.get(0, mapper);

268

SortedSet<TermVectorEntry> set = mapper.getTermVectorEntrySet();

269

assertTrue("set is null and it shouldn't be", set != null);

270

//three fields, 4 terms, all terms are the same

271

assertTrue("set Size: " + set.size() + " is not: " + 4, set.size() == 4);

272

//Check offsets and positions

273

for (Iterator<TermVectorEntry> iterator = set.iterator(); iterator.hasNext();) {

274

TermVectorEntry tve = iterator.next();

275

assertTrue("tve is null and it shouldn't be", tve != null);

276

assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);

277

assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);

278

279

}

280

281

mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());

282

reader.get(1, mapper);

283

set = mapper.getTermVectorEntrySet();

284

assertTrue("set is null and it shouldn't be", set != null);

285

//three fields, 4 terms, all terms are the same

286

assertTrue("set Size: " + set.size() + " is not: " + 4, set.size() == 4);

287

//Should have offsets and positions b/c we are munging all the fields together

288

for (Iterator<TermVectorEntry> iterator = set.iterator(); iterator.hasNext();) {

289

TermVectorEntry tve = iterator.next();

290

assertTrue("tve is null and it shouldn't be", tve != null);

291

assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);

292

assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);

293

294

}

295

296

297

FieldSortedTermVectorMapper fsMapper = new FieldSortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());

298

reader.get(0, fsMapper);

299

Map<String,SortedSet<TermVectorEntry>> map = fsMapper.getFieldToTerms();

300

assertTrue("map Size: " + map.size() + " is not: " + testFields.length, map.size() == testFields.length);

301

for (Map.Entry<String,SortedSet<TermVectorEntry>> entry : map.entrySet()) {

302

SortedSet<TermVectorEntry> sortedSet = entry.getValue();

303

assertTrue("sortedSet Size: " + sortedSet.size() + " is not: " + 4, sortedSet.size() == 4);

304

for (final TermVectorEntry tve : sortedSet) {

305

assertTrue("tve is null and it shouldn't be", tve != null);

306

//Check offsets and positions.

307

assertTrue("tve is null and it shouldn't be", tve != null);

308

String field = tve.getField();

309

if (field.equals(testFields[0])) {

310

//should have offsets

311

312

assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);

313

assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);

314

}

315

else if (field.equals(testFields[1])) {

316

//should not have offsets

317

318

assertTrue("tve.getOffsets() is not null and it shouldn't be", tve.getOffsets() == null);

319

assertTrue("tve.getPositions() is not null and it shouldn't be", tve.getPositions() == null);

320

}

321

}

322

}

323

//Try mapper that ignores offs and positions

324

fsMapper = new FieldSortedTermVectorMapper(true, true, new TermVectorEntryFreqSortedComparator());

325

reader.get(0, fsMapper);

326

map = fsMapper.getFieldToTerms();

327

assertTrue("map Size: " + map.size() + " is not: " + testFields.length, map.size() == testFields.length);

328

for (final Map.Entry<String,SortedSet<TermVectorEntry>> entry : map.entrySet()) {

329

SortedSet<TermVectorEntry> sortedSet = entry.getValue();

330

assertTrue("sortedSet Size: " + sortedSet.size() + " is not: " + 4, sortedSet.size() == 4);

331

for (final TermVectorEntry tve : sortedSet) {

332

assertTrue("tve is null and it shouldn't be", tve != null);

333

//Check offsets and positions.

334

assertTrue("tve is null and it shouldn't be", tve != null);

335

String field = tve.getField();

336

if (field.equals(testFields[0])) {

337

//should have offsets

338

339

assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() == null);

340

assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() == null);

341

}

342

else if (field.equals(testFields[1])) {

343

//should not have offsets

344

345

assertTrue("tve.getOffsets() is not null and it shouldn't be", tve.getOffsets() == null);

346

assertTrue("tve.getPositions() is not null and it shouldn't be", tve.getPositions() == null);

347

}

348

}

349

}

350

351

// test setDocumentNumber()

352

IndexReader ir = IndexReader.open(dir, true);

353

DocNumAwareMapper docNumAwareMapper = new DocNumAwareMapper();

354

assertEquals(-1, docNumAwareMapper.getDocumentNumber());

355

356

ir.getTermFreqVector(0, docNumAwareMapper);

357

assertEquals(0, docNumAwareMapper.getDocumentNumber());

358

docNumAwareMapper.setDocumentNumber(-1);

359

360

ir.getTermFreqVector(1, docNumAwareMapper);

361

assertEquals(1, docNumAwareMapper.getDocumentNumber());

362

docNumAwareMapper.setDocumentNumber(-1);

363

364

ir.getTermFreqVector(0, "f1", docNumAwareMapper);

365

assertEquals(0, docNumAwareMapper.getDocumentNumber());

366

docNumAwareMapper.setDocumentNumber(-1);

367

368

ir.getTermFreqVector(1, "f2", docNumAwareMapper);

369

assertEquals(1, docNumAwareMapper.getDocumentNumber());

370

docNumAwareMapper.setDocumentNumber(-1);

371

372

ir.getTermFreqVector(0, "f1", docNumAwareMapper);

373

assertEquals(0, docNumAwareMapper.getDocumentNumber());

374

375

ir.close();

376

reader.close();

377

}

378

379

380

/**

381

* Make sure exceptions and bad params are handled appropriately

382

383

public void testBadParams() throws IOException {

384

TermVectorsReader reader = null;

385

try {

386

reader = new TermVectorsReader(dir, seg, fieldInfos);

387

//Bad document number, good field number

388

reader.get(50, testFields[0]);

389

fail();

390

} catch (IOException e) {

391

// expected exception

392

} finally {

393

reader.close();

394

}

395

try {

396

reader = new TermVectorsReader(dir, seg, fieldInfos);

397

//Bad document number, no field

398

reader.get(50);

399

fail();

400

} catch (IOException e) {

401

// expected exception

402

} finally {

403

reader.close();

404

}

405

try {

406

reader = new TermVectorsReader(dir, seg, fieldInfos);

407

//good document number, bad field number

408

TermFreqVector vector = reader.get(0, "f50");

409

assertTrue(vector == null);

410

reader.close();

411

} catch (IOException e) {

412

fail();

413

} finally {

414

reader.close();

415

}

416

}

417

418

419

public static class DocNumAwareMapper extends TermVectorMapper {

420

421

public DocNumAwareMapper() {

422

}

423

424

private int documentNumber = -1;

425

426

@Override

427

public void setExpectations(String field, int numTerms, boolean storeOffsets, boolean storePositions) {

428

if (documentNumber == -1) {

429

throw new RuntimeException("Documentnumber should be set at this point!");

430

}

431

}

432

433

@Override

434

public void map(String term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {

435

if (documentNumber == -1) {

436

throw new RuntimeException("Documentnumber should be set at this point!");

437

}

438

}

439

440

public int getDocumentNumber() {

441

return documentNumber;

442

}

443

444

@Override

445

public void setDocumentNumber(int documentNumber) {

446

this.documentNumber = documentNumber;

447

}

448

}

449

}

Older »