~ubuntu-branches/ubuntu/saucy/apt-xapian-index/saucy : revision 5.1.9

1

# -*- coding: utf-8 -*-

2

#

3

# axi/indexer.py - apt-xapian-index indexer

4

#

5

6

#

7

# This program is free software; you can redistribute it and/or modify

8

# it under the terms of the GNU General Public License as published by

9

# the Free Software Foundation; either version 2 of the License, or

10

# (at your option) any later version.

11

#

12

# This program is distributed in the hope that it will be useful,

13

# but WITHOUT ANY WARRANTY; without even the implied warranty of

14

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

15

# GNU General Public License for more details.

16

#

17

# You should have received a copy of the GNU General Public License

18

# along with this program; if not, write to the Free Software

19

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

20

#

21

22

import axi

23

import sys

24

import os

25

import imp

26

import socket, errno

27

import fcntl

28

import textwrap

29

import xapian

30

import shutil

31

import itertools

32

import time

33

import re

34

import urllib

35

import cPickle as pickle

36

37

APTLISTDIR="/var/lib/apt/lists"

38

39

class Addon:

40

"""

41

Indexer plugin wrapper

42

"""

43

def __init__(self, fname, **kw):

44

self.filename = os.path.basename(fname)

45

self.name = os.path.splitext(self.filename)[0]

46

oldpath = sys.path

47

try:

48

sys.path.append(os.path.dirname(fname))

49

self.module = imp.load_source("axi.plugin_" + self.name, fname)

50

finally:

51

sys.path = oldpath

52

try:

53

self.obj = self.module.init(**kw)

54

except TypeError:

55

self.obj = self.module.init()

56

if self.obj:

57

self.info = self.obj.info()

58

59

def send_extra_info(self, **kw):

60

func = getattr(self.obj, "send_extra_info", None)

61

if func is not None:

62

func(**kw)

63

64

class Progress:

65

"""

66

Normal progress report to stdout

67

"""

68

def __init__(self):

69

self.task = None

70

self.halfway = False

71

self.is_verbose = False

72

def begin(self, task):

73

self.task = task

74

print "%s..." % self.task,

75

sys.stdout.flush()

76

self.halfway = True

77

def progress(self, percent):

78

print "\r%s... %d%%" % (self.task, percent),

79

sys.stdout.flush()

80

self.halfway = True

81

def end(self):

82

print "\r%s: done. " % self.task

83

self.halfway = False

84

def verbose(self, *args):

85

if not self.is_verbose: return

86

if self.halfway:

87

print

88

print " ".join(args)

89

self.halfway = False

90

def notice(self, *args):

91

if self.halfway:

92

print

93

print >>sys.stderr, " ".join(args)

94

self.halfway = False

95

def warning(self, *args):

96

if self.halfway:

97

print

98

print >>sys.stderr, " ".join(args)

99

self.halfway = False

100

def error(self, *args):

101

if self.halfway:

102

print

103

print >>sys.stderr, " ".join(args)

104

self.halfway = False

105

106

class BatchProgress:

107

"""

108

Machine readable progress report

109

"""

110

def __init__(self):

111

self.task = None

112

def begin(self, task):

113

self.task = task

114

print "begin: %s\n" % self.task,

115

sys.stdout.flush()

116

def progress(self, percent):

117

print "progress: %d/100\n" % percent,

118

sys.stdout.flush()

119

def end(self):

120

print "done: %s\n" % self.task

121

sys.stdout.flush()

122

def verbose(self, *args):

123

print "verbose: %s" % (" ".join(args))

124

sys.stdout.flush()

125

def notice(self, *args):

126

print "notice: %s" % (" ".join(args))

127

sys.stdout.flush()

128

def warning(self, *args):

129

print "warning: %s" % (" ".join(args))

130

sys.stdout.flush()

131

def error(self, *args):

132

print "error: %s" % (" ".join(args))

133

sys.stdout.flush()

134

135

class SilentProgress:

136

"""

137

Quiet progress report

138

"""

139

def begin(self, task):

140

pass

141

def progress(self, percent):

142

pass

143

def end(self):

144

pass

145

def verbose(self, *args):

146

pass

147

def notice(self, *args):

148

pass

149

def warning(self, *args):

150

print >>sys.stderr, " ".join(args)

151

def error(self, *args):

152

print >>sys.stderr, " ".join(args)

153

154

class ClientProgress:

155

"""

156

Client-side progress report, reporting progress from another running

157

indexer

158

"""

159

def __init__(self, progress):

160

self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

161

self.sock.settimeout(None)

162

self.sock.connect(axi.XAPIANDBUPDATESOCK)

163

self.progress = progress

164

165

def loop(self):

166

hasBegun = False

167

while True:

168

msg = self.sock.recv(4096)

169

try:

170

args = pickle.loads(msg)

171

except EOFError:

172

progress.error("The other update has stopped")

173

return

174

action = args[0]

175

args = args[1:]

176

if action == "begin":

177

progress.begin(*args)

178

hasBegun = True

179

elif action == "progress":

180

if not hasBegun:

181

progress.begin(args[0])

182

hasBegun = True

183

progress.progress(*args[1:])

184

elif action == "end":

185

if not hasBegun:

186

progress.begin(args[0])

187

hasBegun = True

188

progress.end(*args[1:])

189

elif action == "verbose":

190

progress.verbose(*args)

191

elif action == "notice":

192

progress.notice(*args)

193

elif action == "error":

194

progress.error(*args)

195

elif action == "alldone":

196

break

197

else:

198

progress.error("unknown action '%s' from other update-apt-xapian-index. Arguments: '%s'" % (action, ", ".join(map(repr, args))))

199

200

201

class ServerSenderProgress:

202

"""

203

Server endpoint for client-server progress report

204

"""

205

def __init__(self, sock, task = None):

206

self.sock = sock

207

self.task = task

208

def __del__(self):

209

self._send(pickle.dumps(("alldone",)))

210

def _send(self, text):

211

try:

212

self.sock.send(text)

213

except:

214

pass

215

def begin(self, task):

216

self.task = task

217

self._send(pickle.dumps(("begin", self.task)))

218

def progress(self, percent):

219

self._send(pickle.dumps(("progress", self.task, percent)))

220

def end(self):

221

self._send(pickle.dumps(("end", self.task)))

222

def verbose(self, *args):

223

self._send(pickle.dumps(("verbose",) + args))

224

def notice(self, *args):

225

self._send(pickle.dumps(("notice",) + args))

226

def warning(self, *args):

227

self._send(pickle.dumps(("warning",) + args))

228

def error(self, *args):

229

self._send(pickle.dumps(("error",) + args))

230

231

class ServerProgress:

232

"""

233

Send progress report to any progress object, as well as to client indexers

234

"""

235

def __init__(self, mine):

236

self.task = None

237

self.proxied = [mine]

238

self.sockfile = axi.XAPIANDBUPDATESOCK

239

try:

240

os.unlink(self.sockfile)

241

except OSError:

242

pass

243

self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

244

self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

245

self.sock.bind(axi.XAPIANDBUPDATESOCK)

246

self.sock.setblocking(False)

247

self.sock.listen(5)

248

# Disallowing unwanted people to mess with the file is automatic, as

249

# the socket has the ownership of the user we're using, and people

250

# can't connect to it unless they can write to it

251

def __del__(self):

252

self.sock.close()

253

os.unlink(self.sockfile)

254

def _check(self):

255

try:

256

sock = self.sock.accept()[0]

257

self.proxied.append(ServerSenderProgress(sock, self.task))

258

except socket.error, e:

259

if e.args[0] != errno.EAGAIN:

260

raise

261

pass

262

def begin(self, task):

263

self._check()

264

self.task = task

265

for x in self.proxied: x.begin(task)

266

def progress(self, percent):

267

self._check()

268

for x in self.proxied: x.progress(percent)

269

def end(self):

270

self._check()

271

for x in self.proxied: x.end()

272

def verbose(self, *args):

273

self._check()

274

for x in self.proxied: x.verbose(*args)

275

def notice(self, *args):

276

self._check()

277

for x in self.proxied: x.notice(*args)

278

def warning(self, *args):

279

self._check()

280

for x in self.proxied: x.warning(*args)

281

def error(self, *args):

282

self._check()

283

for x in self.proxied: x.error(*args)

284

285

286

class ExecutionTime(object):

287

"""

288

Helper that can be used in with statements to have a simple

289

measure of the timing of a particular block of code, e.g.

290

with ExecutionTime("db flush"):

291

db.flush()

292

"""

293

import time

294

def __init__(self, info=""):

295

self.info = info

296

def __enter__(self):

297

self.now = time.time()

298

def __exit__(self, type, value, stack):

299

print "%s: %s" % (self.info, time.time() - self.now)

300

301

class Indexer(object):

302

"""

303

The indexer

304

"""

305

def __init__(self, progress, quietapt=False):

306

self.progress = progress

307

self.quietapt = quietapt

308

self.verbose = getattr(progress, "is_verbose", False)

309

# Timestamp of the most recent data source

310

self.ds_timestamp = 0

311

# Apt cache instantiated on demand

312

self.apt_cache = None

313

# Languages we index

314

self.langs = set()

315

316

# Look for files like: ftp.uk.debian.org_debian_dists_sid_main_i18n_Translation-it

317

# And extract the language code at the end

318

tfile = re.compile(r"_i18n_Translation-([^-]+)$")

319

for f in os.listdir(APTLISTDIR):

320

mo = tfile.search(f)

321

if not mo: continue

322

self.langs.add(urllib.unquote(mo.group(1)))

323

324

# Create the database directory if missing

325

try:

326

# Try to create it anyway

327

os.mkdir(axi.XAPIANDBPATH)

328

except OSError, e:

329

if e.errno != errno.EEXIST:

330

# If we got an error besides path already existing, fail

331

raise

332

elif not os.path.isdir(axi.XAPIANDBPATH):

333

# If that path already exists, but is not a directory, also fail

334

raise

335

336

def _test_wrap_apt_cache(self, wrapper):

337

"""

338

Wrap the apt-cache in some proxy object.

339

340

This is used to give tests some control over the apt cache results

341

"""

342

if self.apt_cache is not None:

343

raise RuntimeError("the cache has already been instantiated")

344

# Instantiate the cache

345

self.aptcache()

346

# Wrap it

347

self.apt_cache = wrapper(self.apt_cache)

348

349

def aptcache(self):

350

if not self.apt_cache:

351

#import warnings

352

## Yes, apt, thanks, I know, the api isn't stable, thank you so very much

353

##warnings.simplefilter('ignore', FutureWarning)

354

#warnings.filterwarnings("ignore","apt API not stable yet")

355

import apt

356

#warnings.resetwarnings()

357

358

if self.quietapt:

359

class AptSilentProgress(apt.progress.text.OpProgress) :

360

def __init__(self):

361

pass

362

def done(self):

363

pass

364

def update(self,percent):

365

pass

366

aptprogress = AptSilentProgress()

367

else:

368

aptprogress = None

369

370

# memonly=True: force apt to not write a pkgcache.bin

371

self.apt_cache = apt.Cache(memonly=True, progress=aptprogress)

372

return self.apt_cache

373

374

def lock(self):

375

"""

376

Lock the session to prevent further updates.

377

378

@returns

379

True if the session is locked

380

False if another indexer is running

381

"""

382

# Lock the session so that we prevent concurrent updates

383

lockfd = os.open(axi.XAPIANDBLOCK, os.O_RDWR | os.O_CREAT)

384

lockpyfd = os.fdopen(lockfd)

385

try:

386

fcntl.lockf(lockpyfd, fcntl.LOCK_EX | fcntl.LOCK_NB)

387

# Wrap the current progress with the server sender

388

self.progress = ServerProgress(self.progress)

389

return True

390

except IOError, e:

391

if e.errno == errno.EACCES or e.errno == errno.EAGAIN:

392

return False

393

else:

394

raise

395

396

def slave(self):

397

"""

398

Attach to a running indexer and report its progress.

399

400

Return when the other indexer has finished.

401

"""

402

progress.notice("Another update is already running: showing its progress.")

403

childProgress = ClientProgress(self.progress)

404

childProgress.loop()

405

406

def readPlugins(self, **kw):

407

"""

408

Read the addons, in sorted order.

409

410

Pass all the keyword args to the plugin init

411

"""

412

self.addons = []

413

for fname in sorted(os.listdir(axi.PLUGINDIR)):

414

# Skip non-pythons, hidden files and python sources starting with '_'

415

if fname[0] in ['.', '_'] or not fname.endswith(".py"): continue

416

fullname = os.path.join(axi.PLUGINDIR, fname)

417

if not os.path.isfile(fullname): continue

418

self.progress.verbose("Reading plugin %s." % fullname)

419

addon = Addon(fullname, **kw)

420

if addon.obj != None:

421

self.addons.append(addon)

422

423

def setupIndexing(self, force=False):

424

"""

425

Setup indexing: read plugins, check timestamps...

426

427

@param force: if True, reindex also if the index is up to date

428

429

@return:

430

True if there is something to index

431

False if there is no need of indexing

432

"""

433

# Read values database

434

#values = readValueDB(VALUESCONF, progress)

435

436

# Read the addons, in sorted order

437

self.readPlugins(langs=self.langs, progress=self.progress)

438

439

# Ensure that we have something to do

440

if len(self.addons) == 0:

441

self.progress.notice("No indexing plugins found in %s" % axi.PLUGINDIR)

442

return False

443

444

# Get the most recent modification timestamp of the data sources

445

self.ds_timestamp = max([x.info['timestamp'] for x in self.addons])

446

447

# Get the timestamp of the last database update

448

try:

449

if os.path.exists(axi.XAPIANDBSTAMP):

450

cur_timestamp = os.path.getmtime(axi.XAPIANDBSTAMP)

451

else:

452

cur_timestamp = 0

453

except OSError, e:

454

cur_timestamp = 0

455

self.progress.notice("Reading current timestamp failed: %s. Assuming the index has not been created yet." % e)

456

457

if self.verbose:

458

self.progress.verbose("Most recent dataset: %s." % time.ctime(self.ds_timestamp))

459

self.progress.verbose("Most recent update for: %s." % time.ctime(cur_timestamp))

460

461

# See if we need an update

462

if self.ds_timestamp <= cur_timestamp:

463

if force:

464

self.progress.notice("The index %s is up to date, but rebuilding anyway as requested." % axi.XAPIANDBPATH)

465

else:

466

self.progress.notice("The index %s is up to date" % axi.XAPIANDBPATH)

467

return False

468

469

# Build the value database

470

self.progress.verbose("Aggregating value information.")

471

# Read existing value database to keep ids stable in a system

472

self.values, self.values_desc = axi.readValueDB(quiet=True)

473

values_seq = max(self.values.values()) + 1

474

for addon in self.addons:

475

for v in addon.info.get("values", []):

476

if v['name'] in self.values: continue

477

self.values[v['name']] = values_seq

478

values_seq += 1

479

self.values_desc[v['name']] = v['desc']

480

481

# Tell the addons to do the long initialisation bits

482

self.progress.verbose("Initializing plugins.")

483

for addon in self.addons:

484

addon.obj.init(dict(values = self.values), self.progress)

485

486

return True

487

488

def get_document_from_apt(self, pkg):

489

"""

490

Get a xapian.Document for the given apt package record

491

"""

492

document = xapian.Document()

493

# The document data is the package name

494

document.set_data(pkg.name)

495

# add information about the version of the package in slot 0

496

document.add_value(0, pkg.candidate.version)

497

# Index the package name with a special prefix, to be able to find this

498

# document by exact package name match

499

document.add_term("XP"+pkg.name)

500

# Have all the various plugins index their things

501

for addon in self.addons:

502

addon.obj.index(document, pkg)

503

return document

504

505

def get_document_from_deb822(self, pkg):

506

"""

507

Get a xapian.Document for the given deb822 package record

508

"""

509

document = xapian.Document()

510

511

# The document data is the package name

512

document.set_data(pkg["Package"])

513

# add information about the version of the package in slot 0

514

document.add_value(0, pkg["Version"])

515

# Index the package name with a special prefix, to be able to find this

516

# document by exact package name match

517

document.add_term("XP"+pkg["Package"])

518

# Have all the various plugins index their things

519

for addon in self.addons:

520

addon.obj.indexDeb822(document, pkg)

521

return document

522

523

def gen_documents_apt(self):

524

"""

525

Generate Xapian documents from an apt cache

526

"""

527

cache = self.aptcache()

528

count = len(cache)

529

for idx, pkg in enumerate(cache):

530

if not pkg.candidate:

531

continue

532

# Print progress

533

if idx % 200 == 0: self.progress.progress(100*idx/count)

534

yield self.get_document_from_apt(pkg)

535

536

def gen_documents_deb822(self, fname):

537

try:

538

from debian import deb822

539

except ImportError:

540

from debian_bundle import deb822

541

infd = open(fname)

542

# Get file size to compute progress

543

total = os.fstat(infd.fileno())[6]

544

for idx, pkg in enumerate(deb822.Deb822.iter_paragraphs(infd)):

545

# Print approximate progress by checking the current read position

546

# against the file size

547

if total > 0 and idx % 200 == 0:

548

cur = infd.tell()

549

self.progress.progress(100*cur/total)

550

yield self.get_document_from_deb822(pkg)

551

552

def compareCacheToDb(self, cache, db):

553

"""

554

Compare the apt cache to the database and return dicts

555

of the form (pkgname, docid) for the following states:

556

557

unchanged - no new version since the last update

558

outdated - a new version since the last update

559

obsolete - no longer in the apt cache

560

"""

561

unchanged = {}

562

outdated = {}

563

obsolete = {}

564

self.progress.begin("Reading Xapian index")

565

count = db.get_doccount()

566

for (idx, m) in enumerate(db.postlist("")):

567

if idx % 5000 == 0: self.progress.progress(100*idx/count)

568

doc = db.get_document(m.docid)

569

pkg = doc.get_data()

570

# this will return '' if there is no value 0, which is fine because it

571

# will fail the comparison with the candidate version causing a reindex

572

dbver = doc.get_value(0)

573

# check if the package no longer exists

574

if not cache.has_key(pkg) or not cache[pkg].candidate:

575

obsolete[pkg] = m.docid

576

# check if we have a new version, we do not have to delete

577

# the record,

578

elif cache[pkg].candidate.version != dbver:

579

outdated[pkg] = m.docid

580

# its a valid package and we know about it already

581

else:

582

unchanged[pkg] = m.docid

583

self.progress.end()

584

return unchanged, outdated, obsolete

585

586

def updateIndex(self, pathname):

587

"""

588

Update the index

589

"""

590

db = xapian.WritableDatabase(pathname, xapian.DB_CREATE_OR_OPEN)

591

cache = self.aptcache()

592

count = len(cache)

593

594

unchanged, outdated, obsolete = self.compareCacheToDb(cache, db)

595

self.progress.verbose("Unchanged versions: %s, oudated version: %s, "

596

"obsolete versions: %s" % (len(unchanged),

597

len(outdated),

598

len(obsolete)))

599

600

self.progress.begin("Updating Xapian index")

601

for a in self.addons: a.send_extra_info(db=db, aptcache=cache)

602

for idx, pkg in enumerate(cache):

603

if idx % 1000 == 0: self.progress.progress(100*idx/count)

604

if not pkg.candidate:

605

continue

606

if pkg.name in unchanged:

607

continue

608

elif pkg.name in outdated:

609

# update the existing

610

db.replace_document(outdated[pkg.name], self.get_document_from_apt(pkg))

611

else:

612

# add the new ones

613

db.add_document(self.get_document_from_apt(pkg))

614

615

# and remove the obsoletes

616

for docid in obsolete.values():

617

db.delete_document(docid)

618

619

# finished

620

db.flush()

621

self.progress.end()

622

623

def incrementalUpdate(self):

624

if not os.path.exists(axi.XAPIANINDEX):

625

self.progress.notice("No Xapian index built yet: falling back to full rebuild")

626

return self.rebuild()

627

628

dbkind, dbpath = open(axi.XAPIANINDEX).readline().split()

629

self.updateIndex(dbpath)

630

631

# Update the index timestamp

632

if not os.path.exists(axi.XAPIANDBSTAMP):

633

open(axi.XAPIANDBSTAMP, "w").close()

634

os.utime(axi.XAPIANDBSTAMP, (self.ds_timestamp, self.ds_timestamp))

635

636

def buildIndex(self, pathname, documents, addoninfo={}):

637

"""

638

Create a new Xapian index with the content provided by the addons

639

"""

640

self.progress.begin("Rebuilding Xapian index")

641

642

# Create a new Xapian index

643

db = xapian.WritableDatabase(pathname, xapian.DB_CREATE_OR_OVERWRITE)

644

# It seems to be faster without transactions, at the moment

645

#db.begin_transaction(False)

646

647

for a in self.addons: a.send_extra_info(db=db)

648

649

# Add all generated documents to the index

650

for doc in documents:

651

db.add_document(doc)

652

653

#db.commit_transaction();

654

db.flush()

655

self.progress.end()

656

657

def rebuild(self, pkgfile=None):

658

# Create a new Xapian index with the content provided by the addons

659

# Xapian takes care of preventing concurrent updates and removing the old

660

# database if it's left over by a previous crashed update

661

662

# Generate a new index name

663

for idx in itertools.count(1):

664

tmpidxfname = "index.%d" % idx

665

dbdir = axi.XAPIANDBPATH + "/" + tmpidxfname

666

if not os.path.exists(dbdir): break;

667

668

if pkgfile:

669

generator = self.gen_documents_deb822(pkgfile)

670

else:

671

for a in self.addons: a.send_extra_info(aptcache=self.aptcache())

672

generator = self.gen_documents_apt()

673

self.buildIndex(dbdir, generator)

674

675

# Update the 'index' symlink to point at the new index

676

self.progress.verbose("Installing the new index.")

677

678

#os.symlink(tmpidxfname, axi.XAPIANDBPATH + "/index.tmp")

679

out = open(axi.XAPIANINDEX + ".tmp", "w")

680

print >>out, "auto", os.path.join(os.path.abspath(axi.XAPIANDBPATH), tmpidxfname)

681

out.close()

682

os.rename(axi.XAPIANINDEX + ".tmp", axi.XAPIANINDEX)

683

684

# Remove all other index.* directories that are not the newly created one

685

for file in os.listdir(axi.XAPIANDBPATH):

686

if not file.startswith("index."): continue

687

# Only delete directories

688

if not os.path.isdir(axi.XAPIANDBPATH + "/" + file): continue

689

# Don't delete what we just created

690

if file == tmpidxfname: continue

691

fullpath = axi.XAPIANDBPATH + "/" + file

692

self.progress.verbose("Removing old index %s." % fullpath)

693

shutil.rmtree(fullpath)

694

695

# Commit the changes and update the last update timestamp

696

if not os.path.exists(axi.XAPIANDBSTAMP):

697

open(axi.XAPIANDBSTAMP, "w").close()

698

os.utime(axi.XAPIANDBSTAMP, (self.ds_timestamp, self.ds_timestamp))

699

700

self.writeValues()

701

self.writeDoc()

702

703

def writeValues(self, pathname=axi.XAPIANDBVALUES):

704

"""

705

Write the value information on the given file

706

"""

707

self.progress.verbose("Writing value information to %s." % pathname)

708

out = open(pathname+".tmp", "w")

709

710

print >>out, textwrap.dedent("""

711

# This file contains the mapping between names of numeric values indexed in the

712

# APT Xapian index and their index

713

#

714

# Xapian allows to index numeric values as well as keywords and to use them for

715

# all sorts of useful querying tricks. However, every numeric value needs to

716

# have a unique index, and this configuration file is needed to record which

717

# indices are allocated and to provide a mnemonic name for them.

718

#

719

# The format is exactly like /etc/services with name, number and optional

720

# aliases, with the difference that the second column does not use the

721

# "/protocol" part, which would be meaningless here.

722

""").lstrip()

723

724

for name, idx in sorted(self.values.iteritems(), key=lambda x: x[1]):

725

desc = self.values_desc[name]

726

print >>out, "%s\t%d\t# %s" % (name, idx, desc)

727

728

out.close()

729

# Atomic update of the documentation

730

os.rename(pathname+".tmp", pathname)

731

732

def writeDoc(self, pathname=axi.XAPIANDBDOC):

733

"""

734

Write the documentation in the given file

735

"""

736

self.progress.verbose("Writing documentation to %s." % pathname)

737

# Collect the documentation

738

docinfo = []

739

for addon in self.addons:

740

try:

741

doc = addon.obj.doc()

742

if doc != None:

743

docinfo.append(dict(

744

name = doc['name'],

745

shortDesc = doc['shortDesc'],

746

fullDoc = doc['fullDoc']))

747

except:

748

# If a plugin has problem returning documentation, don't worry about it

749

self.progress.notice("Skipping documentation for plugin", addon.filename)

750

751

# Write the documentation in pathname

752

out = open(pathname+".tmp", "w")

753

print >>out, textwrap.dedent("""

754

===============

755

Database layout

756

===============

757

758

This Xapian database indexes Debian package information. To query the

759

database, open it as ``%s/index``.

760

761

Data are indexed either as terms or as values. Words found in package

762

descriptions are indexed lowercase, and all other kinds of terms have an

763

uppercase prefix as documented below.

764

765

Numbers are indexed as Xapian numeric values. A list of the meaning of the

766

numeric values is found in ``%s``.

767

768

The data sources used for indexing are:

769

""").lstrip() % (axi.XAPIANDBPATH, axi.XAPIANDBVALUES)

770

771

for d in docinfo:

772

print >>out, " * %s: %s" % (d['name'], d['shortDesc'])

773

774

print >>out, textwrap.dedent("""

775

This Xapian index follows the conventions for term prefixes described in

776

``/usr/share/doc/xapian-omega/termprefixes.txt.gz``.

777

778

Extra Debian data sources can define more extended prefixes (starting with

779

``X``): their meaning is documented below together with the rest of the data

780

source documentation.

781

782

At the very least, at least the package name (with the ``XP`` prefix) will

783

be present in every document in the database. This allows to quickly

784

lookup a Xapian document by package name.

785

786

The user data associated to a Xapian document is the package name.

787

788

789

-------------------

790

Active data sources

791

-------------------

792

793

""")

794

for d in docinfo:

795

print >>out, d['name']

796

print >>out, '='*len(d['name'])

797

print >>out, textwrap.dedent(d['fullDoc'])

798

print >>out

799

800

out.close()

801

# Atomic update of the documentation

802

os.rename(pathname+".tmp", pathname)

803