« back to all changes in this revision
Viewing changes to yum/sqlitecache.py
- browse files at revision 4
- compare with another revision
- download diff
- download tarball
- view history from revision 4
- Committer: Bazaar Package Importer
- Author(s): Ben Hutchings
- Date: 2008-07-28 23:20:59 UTC
- mfrom: (2.1.3 intrepid)
- Revision ID: james.westby@ubuntu.com-20080728232059-24lo1r17smhr71l8
Tags: 3.2.12-1.2
* Non-maintainer upload
* Updated for compatibility with current python-pyme (Closes: #490368)
based on patch by Martin Meredith <mez@ubuntu.com>
- Changed import in yum/misc.py
- Set versioned dependency on python-pyme
* Updated for compatibility with current python-pyme (Closes: #490368)
based on patch by Martin Meredith <mez@ubuntu.com>
- Changed import in yum/misc.py
- Set versioned dependency on python-pyme
- files added:
- .gitignore~
- test
- files removed:
- bin/yum-arch
- repomd
- files modified:
- AUTHORS
added
removed
yum/sqlitecache.py
1
#!/usr/bin/python -tt
2
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU Library General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16
# Copyright 2005 Duke University
17
18
# TODO
19
# - Add support for multiple checksums per rpm (not required)
20
21
import os
22
import sqlite
23
import time
24
import mdparser
25
from sqlitesack import encodefiletypelist,encodefilenamelist
26
27
# This version refers to the internal structure of the sqlite cache files
28
# increasing this number forces all caches of a lower version number
29
# to be re-generated
30
dbversion = '6'
31
32
class RepodataParserSqlite:
33
def __init__(self, storedir, repoid, callback=None):
34
self.storedir = storedir
35
self.callback = callback
36
self.repodata = {
37
'metadata': {},
38
'filelists': {},
39
'otherdata': {}
40
}
41
self.repoid = repoid
42
self.debug = 0
43
44
def loadCache(self,filename):
45
"""Load cache from filename, check if it is valid and that dbversion
46
matches the required dbversion"""
47
db = sqlite.connect(filename)
48
cur = db.cursor()
49
cur.execute("select * from db_info")
50
info = cur.fetchone()
51
# If info is not in there this is an incompelete cache file
52
# (this could happen when the user hits ctrl-c or kills yum
53
# when the cache is being generated or updated)
54
if (not info):
55
raise sqlite.DatabaseError, "Incomplete database cache file"
56
57
# Now check the database version
58
if (info['dbversion'] != dbversion):
59
self.log(2, "Warning: cache file is version %s, we need %s, will regenerate" % (
60
info['dbversion'], dbversion))
61
raise sqlite.DatabaseError, "Older version of yum sqlite"
62
63
# This appears to be a valid database, return checksum value and
64
# database object
65
return (info['checksum'],db)
66
67
def getFilename(self,location):
68
return location + '.sqlite'
69
70
def getDatabase(self, location, cachetype):
71
filename = self.getFilename(location)
72
dbchecksum = ''
73
# First try to open an existing database
74
try:
75
f = open(filename)
76
(dbchecksum,db) = self.loadCache(filename)
77
except (IOError,sqlite.DatabaseError,KeyError):
78
# If it doesn't exist, create it
79
db = self.makeSqliteCacheFile(filename,cachetype)
80
return (db,dbchecksum)
81
82
def _getbase(self, location, checksum, metadatatype):
83
(db, dbchecksum) = self.getDatabase(location, metadatatype)
84
# db should now contain a valid database object, check if it is
85
# up to date
86
if (checksum != dbchecksum):
87
self.log(3, "%s sqlite cache needs updating, reading in metadata" % (metadatatype))
88
parser = mdparser.MDParser(location)
89
self.updateSqliteCache(db, parser, checksum, metadatatype)
90
db.commit()
91
return db
92
93
def getPrimary(self, location, checksum):
94
"""Load primary.xml.gz from an sqlite cache and update it
95
if required"""
96
return self._getbase(location, checksum, 'primary')
97
98
def getFilelists(self, location, checksum):
99
"""Load filelist.xml.gz from an sqlite cache and update it if
100
required"""
101
return self._getbase(location, checksum, 'filelists')
102
103
def getOtherdata(self, location, checksum):
104
"""Load other.xml.gz from an sqlite cache and update it if required"""
105
return self._getbase(location, checksum, 'other')
106
107
def createTablesFilelists(self,db):
108
"""Create the required tables for filelists metadata in the sqlite
109
database"""
110
cur = db.cursor()
111
self.createDbInfo(cur)
112
# This table is needed to match pkgKeys to pkgIds
113
cur.execute("""CREATE TABLE packages(
114
pkgKey INTEGER PRIMARY KEY,
115
pkgId TEXT)
116
""")
117
cur.execute("""CREATE TABLE filelist(
118
pkgKey INTEGER,
119
dirname TEXT,
120
filenames TEXT,
121
filetypes TEXT)
122
""")
123
cur.execute("CREATE INDEX keyfile ON filelist (pkgKey)")
124
cur.execute("CREATE INDEX pkgId ON packages (pkgId)")
125
126
def createTablesOther(self,db):
127
"""Create the required tables for other.xml.gz metadata in the sqlite
128
database"""
129
cur = db.cursor()
130
self.createDbInfo(cur)
131
# This table is needed to match pkgKeys to pkgIds
132
cur.execute("""CREATE TABLE packages(
133
pkgKey INTEGER PRIMARY KEY,
134
pkgId TEXT)
135
""")
136
cur.execute("""CREATE TABLE changelog(
137
pkgKey INTEGER,
138
author TEXT,
139
date TEXT,
140
changelog TEXT)
141
""")
142
cur.execute("CREATE INDEX keychange ON changelog (pkgKey)")
143
cur.execute("CREATE INDEX pkgId ON packages (pkgId)")
144
145
def createTablesPrimary(self,db):
146
"""Create the required tables for primary metadata in the sqlite
147
database"""
148
149
cur = db.cursor()
150
self.createDbInfo(cur)
151
# The packages table contains most of the information in primary.xml.gz
152
153
q = 'CREATE TABLE packages(\n' \
154
'pkgKey INTEGER PRIMARY KEY,\n'
155
156
cols = []
157
for col in PackageToDBAdapter.COLUMNS:
158
cols.append('%s TEXT' % col)
159
q += ',\n'.join(cols) + ')'
160
161
cur.execute(q)
162
163
# Create requires, provides, conflicts and obsoletes tables
164
# to store prco data
165
for t in ('requires','provides','conflicts','obsoletes'):
166
cur.execute("""CREATE TABLE %s (
167
name TEXT,
168
flags TEXT,
169
epoch TEXT,
170
version TEXT,
171
release TEXT,
172
pkgKey TEXT)
173
""" % (t))
174
# Create the files table to hold all the file information
175
cur.execute("""CREATE TABLE files (
176
name TEXT,
177
type TEXT,
178
pkgKey TEXT)
179
""")
180
# Create indexes for faster searching
181
cur.execute("CREATE INDEX packagename ON packages (name)")
182
cur.execute("CREATE INDEX providesname ON provides (name)")
183
cur.execute("CREATE INDEX packageId ON packages (pkgId)")
184
db.commit()
185
186
def createDbInfo(self,cur):
187
# Create the db_info table, this contains sqlite cache metadata
188
cur.execute("""CREATE TABLE db_info (
189
dbversion TEXT,
190
checksum TEXT)
191
""")
192
193
def insertHash(self,table,hash,cursor):
194
"""Insert the key value pairs in hash into a database table"""
195
196
keys = hash.keys()
197
values = hash.values()
198
query = "INSERT INTO %s (" % (table)
199
query += ",".join(keys)
200
query += ") VALUES ("
201
# Quote all values by replacing None with NULL and ' by ''
202
for x in values:
203
if (x == None):
204
query += "NULL,"
205
else:
206
try:
207
query += "'%s'," % (x.replace("'","''"))
208
except AttributeError:
209
query += "'%s'," % x
210
# Remove the last , from query
211
query = query[:-1]
212
# And replace it with )
213
query += ")"
214
cursor.execute(query.encode('utf8'))
215
return cursor.lastrowid
216
217
def makeSqliteCacheFile(self, filename, cachetype):
218
"""Create an initial database in the given filename"""
219
220
# If it exists, remove it as we were asked to create a new one
221
if (os.path.exists(filename)):
222
self.log(3, "Warning: cache already exists, removing old version")
223
try:
224
os.unlink(filename)
225
except OSError:
226
pass
227
228
# Try to create the databse in filename, or use in memory when
229
# this fails
230
try:
231
f = open(filename,'w')
232
db = sqlite.connect(filename)
233
except IOError:
234
self.log(1, "Warning could not create sqlite cache file, using in memory cache instead")
235
db = sqlite.connect(":memory:")
236
237
# The file has been created, now create the tables and indexes
238
if (cachetype == 'primary'):
239
self.createTablesPrimary(db)
240
elif (cachetype == 'filelists'):
241
self.createTablesFilelists(db)
242
elif (cachetype == 'other'):
243
self.createTablesOther(db)
244
else:
245
raise sqlite.DatabaseError, "Sorry don't know how to store %s" % (cachetype)
246
return db
247
248
def addPrimary(self, pkgId, package, cur):
249
"""Add a package to the primary cache"""
250
# Store the package info into the packages table
251
pkgKey = self.insertHash('packages', PackageToDBAdapter(package), cur)
252
253
# Now store all prco data
254
for ptype in package.prco:
255
for entry in package.prco[ptype]:
256
data = {
257
'pkgKey': pkgKey,
258
'name': entry.get('name'),
259
'flags': entry.get('flags'),
260
'epoch': entry.get('epoch'),
261
'version': entry.get('ver'),
262
'release': entry.get('rel'),
263
}
264
self.insertHash(ptype,data,cur)
265
266
# Now store all file information
267
for f in package.files:
268
data = {
269
'name': f,
270
'type': package.files[f],
271
'pkgKey': pkgKey,
272
}
273
self.insertHash('files',data,cur)
274
275
def addFilelists(self, pkgId, package,cur):
276
"""Add a package to the filelists cache"""
277
pkginfo = {'pkgId': pkgId}
278
pkgKey = self.insertHash('packages',pkginfo, cur)
279
dirs = {}
280
for (filename,filetype) in package.files.iteritems():
281
(dirname,filename) = (os.path.split(filename))
282
if (dirs.has_key(dirname)):
283
dirs[dirname]['files'].append(filename)
284
dirs[dirname]['types'].append(filetype)
285
else:
286
dirs[dirname] = {}
287
dirs[dirname]['files'] = [filename]
288
dirs[dirname]['types'] = [filetype]
289
290
for (dirname,dir) in dirs.items():
291
data = {
292
'pkgKey': pkgKey,
293
'dirname': dirname,
294
'filenames': encodefilenamelist(dir['files']),
295
'filetypes': encodefiletypelist(dir['types'])
296
}
297
self.insertHash('filelist',data,cur)
298
299
def addOther(self, pkgId, package,cur):
300
pkginfo = {'pkgId': pkgId}
301
pkgKey = self.insertHash('packages', pkginfo, cur)
302
for entry in package['changelog']:
303
data = {
304
'pkgKey': pkgKey,
305
'author': entry.get('author'),
306
'date': entry.get('date'),
307
'changelog': entry.get('value'),
308
}
309
self.insertHash('changelog', data, cur)
310
311
def updateSqliteCache(self, db, parser, checksum, cachetype):
312
"""Update the sqlite cache by making it fit the packages described
313
in dobj (info that has been read from primary.xml metadata) afterwards
314
update the checksum of the database to checksum"""
315
316
t = time.time()
317
delcount = 0
318
newcount = 0
319
320
# We start be removing the old db_info, as it is no longer valid
321
cur = db.cursor()
322
cur.execute("DELETE FROM db_info")
323
324
# First create a list of all pkgIds that are in the database now
325
cur.execute("SELECT pkgId, pkgKey from packages")
326
currentpkgs = {}
327
for pkg in cur.fetchall():
328
currentpkgs[pkg['pkgId']] = pkg['pkgKey']
329
330
if (cachetype == 'primary'):
331
deltables = ("packages","files","provides","requires",
332
"conflicts","obsoletes")
333
elif (cachetype == 'filelists'):
334
deltables = ("packages","filelist")
335
elif (cachetype == 'other'):
336
deltables = ("packages","changelog")
337
else:
338
raise sqlite.DatabaseError,"Unknown type %s" % (cachetype)
339
340
# Add packages that are not in the database yet and build up a list of
341
# all pkgids in the current metadata
342
343
all_pkgIds = {}
344
for package in parser:
345
346
if self.callback is not None:
347
self.callback.progressbar(parser.count, parser.total, self.repoid)
348
349
pkgId = package['pkgId']
350
all_pkgIds[pkgId] = 1
351
352
# This package is already in the database, skip it now
353
if (currentpkgs.has_key(pkgId)):
354
continue
355
356
# This is a new package, lets insert it
357
newcount += 1
358
if cachetype == 'primary':
359
self.addPrimary(pkgId, package, cur)
360
elif cachetype == 'filelists':
361
self.addFilelists(pkgId, package, cur)
362
elif cachetype == 'other':
363
self.addOther(pkgId, package, cur)
364
365
# Remove those which are not in dobj
366
delpkgs = []
367
for (pkgId, pkgKey) in currentpkgs.items():
368
if not all_pkgIds.has_key(pkgId):
369
delcount += 1
370
delpkgs.append(str(pkgKey))
371
delpkgs = "("+",".join(delpkgs)+")"
372
for table in deltables:
373
cur.execute("DELETE FROM "+table+ " where pkgKey in %s" % delpkgs)
374
375
cur.execute("INSERT into db_info (dbversion,checksum) VALUES (%s,%s)",
376
(dbversion,checksum))
377
db.commit()
378
self.log(2, "Added %s new packages, deleted %s old in %.2f seconds" % (
379
newcount, delcount, time.time()-t))
380
return db
381
382
def log(self, level, msg):
383
'''Log to callback (if set)
384
'''
385
if self.callback:
386
self.callback.log(level, msg)
387
388
class PackageToDBAdapter:
389
390
'''
391
Adapt a PrimaryEntry instance to suit the sqlite database.
392
393
This hides certain attributes and converts some column names in order to
394
decouple the parser implementation from the sqlite database schema.
395
'''
396
397
NAME_MAPS = {
398
'rpm_package': 'package',
399
'version': 'ver',
400
'release': 'rel',
401
'rpm_license': 'license',
402
'rpm_vendor': 'vendor',
403
'rpm_group': 'group',
404
'rpm_buildhost': 'buildhost',
405
'rpm_sourcerpm': 'sourcerpm',
406
'rpm_packager': 'packager',
407
}
408
409
COLUMNS = (
410
'pkgId',
411
'name',
412
'arch',
413
'version',
414
'epoch',
415
'release',
416
'summary',
417
'description',
418
'url',
419
'time_file',
420
'time_build',
421
'rpm_license',
422
'rpm_vendor',
423
'rpm_group',
424
'rpm_buildhost',
425
'rpm_sourcerpm',
426
'rpm_header_start',
427
'rpm_header_end',
428
'rpm_packager',
429
'size_package',
430
'size_installed',
431
'size_archive',
432
'location_href',
433
'checksum_type',
434
'checksum_value',
435
)
436
437
def __init__(self, package):
438
self._pkg = package
439
440
def __getitem__(self, k):
441
return self._pkg[self.NAME_MAPS.get(k, k)]
442
443
def keys(self):
444
return self.COLUMNS
445
446
def values(self):
447
out = []
448
for k in self.keys():
449
out.append(self[k])
450
return out
451
Loggerhead is a web-based interface for Breezy
Version: 2.0.1