1
# Miro - an RSS based video player application
2
# Copyright (C) 2005-2009 Participatory Culture Foundation
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU General Public License for more details.
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18
# In addition, as a special exception, the copyright holders give
19
# permission to link the code of portions of this program with the OpenSSL
22
# You must obey the GNU General Public License in all respects for all of
23
# the code used other than OpenSSL. If you modify file(s) with this
24
# exception, you may extend this exception to your version of the file(s),
25
# but you are not obligated to do so. If you do not wish to do so, delete
26
# this exception statement from your version. If you delete this exception
27
# statement from all source files in the program, then also delete it here.
29
"""Module used to upgrade from databases before we had our current scheme.
32
* Unpickle old databases using a subclass of pickle.Unpickle that loads
33
fake class objects for all our DDBObjects. The fake classes are just
34
empty shells with the upgrade code that existed when we added the schema
37
* Save those objects to disk, using the initial schema of the new system.
41
from new import classobj
43
from datetime import datetime
50
from miro.schema import ObjectSchema, SchemaInt, SchemaFloat, SchemaSimpleItem
51
from miro.schema import SchemaObject, SchemaBool, SchemaDateTime, SchemaTimeDelta
52
from miro.schema import SchemaList, SchemaDict
53
from fasttypes import LinkedList
54
from types import NoneType
55
from miro import storedatabase
57
######################### STAGE 1 helpers #############################
58
# Below is a snapshot of what the database looked like at 0.8.2. DDBObject
59
# classes and other classes that get saved in the database are present only as
60
# skeletons, all we want from them is their __setstate__ method.
62
# The __setstate_ methods are almost exactly like they were in 0.8.2. I
63
# removed some things that don't apply to us simple restoring, then saving the
64
# database (starting a Thread, sending messages to the downloader daemon,
65
# etc.). I added some things to make things compatible with our schema,
66
# mostly this means setting attributes to None, where before we used the fact
67
# that access the attribute would throw an AttributeError (ugh!).
69
# We prepend "Old" to the DDBObject so they're easy to recognize if
70
# somehow they slip through to a real database
73
# classes are exactly as they appeared in version 6 of the schema.
76
# Previous versions were in RC's. They dropped some of the data that we
77
# need to import from old databases By making olddatabaseupgrade start on
78
# version 6 we avoid that bug, while still giving the people using version 1
79
# and 2 an upgrade path that does something.
82
def defaultFeedIconURL():
83
from miro.plat import resources
84
return resources.url("images/feedicon.png")
86
#Dummy class for removing bogus FileItem instances
87
class DropItLikeItsHot(object):
88
__DropMeLikeItsHot = True
89
def __slurp(self, *args, **kwargs):
91
def __getattr__(self, attr):
92
if attr == '__DropMeLikeItsHot':
93
return self.__DropMeLikeItsHot
95
print "DTV: WARNING! Attempt to call '%s' on DropItLikeItsHot instance" % attr
97
traceback.print_stack()
99
__setstate__ = __slurp
101
return "DropMeLikeItsHot"
103
return "DropMeLikeItsHot"
105
class OldDDBObject(object):
109
class OldItem(OldDDBObject):
110
# allOldItems is a hack to get around the fact that old databases can have
111
# items that aren't at the top level. In fact, they can be in fairly
112
# crazy places. See bug #2515. So we need to keep track of the items
113
# when we unpickle the objects.
116
def __setstate__(self, state):
117
(version, data) = state
119
data['pendingManualDL'] = False
120
if not data.has_key('linkNumber'):
121
data['linkNumber'] = 0
125
data['pendingReason'] = ""
128
data['creationTime'] = datetime.now()
131
data['startingDownload'] = False
134
# Older versions of the database allowed Feed Implementations
135
# to act as feeds. If that's the case, change feed attribute
136
# to contain the actual feed.
137
# NOTE: This assumes that the feed object is decoded
138
# before its items. That appears to be generally true
139
if not issubclass(self.feed.__class__, OldDDBObject):
141
self.feed = self.feed.ufeed
142
except (SystemExit, KeyboardInterrupt):
145
self.__class__ = DropItLikeItsHot
146
if self.__class__ is OldFileItem:
147
self.__class__ = DropItLikeItsHot
149
self.iconCache = None
150
if not 'downloadedTime' in data:
151
self.downloadedTime = None
152
OldItem.allOldItems.add(self)
154
class OldFileItem(OldItem):
157
class OldFeed(OldDDBObject):
158
def __setstate__(self,state):
159
(version, data) = state
163
data['thumbURL'] = defaultFeedIconURL()
166
data['lastViewed'] = datetime.min
167
data['unwatched'] = 0
168
data['available'] = 0
171
data['updating'] = False
172
if not data.has_key('initiallyAutoDownloadable'):
173
data['initiallyAutoDownloadable'] = True
175
# This object is useless without a FeedImpl associated with it
176
if not data.has_key('actualFeed'):
177
self.__class__ = DropItLikeItsHot
179
self.iconCache = None
181
class OldFolder(OldDDBObject):
184
class OldHTTPAuthPassword(OldDDBObject):
188
def __setstate__(self, data):
190
if 'expireTime' not in data:
191
self.expireTime = None
193
# Some feeds had invalid updating freq. Catch that error here, so we
194
# don't lose the dabatase when we restore it.
196
self.updateFreq = int(self.updateFreq)
200
class OldScraperFeedImpl(OldFeedImpl):
201
def __setstate__(self,state):
202
(version, data) = state
204
data['updating'] = False
205
data['tempHistory'] = {}
206
OldFeedImpl.__setstate__(self, data)
208
class OldRSSFeedImpl(OldFeedImpl):
209
def __setstate__(self,state):
210
(version, data) = state
212
data['updating'] = False
213
OldFeedImpl.__setstate__(self, data)
215
class OldSearchFeedImpl(OldRSSFeedImpl):
218
class OldSearchDownloadsFeedImpl(OldFeedImpl):
221
class OldDirectoryFeedImpl(OldFeedImpl):
222
def __setstate__(self,state):
223
(version, data) = state
225
data['updating'] = False
226
if not data.has_key('initialUpdate'):
227
data['initialUpdate'] = False
228
OldFeedImpl.__setstate__(self, data)
230
class OldRemoteDownloader(OldDDBObject):
231
def __setstate__(self,state):
232
(version, data) = state
233
self.__dict__ = copy(data)
235
for key in ('startTime', 'endTime', 'filename', 'state',
236
'currentSize', 'totalSize', 'reasonFailed'):
237
self.status[key] = self.__dict__[key]
238
del self.__dict__[key]
239
# force the download daemon to create a new downloader object.
242
class OldChannelGuide(OldDDBObject):
243
def __setstate__(self,state):
244
(version, data) = state
247
self.sawIntro = data['viewed']
248
self.cachedGuideBody = None
249
self.loadedThisSession = False
250
self.cond = threading.Condition()
254
self.cond = threading.Condition()
255
self.loadedThisSession = False
256
if not data.has_key('id'):
257
self.__class__ = DropItLikeItsHot
259
# No need to load a fresh channel guide here.
261
class OldMetainfo(OldDDBObject):
265
'item.Item': OldItem,
266
'item.FileItem': OldFileItem,
267
'feed.Feed': OldFeed,
268
'feed.FeedImpl': OldFeedImpl,
269
'feed.RSSFeedImpl': OldRSSFeedImpl,
270
'feed.ScraperFeedImpl': OldScraperFeedImpl,
271
'feed.SearchFeedImpl': OldSearchFeedImpl,
272
'feed.DirectoryFeedImpl': OldDirectoryFeedImpl,
273
'feed.SearchDownloadsFeedImpl': OldSearchDownloadsFeedImpl,
274
'downloader.HTTPAuthPassword': OldHTTPAuthPassword,
275
'downloader.RemoteDownloader': OldRemoteDownloader,
276
'guide.ChannelGuide': OldChannelGuide,
278
# Drop these classes like they're hot!
280
# YahooSearchFeedImpl is a leftover class that we don't use anymore.
282
# The HTTPDownloader and BTDownloader classes were removed in 0.8.2. The
283
# cleanest way to handle them is to just drop them. If the user still has
284
# these in their database, too bad. BTDownloaders may contain BTDisplay
285
# and BitTorrent.ConvertedMetainfo.ConvertedMetainfo objects, drop those
288
# We use BitTornado now, so drop the metainfo... We should recreate it
291
# DownloaderFactory and StaticTab shouldn't be pickled, but I've seen
292
# databases where it is.
294
# We used to have classes called RSSFeed, ScraperFeed, etc. Now we have
295
# the Feed class which contains a FeedImpl subclass. Since this only
296
# happens on really old databases, we should just drop the old ones.
297
'BitTorrent.ConvertedMetainfo.ConvertedMetainfo': DropItLikeItsHot,
298
'downloader.DownloaderFactory': DropItLikeItsHot,
299
'app.StaticTab': DropItLikeItsHot,
300
'feed.YahooSearchFeedImpl': DropItLikeItsHot,
301
'downloader.BTDownloader': DropItLikeItsHot,
302
'downloader.BTDisplay': DropItLikeItsHot,
303
'downloader.HTTPDownloader': DropItLikeItsHot,
304
'scheduler.ScheduleEvent': DropItLikeItsHot,
305
'feed.UniversalFeed' : DropItLikeItsHot,
306
'feed.RSSFeed': DropItLikeItsHot,
307
'feed.ScraperFeed': DropItLikeItsHot,
308
'feed.SearchFeed': DropItLikeItsHot,
309
'feed.DirectoryFeed': DropItLikeItsHot,
310
'feed.SearchDownloadsFeed': DropItLikeItsHot,
314
class FakeClassUnpickler(pickle.Unpickler):
315
unpickleNormallyWhitelist = [
317
'datetime.timedelta',
319
'miro.feedparser.FeedParserDict',
320
'__builtin__.unicode',
323
def find_class(self, module, name):
324
if module == 'feedparser':
325
# hack to handle the fact that everything is inside the miro
327
module = 'miro.feedparser'
328
fullyQualifiedName = "%s.%s" % (module, name)
329
if fullyQualifiedName in fakeClasses:
330
return fakeClasses[fullyQualifiedName]
331
elif fullyQualifiedName in self.unpickleNormallyWhitelist:
332
return pickle.Unpickler.find_class(self, module, name)
334
raise ValueError("Unrecognized class: %s" % fullyQualifiedName)
337
# We need to define this class for the ItemSchema. In practice we will
338
# always use None instead of one of these objects.
342
######################### STAGE 2 helpers #############################
344
class DDBObjectSchema(ObjectSchema):
346
classString = 'ddb-object'
351
# Unlike the SchemaString in schema.py, this allows binary strings or
353
class SchemaString(SchemaSimpleItem):
354
def validate(self, data):
355
super(SchemaSimpleItem, self).validate(data)
356
self.validateTypes(data, (unicode, str))
358
# Unlike the simple container in schema.py, this allows binary strings
359
class SchemaSimpleContainer(SchemaSimpleItem):
360
"""Allows nested dicts, lists and tuples, however the only thing they can
361
store are simple objects. This currently includes bools, ints, longs,
362
floats, strings, unicode, None, datetime and struct_time objects.
365
def validate(self, data):
366
super(SchemaSimpleContainer, self).validate(data)
367
self.validateTypes(data, (dict, list, tuple))
369
toValidate = LinkedList()
371
if id(data) in self.memory:
374
self.memory.add(id(data))
376
if isinstance(data, list) or isinstance(data, tuple):
378
toValidate.append(item)
379
elif isinstance(data, dict):
380
for key, value in data.items():
381
self.validateTypes(key, [bool, int, long, float, unicode,
382
str, NoneType, datetime, time.struct_time])
383
toValidate.append(value)
385
self.validateTypes(data, [bool, int, long, float, unicode,str,
386
NoneType, datetime, time.struct_time])
388
data = toValidate.pop()
389
except (SystemExit, KeyboardInterrupt):
395
class ItemSchema(DDBObjectSchema):
398
fields = DDBObjectSchema.fields + [
399
('feed', SchemaObject(OldFeed)),
400
('seen', SchemaBool()),
401
('downloaders', SchemaList(SchemaObject(OldRemoteDownloader))),
402
('autoDownloaded', SchemaBool()),
403
('startingDownload', SchemaBool()),
404
('lastDownloadFailed', SchemaBool()),
405
('pendingManualDL', SchemaBool()),
406
('pendingReason', SchemaString()),
407
('entry', SchemaSimpleContainer()),
408
('expired', SchemaBool()),
409
('keep', SchemaBool()),
410
('creationTime', SchemaDateTime()),
411
('linkNumber', SchemaInt(noneOk=True)),
412
('iconCache', SchemaObject(IconCache, noneOk=True)),
413
('downloadedTime', SchemaDateTime(noneOk=True)),
416
class FileItemSchema(ItemSchema):
418
classString = 'file-item'
419
fields = ItemSchema.fields + [
420
('filename', SchemaString()),
423
class FeedSchema(DDBObjectSchema):
426
fields = DDBObjectSchema.fields + [
427
('origURL', SchemaString()),
428
('errorState', SchemaBool()),
429
('initiallyAutoDownloadable', SchemaBool()),
430
('loading', SchemaBool()),
431
('actualFeed', SchemaObject(OldFeedImpl)),
432
('iconCache', SchemaObject(IconCache, noneOk=True)),
435
class FeedImplSchema(ObjectSchema):
437
classString = 'field-impl'
439
('available', SchemaInt()),
440
('unwatched', SchemaInt()),
441
('url', SchemaString()),
442
('ufeed', SchemaObject(OldFeed)),
443
('items', SchemaList(SchemaObject(OldItem))),
444
('title', SchemaString()),
445
('created', SchemaDateTime()),
446
('autoDownloadable', SchemaBool()),
447
('startfrom', SchemaDateTime()),
448
('getEverything', SchemaBool()),
449
('maxNew', SchemaInt()),
450
('fallBehind', SchemaInt()),
451
('expire', SchemaString()),
452
('visible', SchemaBool()),
453
('updating', SchemaBool()),
454
('lastViewed', SchemaDateTime()),
455
('thumbURL', SchemaString()),
456
('updateFreq', SchemaInt()),
457
('expireTime', SchemaTimeDelta(noneOk=True)),
460
class RSSFeedImplSchema(FeedImplSchema):
461
klass = OldRSSFeedImpl
462
classString = 'rss-feed-impl'
463
fields = FeedImplSchema.fields + [
464
('initialHTML', SchemaString(noneOk=True)),
465
('etag', SchemaString(noneOk=True)),
466
('modified', SchemaString(noneOk=True)),
469
class ScraperFeedImplSchema(FeedImplSchema):
470
klass = OldScraperFeedImpl
471
classString = 'scraper-feed-impl'
472
fields = FeedImplSchema.fields + [
473
('initialHTML', SchemaString(noneOk=True)),
474
('initialCharset', SchemaString(noneOk=True)),
475
('linkHistory', SchemaSimpleContainer()),
478
class SearchFeedImplSchema(FeedImplSchema):
479
klass = OldSearchFeedImpl
480
classString = 'search-feed-impl'
481
fields = FeedImplSchema.fields + [
482
('searching', SchemaBool()),
483
('lastEngine', SchemaString()),
484
('lastQuery', SchemaString()),
487
class DirectoryFeedImplSchema(FeedImplSchema):
488
klass = OldDirectoryFeedImpl
489
classString = 'directory-feed-impl'
490
# DirectoryFeedImpl doesn't have any addition fields over FeedImpl
492
class SearchDownloadsFeedImplSchema(FeedImplSchema):
493
klass = OldSearchDownloadsFeedImpl
494
classString = 'search-downloads-feed-impl'
495
# SearchDownloadsFeedImpl doesn't have any addition fields over FeedImpl
497
class RemoteDownloaderSchema(DDBObjectSchema):
498
klass = OldRemoteDownloader
499
classString = 'remote-downloader'
500
fields = DDBObjectSchema.fields + [
501
('url', SchemaString()),
502
('itemList', SchemaList(SchemaObject(OldItem))),
503
('dlid', SchemaString()),
504
('contentType', SchemaString(noneOk=True)),
505
('status', SchemaSimpleContainer()),
508
class HTTPAuthPasswordSchema(DDBObjectSchema):
509
klass = OldHTTPAuthPassword
510
classString = 'http-auth-password'
511
fields = DDBObjectSchema.fields + [
512
('username', SchemaString()),
513
('password', SchemaString()),
514
('host', SchemaString()),
515
('realm', SchemaString()),
516
('path', SchemaString()),
517
('authScheme', SchemaString()),
520
class FolderSchema(DDBObjectSchema):
522
classString = 'folder'
523
fields = DDBObjectSchema.fields + [
524
('feeds', SchemaList(SchemaInt())),
525
('title', SchemaString()),
528
class ChannelGuideSchema(DDBObjectSchema):
529
klass = OldChannelGuide
530
classString = 'channel-guide'
531
fields = DDBObjectSchema.fields + [
532
('sawIntro', SchemaBool()),
533
('cachedGuideBody', SchemaString(noneOk=True)),
534
('loadedThisSession', SchemaBool()),
538
DDBObjectSchema, ItemSchema, FileItemSchema, FeedSchema, FeedImplSchema,
539
RSSFeedImplSchema, ScraperFeedImplSchema, SearchFeedImplSchema,
540
DirectoryFeedImplSchema, SearchDownloadsFeedImplSchema,
541
RemoteDownloaderSchema, HTTPAuthPasswordSchema, FolderSchema,
545
def convertOldDatabase(databasePath):
546
OldItem.allOldItems = set()
547
shutil.copyfile(databasePath, databasePath + '.old')
548
f = open(databasePath, 'rb')
549
p = FakeClassUnpickler(f)
551
if type(data) == types.ListType:
556
(version, objects) = data
557
# Objects used to be stored as (object, object) tuples. Remove the dup
558
objects = [o[0] for o in objects]
559
# drop any top-level DropItLikeItsHot instances
560
objects = [o for o in objects if not hasattr(o, '__DropMeLikeItsHot')]
561
# Set obj.id for any objects missing it
573
# drop any downloaders that are referenced by items
574
def dropItFilter(obj):
575
return not hasattr(obj, '__DropMeLikeItsHot')
576
for i in OldItem.allOldItems:
577
i.downloaders = filter(dropItFilter, i.downloaders)
579
storedatabase.saveObjectList(objects, databasePath,
580
objectSchemas=objectSchemas, version=6)
added
removed
portable/olddatabaseupgrade.py
