1
# Miro - an RSS based video player application
2
# Copyright (C) 2005-2010 Participatory Culture Foundation
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU General Public License for more details.
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18
# In addition, as a special exception, the copyright holders give
19
# permission to link the code of portions of this program with the OpenSSL
22
# You must obey the GNU General Public License in all respects for all of
23
# the code used other than OpenSSL. If you modify file(s) with this
24
# exception, you may extend this exception to your version of the file(s),
25
# but you are not obligated to do so. If you do not wish to do so, delete
26
# this exception statement from your version. If you delete this exception
27
# statement from all source files in the program, then also delete it here.
29
"""schemav79.py -- schema.py, frozen at version 79.
31
This is the magic version where we switched the database style. We keep this
32
schema version around in order to create the tables needed to upgrade
33
databases from version 2.0 (and before).
35
The schema module is responsible for defining what data in the database
38
The goals of this modules are:
40
* Clearly defining which data from DDBObjects gets stored and which doesn't.
41
* Validating that all data we write can be read back in
42
* Making upgrades of the database schema as easy as possible
44
Module-level variables:
45
objectSchemas -- Schemas to use with the current database.
46
VERSION -- Current schema version. If you change the schema you must bump
47
this number and add a function in the databaseupgrade module.
49
Go to the bottom of this file for the current database schema.
54
from types import NoneType
55
from miro.plat.utils import FilenameType
56
from miro.frontendstate import WidgetsFrontendState
58
class ValidationError(Exception):
59
"""Error thrown when we try to save invalid data."""
62
class ValidationWarning(Warning):
63
"""Warning issued when we try to restore invalid data."""
66
class SchemaItem(object):
67
"""SchemaItem represents a single attribute that gets stored on disk.
69
SchemaItem is an abstract class. Subclasses of SchemaItem such as
70
SchemaAttr, SchemaList are used in actual object schemas.
73
noneOk -- specifies if None is a valid value for this attribute
76
def __init__(self, noneOk=False):
79
def validate(self, data):
80
"""Validate that data is a valid value for this SchemaItem.
82
validate is "dumb" when it comes to container types like SchemaList,
83
etc. It only checks that the container is the right type, not its
84
children. This isn't a problem because saveObject() calls
85
validate() recursively on all the data it saves, therefore validate
86
doesn't have to recursively validate things.
91
raise ValidationError("None value is not allowed")
94
def validateType(self, data, correctType):
95
"""Helper function that many subclasses use"""
96
if data is not None and not isinstance(data, correctType):
97
raise ValidationError("%r (type: %s) is not a %s" %
98
(data, type(data), correctType))
100
def validateTypes(self, data, possibleTypes):
103
for t in possibleTypes:
104
if isinstance(data, t):
106
raise ValidationError("%r (type: %s) is not any of: %s" %
107
(data, type(data), possibleTypes))
109
class SchemaSimpleItem(SchemaItem):
110
"""Base class for SchemaItems for simple python types."""
112
class SchemaBool(SchemaSimpleItem):
113
def validate(self, data):
114
super(SchemaSimpleItem, self).validate(data)
115
self.validateType(data, bool)
117
class SchemaFloat(SchemaSimpleItem):
118
def validate(self, data):
119
super(SchemaSimpleItem, self).validate(data)
120
self.validateType(data, float)
122
class SchemaString(SchemaSimpleItem):
123
def validate(self, data):
124
super(SchemaSimpleItem, self).validate(data)
125
self.validateType(data, unicode)
127
class SchemaBinary(SchemaSimpleItem):
128
def validate(self, data):
129
super(SchemaSimpleItem, self).validate(data)
130
self.validateType(data, str)
132
class SchemaFilename(SchemaSimpleItem):
133
def validate(self, data):
134
super(SchemaSimpleItem, self).validate(data)
135
self.validateType(data, FilenameType)
137
class SchemaURL(SchemaSimpleItem):
138
def validate(self, data):
139
super(SchemaSimpleItem, self).validate(data)
140
self.validateType(data, unicode)
144
except UnicodeEncodeError:
145
ValidationError(u"URL (%s) is not ASCII" % data)
147
class SchemaInt(SchemaSimpleItem):
148
def validate(self, data):
149
super(SchemaSimpleItem, self).validate(data)
150
self.validateTypes(data, [int, long])
152
class SchemaDateTime(SchemaSimpleItem):
153
def validate(self, data):
154
super(SchemaSimpleItem, self).validate(data)
155
self.validateType(data, datetime.datetime)
157
class SchemaTimeDelta(SchemaSimpleItem):
158
def validate(self, data):
159
super(SchemaSimpleItem, self).validate(data)
160
self.validateType(data, datetime.timedelta)
162
class SchemaReprContainer(SchemaItem):
163
"""SchemaItem saved using repr() to save nested lists, dicts and tuples
164
that store simple types. The look is similar to JSON, but supports a
165
couple different things, for example unicode and str values are distinct.
166
The types that we support are bool, int, long, float, unicode, None and
167
datetime. Dictionary keys can also be byte strings (AKA str types)
170
VALID_TYPES = [bool, int, long, float, unicode, NoneType,
171
datetime.datetime, time.struct_time]
173
VALID_KEY_TYPES = VALID_TYPES + [str]
175
def validate(self, data):
177
# let the super class handle the noneOkay attribute
178
super(self, SchemaReprContainer).validate(data)
184
obj = to_validate.pop()
185
if id(obj) in memory:
190
if isinstance(obj, list) or isinstance(obj, tuple):
192
to_validate.append(item)
193
elif isinstance(obj, dict):
194
for key, value in obj.items():
195
self.validateTypes(key, self.VALID_KEY_TYPES)
196
to_validate.append(value)
198
self.validateTypes(obj, self.VALID_TYPES)
200
class SchemaList(SchemaReprContainer):
201
"""Special case of SchemaReprContainer that stores a simple list
203
All values must have the same type
205
def __init__(self, childSchema, noneOk=False):
206
super(SchemaList, self).__init__(noneOk)
207
self.childSchema = childSchema
209
def validate(self, data):
211
super(SchemaList, self).validate(data)
213
self.validateType(data, list)
214
for i, value in enumerate(data):
216
self.childSchema.validate(value)
217
except ValidationError:
218
raise ValidationError("%r (index: %s) has the wrong type" %
221
class SchemaDict(SchemaReprContainer):
222
"""Special case of SchemaReprContainer that stores a simple dict
224
All keys and values must have the same type.
227
def __init__(self, keySchema, valueSchema, noneOk=False):
228
super(SchemaDict, self).__init__(noneOk)
229
self.keySchema = keySchema
230
self.valueSchema = valueSchema
232
def validate(self, data):
234
super(SchemaDict, self).validate(data)
236
self.validateType(data, dict)
237
for key, value in data.items():
239
self.keySchema.validate(key)
240
except ValidationError:
241
raise ValidationError("key %r has the wrong type" % key)
243
self.valueSchema.validate(value)
244
except ValidationError:
245
raise ValidationError("value %r (key: %r) has the wrong type"
248
class SchemaStatusContainer(SchemaReprContainer):
249
"""Version of SchemaReprContainer that stores the status dict for
250
RemoteDownloaders. It allows some values to be byte strings rather than
255
def validate(self, data):
256
binary_fields = self._binary_fields()
257
self.validateType(data, dict)
258
for key, value in data.items():
259
self.validateTypes(key, [bool, int, long, float, unicode,
260
str, NoneType, datetime.datetime, time.struct_time])
261
if key not in binary_fields:
262
self.validateTypes(value, [bool, int, long, float, unicode,
263
NoneType, datetime.datetime, time.struct_time])
265
self.validateType(value, str)
267
def _binary_fields(self):
268
if FilenameType == unicode:
269
return ('metainfo', 'fastResumeData')
271
return ('channelName', 'shortFilename', 'filename', 'metainfo',
274
class SchemaObject(SchemaItem):
275
def __init__(self, klass, noneOk=False):
276
super(SchemaObject, self).__init__(noneOk)
279
def validate(self, data):
280
super(SchemaObject, self).validate(data)
281
self.validateType(data, self.klass)
283
class ObjectSchema(object):
284
"""The schema to save/restore an object with. Object schema isn't a
285
SchemaItem, it's the schema for an entire object.
289
klass -- the python class
290
classString -- a human readable string that represents objectClass
291
fields -- list of (name, SchemaItem) pairs. One item for each attribute
292
that should be stored to disk.
294
Note: klass is always None in this module. We don't need it for what we
295
use the schema for, and we want to be able to delete old classes
299
class DDBObjectSchema(ObjectSchema):
301
classString = 'ddb-object'
306
class IconCacheSchema (DDBObjectSchema):
308
classString = 'icon-cache'
309
fields = DDBObjectSchema.fields + [
310
('etag', SchemaString(noneOk=True)),
311
('modified', SchemaString(noneOk=True)),
312
('filename', SchemaFilename(noneOk=True)),
313
('url', SchemaURL(noneOk=True)),
316
class ItemSchema(DDBObjectSchema):
319
fields = DDBObjectSchema.fields + [
320
('feed_id', SchemaInt(noneOk=True)),
321
('downloader_id', SchemaInt(noneOk=True)),
322
('parent_id', SchemaInt(noneOk=True)),
323
('seen', SchemaBool()),
324
('autoDownloaded', SchemaBool()),
325
('pendingManualDL', SchemaBool()),
326
('pendingReason', SchemaString()),
327
('title', SchemaString()),
328
('expired', SchemaBool()),
329
('keep', SchemaBool()),
330
('creationTime', SchemaDateTime()),
331
('linkNumber', SchemaInt(noneOk=True)),
332
('icon_cache_id', SchemaInt(noneOk=True)),
333
('downloadedTime', SchemaDateTime(noneOk=True)),
334
('watchedTime', SchemaDateTime(noneOk=True)),
335
('isContainerItem', SchemaBool(noneOk=True)),
336
('videoFilename', SchemaFilename()),
337
('isVideo', SchemaBool()),
338
('releaseDateObj', SchemaDateTime()),
339
('eligibleForAutoDownload', SchemaBool()),
340
('duration', SchemaInt(noneOk=True)),
341
('screenshot', SchemaFilename(noneOk=True)),
342
('resumeTime', SchemaInt()),
343
('channelTitle', SchemaString(noneOk=True)),
344
('license', SchemaString(noneOk=True)),
345
('rss_id', SchemaString(noneOk=True)),
346
('thumbnail_url', SchemaURL(noneOk=True)),
347
('entry_title', SchemaString(noneOk=True)),
348
('raw_descrption', SchemaString(noneOk=False)),
349
('link', SchemaURL(noneOk=False)),
350
('payment_link', SchemaURL(noneOk=False)),
351
('comments_link', SchemaURL(noneOk=False)),
352
('url', SchemaURL(noneOk=False)),
353
('enclosure_size', SchemaInt(noneOk=True)),
354
('enclosure_type', SchemaString(noneOk=True)),
355
('enclosure_format', SchemaString(noneOk=True)),
356
('feedparser_output', SchemaReprContainer()),
359
class FileItemSchema(ItemSchema):
361
classString = 'file-item'
362
fields = ItemSchema.fields + [
363
('filename', SchemaFilename()),
364
('deleted', SchemaBool()),
365
('shortFilename', SchemaFilename(noneOk=True)),
366
('offsetPath', SchemaFilename(noneOk=True)),
369
class FeedSchema(DDBObjectSchema):
372
fields = DDBObjectSchema.fields + [
373
('origURL', SchemaURL()),
374
('baseTitle', SchemaString(noneOk=True)),
375
('errorState', SchemaBool()),
376
('loading', SchemaBool()),
377
('feed_impl_id', SchemaInt()),
378
('icon_cache_id', SchemaInt(noneOk=True)),
379
('folder_id', SchemaInt(noneOk=True)),
380
('searchTerm', SchemaString(noneOk=True)),
381
('userTitle', SchemaString(noneOk=True)),
382
('autoDownloadable', SchemaBool()),
383
('getEverything', SchemaBool()),
384
('maxNew', SchemaInt()),
385
('maxOldItems', SchemaInt(noneOk=True)),
386
('fallBehind', SchemaInt()),
387
('expire', SchemaString()),
388
('expireTime', SchemaTimeDelta(noneOk=True)),
389
('section', SchemaString()),
390
('visible', SchemaBool()),
393
class FeedImplSchema(DDBObjectSchema):
395
classString = 'field-impl'
396
fields = DDBObjectSchema.fields + [
397
('url', SchemaURL()),
398
('ufeed_id', SchemaInt()),
399
('title', SchemaString(noneOk=True)),
400
('created', SchemaDateTime()),
401
('lastViewed', SchemaDateTime()),
402
('thumbURL', SchemaURL(noneOk=True)),
403
('updateFreq', SchemaInt()),
404
('initialUpdate', SchemaBool()),
407
class RSSFeedImplSchema(FeedImplSchema):
409
classString = 'rss-feed-impl'
410
fields = FeedImplSchema.fields + [
411
('initialHTML', SchemaBinary(noneOk=True)),
412
('etag', SchemaString(noneOk=True)),
413
('modified', SchemaString(noneOk=True)),
416
class RSSMultiFeedImplSchema(FeedImplSchema):
418
classString = 'rss-multi-feed-impl'
419
fields = FeedImplSchema.fields + [
420
('etag', SchemaDict(SchemaString(),SchemaString(noneOk=True))),
421
('modified', SchemaDict(SchemaString(),SchemaString(noneOk=True))),
422
('query', SchemaString(noneOk=True)),
425
class ScraperFeedImplSchema(FeedImplSchema):
427
classString = 'scraper-feed-impl'
428
fields = FeedImplSchema.fields + [
429
('initialHTML', SchemaBinary(noneOk=True)),
430
('initialCharset', SchemaString(noneOk=True)),
431
('linkHistory', SchemaReprContainer()),
434
class SearchFeedImplSchema(RSSMultiFeedImplSchema):
436
classString = 'search-feed-impl'
437
fields = RSSMultiFeedImplSchema.fields + [
438
('searching', SchemaBool()),
439
('lastEngine', SchemaString()),
440
('lastQuery', SchemaString()),
443
class DirectoryWatchFeedImplSchema(FeedImplSchema):
445
classString = 'directory-watch-feed-impl'
446
fields = FeedImplSchema.fields + [
447
('firstUpdate', SchemaBool()),
448
('dir', SchemaFilename(noneOk=True)),
451
class DirectoryFeedImplSchema(FeedImplSchema):
453
classString = 'directory-feed-impl'
454
# DirectoryFeedImpl doesn't have any addition fields over FeedImpl
456
class SearchDownloadsFeedImplSchema(FeedImplSchema):
458
classString = 'search-downloads-feed-impl'
459
# SearchDownloadsFeedImpl doesn't have any addition fields over FeedImpl
461
class ManualFeedImplSchema(FeedImplSchema):
463
classString = 'manual-feed-impl'
464
# no addition fields over FeedImplSchema
466
class SingleFeedImplSchema(FeedImplSchema):
468
classString = 'single-feed-impl'
469
# no addition fields over FeedImplSchema
471
class RemoteDownloaderSchema(DDBObjectSchema):
473
classString = 'remote-downloader'
474
fields = DDBObjectSchema.fields + [
475
('url', SchemaURL()),
476
('origURL', SchemaURL()),
477
('dlid', SchemaString()),
478
('contentType', SchemaString(noneOk=True)),
479
('channelName', SchemaFilename(noneOk=True)),
480
('status', SchemaStatusContainer()),
481
('manualUpload', SchemaBool()),
484
class HTTPAuthPasswordSchema(DDBObjectSchema):
486
classString = 'http-auth-password'
487
fields = DDBObjectSchema.fields + [
488
('username', SchemaString()),
489
('password', SchemaString()),
490
('host', SchemaString()),
491
('realm', SchemaString()),
492
('path', SchemaString()),
493
('authScheme', SchemaString()),
496
class ChannelFolderSchema(DDBObjectSchema):
498
classString = 'channel-folder'
499
fields = DDBObjectSchema.fields + [
500
('expanded', SchemaBool()),
501
('title', SchemaString()),
502
('section', SchemaString()),
505
class PlaylistFolderSchema(DDBObjectSchema):
507
classString = 'playlist-folder'
508
fields = DDBObjectSchema.fields + [
509
('expanded', SchemaBool()),
510
('title', SchemaString()),
511
('item_ids', SchemaList(SchemaInt())),
514
class PlaylistSchema(DDBObjectSchema):
516
classString = 'playlist'
517
fields = DDBObjectSchema.fields + [
518
('title', SchemaString()),
519
('item_ids', SchemaList(SchemaInt())),
520
('folder_id', SchemaInt(noneOk=True)),
523
class TabOrderSchema(DDBObjectSchema):
525
classString = 'taborder-order'
526
fields = DDBObjectSchema.fields + [
527
('type', SchemaString()),
528
('tab_ids', SchemaList(SchemaInt())),
531
class ChannelGuideSchema(DDBObjectSchema):
533
classString = 'channel-guide'
534
fields = DDBObjectSchema.fields + [
535
('url', SchemaURL(noneOk=True)),
536
('allowedURLs', SchemaList(SchemaURL())),
537
('updated_url', SchemaURL(noneOk=True)),
538
('favicon', SchemaURL(noneOk=True)),
539
('title', SchemaString(noneOk=True)),
540
('userTitle', SchemaString(noneOk=True)),
541
('icon_cache_id', SchemaInt(noneOk=True)),
542
('firstTime', SchemaBool()),
545
class ThemeHistorySchema(DDBObjectSchema):
547
classString = 'theme-history'
548
fields = DDBObjectSchema.fields + [
549
('lastTheme', SchemaString(noneOk=True)),
550
('pastThemes', SchemaList(SchemaString(noneOk=True), noneOk=False)),
553
class WidgetsFrontendStateSchema(DDBObjectSchema):
555
classString = 'widgets-frontend-state'
556
fields = DDBObjectSchema.fields + [
557
('list_view_displays', SchemaList(SchemaBinary())),
562
IconCacheSchema, ItemSchema, FileItemSchema, FeedSchema,
563
FeedImplSchema, RSSFeedImplSchema, RSSMultiFeedImplSchema, ScraperFeedImplSchema,
564
SearchFeedImplSchema, DirectoryFeedImplSchema, DirectoryWatchFeedImplSchema,
565
SearchDownloadsFeedImplSchema, RemoteDownloaderSchema,
566
HTTPAuthPasswordSchema, ChannelGuideSchema, ManualFeedImplSchema, SingleFeedImplSchema,
567
PlaylistSchema, ChannelFolderSchema, PlaylistFolderSchema,
568
TabOrderSchema, ThemeHistorySchema, WidgetsFrontendStateSchema