3
# Miro - an RSS based video player application
4
# Copyright (C) 2005-2010 Participatory Culture Foundation
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20
# In addition, as a special exception, the copyright holders give
21
# permission to link the code of portions of this program with the OpenSSL
24
# You must obey the GNU General Public License in all respects for all of
25
# the code used other than OpenSSL. If you modify file(s) with this
26
# exception, you may extend this exception to your version of the file(s),
27
# but you are not obligated to do so. If you do not wish to do so, delete
28
# this exception statement from your version. If you delete this exception
29
# statement from all source files in the program, then also delete it here.
36
from miro import signals
38
class DatabaseException(Exception):
39
"""Superclass for classes that subclass Exception and are all
44
class DatabaseConstraintError(DatabaseException):
45
"""Raised when a DDBObject fails its constraint checking during
50
class DatabaseConsistencyError(DatabaseException):
51
"""Raised when the database encounters an internal consistency
56
class DatabaseThreadError(DatabaseException):
57
"""Raised when the database encounters an internal consistency
62
class DatabaseStandardError(StandardError):
65
class DatabaseVersionError(DatabaseStandardError):
66
"""Raised when an attempt is made to restore a database newer than
71
class ObjectNotFoundError(DatabaseStandardError):
72
"""Raised when an attempt is made to lookup an object that doesn't
77
class TooManyObjects(DatabaseStandardError):
78
"""Raised when an attempt is made to lookup a singleton and
79
multiple rows match the query.
83
class NotRootDBError(DatabaseStandardError):
84
"""Raised when an attempt is made to call a function that's only
85
allowed to be called from the root database.
89
class NoValue(object):
90
"""Used as a dummy value so that "None" can be treated as a valid
95
# begin* and end* no longer actually lock the database. Instead
96
# confirm_db_thread prints a warning if it's run from any thread that
97
# isn't the main thread. This can be removed from releases for speed
101
def set_thread(thread):
103
if event_thread is None:
104
event_thread = thread
106
def confirm_db_thread():
107
if event_thread is None or event_thread != threading.currentThread():
108
if event_thread is None:
109
error_string = "Database event thread not set"
111
error_string = "Database called from %s" % threading.currentThread()
112
traceback.print_stack()
113
raise DatabaseThreadError, error_string
116
def __init__(self, klass, where, values, order_by, joins, limit):
120
self.order_by = order_by
125
return app.db.query(self.klass, self.where, self.values,
126
self.order_by, self.joins, self.limit)
129
return app.db.query_count(self.klass, self.where, self.values,
130
self.joins, self.limit)
132
def get_singleton(self):
134
if len(results) == 1:
136
elif len(results) == 0:
137
raise ObjectNotFoundError("Can't find singleton")
139
raise TooManyObjects("Too many results returned")
141
def make_tracker(self):
142
if self.limit is not None:
143
raise ValueError("tracking views with limits not supported")
144
return ViewTracker(self.klass, self.where, self.values, self.joins)
146
class ViewTrackerManager(object):
148
# maps table_name to trackers
149
self.table_to_tracker = {}
150
# maps joined tables to trackers
151
self.joined_table_to_tracker = {}
153
def trackers_for_table(self, table_name):
155
return self.table_to_tracker[table_name]
157
self.table_to_tracker[table_name] = set()
158
return self.table_to_tracker[table_name]
160
def trackers_for_ddb_class(self, klass):
161
return self.trackers_for_table(app.db.table_name(klass))
163
def update_view_trackers(self, obj):
164
"""Update view trackers based on an object change."""
166
for tracker in self.trackers_for_ddb_class(obj.__class__):
167
tracker.object_changed(obj)
169
def bulk_update_view_trackers(self, table_name):
170
for tracker in self.trackers_for_table(table_name):
171
tracker.check_all_objects()
173
def bulk_remove_from_view_trackers(self, table_name, objects):
174
for tracker in self.trackers_for_table(table_name):
175
tracker.remove_objects(objects)
177
def remove_from_view_trackers(self, obj):
178
"""Update view trackers based on an object change."""
180
for tracker in self.trackers_for_ddb_class(obj.__class__):
181
tracker.remove_object(obj)
183
class ViewTracker(signals.SignalEmitter):
184
def __init__(self, klass, where, values, joins):
185
signals.SignalEmitter.__init__(self, 'added', 'removed', 'changed')
188
if isinstance(values, list):
189
raise TypeError("values must be a tuple")
192
self.current_ids = set(app.db.query_ids(klass, where, values,
194
self.table_name = app.db.table_name(klass)
195
vt_manager = app.view_tracker_manager
196
vt_manager.trackers_for_table(self.table_name).add(self)
199
vt_manager = app.view_tracker_manager
200
vt_manager.trackers_for_table(self.table_name).discard(self)
202
def _obj_in_view(self, obj):
203
where = '%s.id = ?' % (self.table_name,)
205
where += ' AND (%s)' % (self.where,)
207
values = (obj.id,) + self.values
208
return app.db.query_count(self.klass, where, values, self.joins) > 0
210
def object_changed(self, obj):
211
self.check_object(obj)
213
def remove_object(self, obj):
214
if obj.id in self.current_ids:
215
self.current_ids.remove(obj.id)
216
self.emit('removed', obj)
218
def remove_objects(self, objects):
219
object_map = dict((o.id, o) for o in objects)
220
object_ids = set(object_map.keys())
221
for removed_id in self.current_ids.intersection(object_ids):
222
self.current_ids.remove(removed_id)
223
self.emit('removed', object_map[removed_id])
225
def check_object(self, obj):
226
before = (obj.id in self.current_ids)
227
now = self._obj_in_view(obj)
228
if before and not now:
229
self.current_ids.remove(obj.id)
230
self.emit('removed', obj)
231
elif now and not before:
232
self.current_ids.add(obj.id)
233
self.emit('added', obj)
235
self.emit('changed', obj)
237
def check_all_objects(self):
238
new_ids = set(app.db.query_ids(self.klass, self.where,
239
self.values, joins=self.joins))
240
old_ids = self.current_ids
241
self.current_ids = new_ids
242
for id_ in new_ids.difference(old_ids):
243
self.emit('added', app.db.get_obj_by_id(id_))
244
for id_ in old_ids.difference(new_ids):
245
self.emit('removed', app.db.get_obj_by_id(id_))
246
for id_ in old_ids.intersection(new_ids):
247
# XXX this hits all the IDs, but there doesn't seem to be
248
# a way to check if the objects have actually been
249
# changed. luckily, this isn't called very often.
250
self.emit('changed', app.db.get_obj_by_id(id_))
253
return len(self.current_ids)
255
class BulkSQLManager(object):
260
self.pending_inserts = set()
264
raise ValueError("BulkSQLManager.start() called twice")
269
raise ValueError("BulkSQLManager.finish() called twice")
275
to_insert = self.to_insert
276
to_remove = self.to_remove
279
self._commit_sql(to_insert, to_remove)
280
self._update_view_trackers(to_insert, to_remove)
281
if len(self.to_insert) == len(self.to_remove) == 0:
283
# inside _commit_sql() or _update_view_trackers(), we were
284
# asked to insert or remove more items, repeat the
287
raise AssertionError("Called _commit_sql 100 times and still "
288
"have items to commit. Are we in a circular loop?")
291
self.pending_inserts = set()
293
def _commit_sql(self, to_insert, to_remove):
294
for table_name, objects in to_insert.items():
295
logging.debug('bulk insert: %s %s', table_name, len(objects))
296
app.db.bulk_insert(objects)
298
obj.inserted_into_db()
300
for table_name, objects in to_remove.items():
301
logging.debug('bulk remove: %s %s', table_name, len(objects))
302
app.db.bulk_remove(objects)
304
obj.removed_from_db()
306
def _update_view_trackers(self, to_insert, to_remove):
307
for table_name in to_insert:
308
app.view_tracker_manager.bulk_update_view_trackers(table_name)
310
for table_name, objects in to_remove.items():
311
if table_name in to_insert:
312
# already updated the view above
314
app.view_tracker_manager.bulk_remove_from_view_trackers(
317
def add_insert(self, obj):
318
table_name = app.db.table_name(obj.__class__)
320
inserts_for_table = self.to_insert[table_name]
322
inserts_for_table = []
323
self.to_insert[table_name] = inserts_for_table
324
inserts_for_table.append(obj)
325
self.pending_inserts.add(obj)
327
def will_insert(self, obj):
328
return obj in self.pending_inserts
330
def add_remove(self, obj):
331
table_name = app.db.table_name(obj.__class__)
332
if self.will_insert(obj):
333
self.to_insert[table_name].remove(obj)
334
self.pending_inserts.remove(obj)
337
removes_for_table = self.to_remove[table_name]
339
removes_for_table = []
340
self.to_remove[table_name] = removes_for_table
341
removes_for_table.append(obj)
343
class AttributeUpdateTracker(object):
344
"""Used by DDBObject to track changes to attributes."""
346
def __init__(self, name):
349
# Simple implementation of the python descriptor protocol. We
350
# just want to update changed_attributes when attributes are set.
352
def __get__(self, instance, owner):
354
return instance.__dict__[self.name]
356
raise AttributeError(self.name)
357
except AttributeError:
359
raise AttributeError(
360
"Can't access '%s' as a class attribute" % self.name)
364
def __set__(self, instance, value):
365
if instance.__dict__.get(self.name, "BOGUS VALUE FOO") != value:
366
instance.changed_attributes.add(self.name)
367
instance.__dict__[self.name] = value
369
class DDBObject(signals.SignalEmitter):
370
"""Dynamic Database object
372
#The last ID used in this class
375
def __init__(self, *args, **kwargs):
376
self.in_db_init = True
377
signals.SignalEmitter.__init__(self, 'removed')
378
self.changed_attributes = set()
380
if len(args) == 0 and kwargs.keys() == ['restored_data']:
386
self.__dict__.update(kwargs['restored_data'])
387
app.db.remember_object(self)
388
self.setup_restored()
389
# handle setup_restored() calling remove()
390
if not self.id_exists():
393
self.id = DDBObject.lastID = DDBObject.lastID + 1
394
# call remember_object so that id_exists will return True
395
# when setup_new() is being run
396
app.db.remember_object(self)
397
self.setup_new(*args, **kwargs)
398
# handle setup_new() calling remove()
399
if not self.id_exists():
402
self.in_db_init = False
405
self._insert_into_db()
407
def _insert_into_db(self):
408
if not app.bulk_sql_manager.active:
409
app.db.insert_obj(self)
410
self.inserted_into_db()
411
app.view_tracker_manager.update_view_trackers(self)
413
app.bulk_sql_manager.add_insert(self)
415
def inserted_into_db(self):
416
self.check_constraints()
420
def make_view(cls, where=None, values=None, order_by=None, joins=None,
424
return View(cls, where, values, order_by, joins, limit)
427
def get_by_id(cls, id_):
429
# try memory first before going to sqlite.
430
obj = app.db.get_obj_by_id(id_)
431
if app.db.object_from_class_table(obj, cls):
434
raise ObjectNotFoundError(id_)
436
return cls.make_view('id=?', (id_,)).get_singleton()
439
def delete(cls, where, values=None):
440
return app.db.delete(cls, where, values)
443
def select(cls, columns, where=None, values=None, convert=True):
444
return app.db.select(cls, columns, where, values, convert)
447
"""Initialize a newly created object."""
450
def setup_restored(self):
451
"""Initialize an object restored from disk."""
454
def on_db_insert(self):
455
"""Called after an object has been inserted into the db."""
459
def track_attribute_changes(cls, name):
460
"""Set up tracking when attributes get set.
462
Call this on a DDBObject subclass to track changes to certain
463
attributes. Each DDBObject has a changed_attributes set,
464
which contains the attributes that have changed.
466
This is used by the SQLite storage layer to track which
467
attributes are changed between SQL UPDATE statements.
471
>> MyDDBObjectSubclass.track_attribute_changes('foo')
472
>> MyDDBObjectSubclass.track_attribute_changes('bar')
473
>>> obj = MyDDBObjectSubclass()
474
>>> print obj.changed_attributes
476
>> obj.foo = obj.bar = obj.baz = 3
477
>>> print obj.changed_attributes
480
# The AttributeUpdateTracker class does all the work
481
setattr(cls, name, AttributeUpdateTracker(name))
483
def reset_changed_attributes(self):
484
self.changed_attributes = set()
487
"""Returns unique integer assocaited with this object
493
self.get_by_id(self.id)
494
except ObjectNotFoundError:
500
"""Call this after you've removed all references to the object
502
if not app.bulk_sql_manager.active:
503
app.db.remove_obj(self)
504
self.removed_from_db()
505
app.view_tracker_manager.remove_from_view_trackers(self)
507
app.bulk_sql_manager.add_remove(self)
509
def removed_from_db(self):
512
def confirm_db_thread(self):
513
"""Call this before you grab data from an object
517
view.confirm_db_thread()
522
def check_constraints(self):
523
"""Subclasses can override this method to do constraint
524
checking before they get saved to disk. They should raise a
525
DatabaseConstraintError on problems.
529
def signal_change(self, needs_save=True):
530
"""Call this after you change the object
533
# signal_change called while we were setting up a object,
536
if not self.id_exists():
537
msg = ("signal_change() called on non-existant object (id is %s)"
539
raise DatabaseConstraintError, msg
540
self.on_signal_change()
541
self.check_constraints()
542
if app.bulk_sql_manager.will_insert(self):
543
# Don't need to send an UPDATE SQL command, or check the
544
# view trackers in this case. Both will be done when the
545
# BulkSQLManager.finish() is called.
548
app.db.update_obj(self)
549
app.view_tracker_manager.update_view_trackers(self)
551
def on_signal_change(self):
554
def update_last_id():
555
DDBObject.lastID = app.db.get_last_id()
557
def setup_managers():
558
app.view_tracker_manager = ViewTrackerManager()
559
app.bulk_sql_manager = BulkSQLManager()
added
removed
lib/database.py
