« back to all changes in this revision
Viewing changes to _zeitgeist/engine/engine.py
- browse files at revision 3
- compare with another revision
- download diff
- download tarball
- view history from revision 3
- Committer: Bazaar Package Importer
- Author(s): Siegfried-Angel Gevatter Pujals
- Date: 2009-08-17 00:12:51 UTC
- mfrom: (1.1.1 upstream)
- Revision ID: james.westby@ubuntu.com-20090817001251-lfpvqpxxf9bxefzu
Tags: 0.2.1-0ubuntu1
* New upstream release.
- Update dependencies.
- Update dependencies.
- files added:
- NEWS
- doc/dbus/source/.static
- files removed:
- _zeitgeist/engine/base.py
- files modified:
- Makefile.in
added
removed
_zeitgeist/engine/engine.py
1
# -.- encoding: utf-8 -.-
2
3
# Zeitgeist
4
#
5
# Copyright © 2009 Seif Lotfy <seif@lotfy.com>
6
# Copyright © 2009 Siegfried-Angel Gevatter Pujals <rainct@ubuntu.com>
7
# Copyright © 2009 Natan Yellin <aantny@gmail.com>
8
# Copyright © 2009 Mikkel Kamstrup Erlandsen <mikkel.kamstrup@gmail.com>
9
# Copyright © 2009 Markus Korn <thekorn@gmx.de>
10
#
11
# This program is free software: you can redistribute it and/or modify
12
# it under the terms of the GNU Lesser General Public License as published by
13
# the Free Software Foundation, either version 3 of the License, or
14
# (at your option) any later version.
15
#
16
# This program is distributed in the hope that it will be useful,
17
# but WITHOUT ANY WARRANTY; without even the implied warranty of
18
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19
# GNU Lesser General Public License for more details.
20
#
21
# You should have received a copy of the GNU Lesser General Public License
22
# along with this program. If not, see <http://www.gnu.org/licenses/>.
23
24
import time
25
import sys
26
import os
27
import gettext
28
import gobject
29
import logging
30
from xdg import BaseDirectory
31
from xdg.DesktopEntry import DesktopEntry
32
try:
33
import pysqlite2.dbapi2 as sqlite3 # Storm prefers this module
34
except ImportError:
35
import sqlite3
36
37
from _zeitgeist.engine.base import *
38
from _zeitgeist.lrucache import LRUCache
39
from zeitgeist.dbusutils import EventDict
40
41
logging.basicConfig(level=logging.DEBUG)
42
log = logging.getLogger("zeitgeist.engine")
43
44
class ZeitgeistEngine(gobject.GObject):
45
46
ALLOWED_FILTER_KEYS = set(["name", "uri", "tags", "mimetypes",
47
"source", "content", "application", "bookmarked"])
48
49
def __init__(self, storm_store):
50
51
gobject.GObject.__init__(self)
52
53
assert storm_store is not None
54
self.store = storm_store
55
self._apps = set()
56
self._last_time_from_app = {}
57
self._applications = LRUCache(10)
58
59
'''
60
path = BaseDirectory.save_data_path("zeitgeist")
61
database = os.path.join(path, "zeitgeist.sqlite")
62
self.connection = self._get_database(database)
63
self.cursor = self.connection.cursor()
64
'''
65
66
def _get_ids(self, uri, content, source):
67
uri_id = URI.lookup_or_create(uri).id if uri else None
68
content_id = Content.lookup_or_create(content).id if content else None
69
source_id = Source.lookup_or_create(source).id if source else None
70
return uri_id, content_id, source_id
71
72
def _get_item(self, id, content_id, source_id, text, origin=None, mimetype=None, icon=None):
73
self._insert_event(id, content_id, source_id, text, origin, mimetype, icon)
74
return self.store.find(Item, Item.id == id)
75
76
def _insert_event(self, id, content_id, source_id, text, origin=None, mimetype=None, icon=None):
77
try:
78
self.store.execute("""
79
INSERT INTO Item
80
(id, content_id, source_id, text, origin, mimetype, icon)
81
VALUES (?,?,?,?,?,?,?)""",
82
(id, content_id, source_id, text, origin, mimetype, icon),
83
noresult=True)
84
except Exception:
85
self.store.execute("""
86
UPDATE Item SET
87
content_id=?, source_id=?, text=?, origin=?,
88
mimetype=?, icon=? WHERE id=?""",
89
(content_id, source_id, text, origin, mimetype, icon, id),
90
noresult=True)
91
92
def insert_event(self, ritem, commit=True, force=False):
93
"""
94
Inserts an item into the database. Returns a positive number on success,
95
zero otherwise (for example, if the item already is in the
96
database). In case the positive number is 1, the inserted event is new,
97
in case it's 2 the event already existed and was updated (this only
98
happens when `force' is True).
99
"""
100
101
# check for required items and make sure all items have the correct type
102
ritem = EventDict.check_missing_items(ritem)
103
104
# FIXME: uri, content, source are now required items, the statement above
105
# will raise a KeyError if they are not there. What about mimetype?
106
# and why are we printing a warning and returning False here instead of raising
107
# an error at all? - Markus Korn
108
if not ritem["uri"].strip():
109
log.warning("Discarding item without a URI: %s" % ritem)
110
return False
111
if not ritem["content"].strip():
112
log.warning("Discarding item without a Content type: %s" % ritem)
113
return False
114
if not ritem["source"].strip():
115
log.warning("Discarding item without a Source type: %s" % ritem)
116
return False
117
if not ritem["mimetype"].strip():
118
log.warning("Discarding item without a mimetype: %s" % ritem)
119
return False
120
121
# Get the IDs for the URI, the content and the source
122
uri_id, content_id, source_id = self._get_ids(ritem["uri"],
123
ritem["content"], ritem["source"])
124
125
# Generate the URI for the event
126
event_uri = "zeitgeist://event/%s/%%s/%s#%d" % (ritem["use"],
127
ritem["timestamp"], uri_id)
128
129
# Check whether the events is already in the database. If so,
130
# don't do anything. If it isn't there yet, we proceed with the
131
# process. Except if `force' is true, then we always proceed.
132
event_exists = bool(self.store.execute(
133
"SELECT id FROM uri WHERE value = ?", (event_uri,)).get_one())
134
if not force and event_exists:
135
return 0
136
137
# Insert or update the item
138
item = self._get_item(uri_id, content_id, source_id, ritem["text"],
139
ritem["origin"], ritem["mimetype"], ritem["icon"])
140
141
# Insert or update the tags
142
for tag in (tag.strip() for tag in ritem["tags"].split(",") if tag):
143
anno_uri = "zeitgeist://tag/%s" % tag
144
anno_id, discard, discard = self._get_ids(anno_uri, None, None)
145
anno_item = self._get_item(anno_id, Content.TAG.id, Source.USER_ACTIVITY.id, tag)
146
try:
147
self.store.execute(
148
"INSERT INTO annotation (item_id, subject_id) VALUES (?,?)",
149
(anno_id, uri_id), noresult=True)
150
except sqlite3.IntegrityError:
151
pass # Tag already registered
152
153
# Set the item as bookmarked, if it should be
154
if ritem["bookmark"]:
155
anno_uri = "zeitgeist://bookmark/%s" % ritem["uri"]
156
anno_id, discard, discard = self._get_ids(anno_uri, None, None)
157
anno_item = self._get_item(anno_id, Content.BOOKMARK.id,
158
Source.USER_ACTIVITY.id, u"Bookmark")
159
try:
160
self.store.execute(
161
"INSERT INTO annotation (item_id, subject_id) VALUES (?,?)",
162
(anno_id, uri_id), noresult=True)
163
except sqlite3.IntegrityError:
164
pass # Item already bookmarked
165
166
# Do not update the application nor insert the event if `force' is
167
# True, ie., if we are updating an existing item.
168
if force:
169
return 2 if event_exists else 1
170
171
# Insert the application
172
if ritem["app"] in self._applications:
173
app_uri_id = self._applications[ritem["app"]]
174
elif ritem["app"]:
175
try:
176
self.store.execute("INSERT INTO app (info) VALUES (?)",
177
(ritem["app"],), noresult=True)
178
except sqlite3.IntegrityError:
179
pass
180
app_uri_id = self.store.execute(
181
"SELECT item_id FROM app WHERE info=?", (ritem["app"],)).get_one()[0]
182
self._applications[ritem["app"]] = app_uri_id
183
else:
184
# No application specified:
185
app_uri_id = 0
186
187
# Insert the event
188
e_id, e_content_id, e_subject_id = self._get_ids(event_uri, ritem["use"], None)
189
e_item = self._get_item(e_id, e_content_id, Source.USER_ACTIVITY.id, u"Activity")
190
try:
191
self.store.execute(
192
"INSERT INTO event (item_id, subject_id, start, app_id) VALUES (?,?,?,?)",
193
(e_id, uri_id, ritem["timestamp"], app_uri_id), noresult=True)
194
except sqlite3.IntegrityError:
195
# This shouldn't happen.
196
log.exception("Couldn't insert event into DB.")
197
198
return 1
199
200
def insert_events(self, items):
201
"""
202
Inserts items into the database and returns those items which were
203
successfully inserted. If an item fails, that's usually because it
204
already was in the database.
205
"""
206
207
inserted_items = []
208
209
time1 = time.time()
210
for item in items:
211
# This is always 0 or 1, no need to consider 2 as we don't
212
# use the `force' option.
213
if self.insert_event(item, commit=False):
214
inserted_items.append(item)
215
self.store.commit()
216
time2 = time.time()
217
log.debug("Inserted %s items in %.5f s." % (len(inserted_items),
218
time2 - time1))
219
220
return inserted_items
221
222
def get_item(self, uri):
223
""" Returns basic information about the indicated URI. As we are
224
fetching an item, and not an event, `timestamp' is 0 and `use'
225
and `app' are empty strings."""
226
227
item = self.store.execute("""
228
SELECT uri.value, 0 AS timestamp, main_item.id, content.value,
229
"" AS use, source.value, main_item.origin, main_item.text,
230
main_item.mimetype, main_item.icon, "" AS app,
231
(SELECT id
232
FROM item
233
INNER JOIN annotation ON annotation.item_id = item.id
234
WHERE annotation.subject_id = main_item.id AND
235
item.content_id = ?) AS bookmark,
236
(SELECT group_concat(item.text, ", ")
237
FROM item
238
INNER JOIN annotation ON annotation.item_id = item.id
239
WHERE annotation.subject_id = main_item.id AND
240
item.content_id = ?
241
) AS tags
242
FROM item main_item
243
INNER JOIN uri ON (uri.id = main_item.id)
244
INNER JOIN content ON (content.id == main_item.content_id)
245
INNER JOIN source ON (source.id == main_item.source_id)
246
WHERE uri.value = ? LIMIT 1
247
""", (Content.BOOKMARK.id, Content.TAG.id, unicode(uri))).get_one()
248
249
if item:
250
return EventDict.convert_result_to_dict(item)
251
252
def find_events(self, min=0, max=sys.maxint, limit=0,
253
sorting_asc=True, mode="event", filters=(), return_mode=0):
254
"""
255
Returns all items from the database between the indicated
256
timestamps `min' and `max'. Optionally the argument `tags'
257
may be used to filter on tags or `mimetypes' to filter on
258
mimetypes.
259
260
Parameter `mode' can be one of "event", "item" or "mostused".
261
The first mode returns all events, the second one only returns
262
the last event when items are repeated and the "mostused" mode
263
is like "item" but returns the results sorted by the number of
264
events.
265
266
Parameter `filters' is an array of structs containing: (text
267
to search in the name, text to search in the URI, tags,
268
mimetypes, source, content). The filter between characteristics
269
inside the same struct is of type AND (all need to match), but
270
between diferent structs it is OR-like (only the conditions
271
described in one of the structs need to match for the item to
272
be returned).
273
274
Possible values for return_mode, which is an internal variable
275
not exposed in the API:
276
- 0: Return the events/items.
277
- 1: Return the amount of events/items which would be returned.
278
- 2: Return only the applications for the matching events.
279
"""
280
281
time1 = time.time()
282
283
# Emulate optional arguments for the D-Bus interface
284
if not max:
285
max = sys.maxint
286
if not mode:
287
mode = "event"
288
289
if not mode in ("event", "item", "mostused"):
290
raise ValueError, \
291
"Bad find_events call: mode \"%s\" not recongized." % mode
292
293
# filters is a list of dicts, where each dict can have the following items:
294
# name: <str>
295
# uri: <str>
296
# tags: <list> of <str>
297
# mimetypes: <list> of <str>
298
# source: <list> of <str>
299
# content: <list> of <str>
300
# bookmarked: <bool> (True means bookmarked items, and vice versa
301
expressions = []
302
additional_args = []
303
for filter in filters:
304
invalid_filter_keys = set(filter.keys()) - self.ALLOWED_FILTER_KEYS
305
if invalid_filter_keys:
306
raise KeyError, "Invalid key(s) for filter in FindEvents: %s" %\
307
", ".join(invalid_filter_keys)
308
filterset = []
309
if "name" in filter:
310
filterset += [ "main_item.text LIKE ? ESCAPE \"\\\"" ]
311
additional_args += [ filter["name"] ]
312
if "uri" in filter:
313
filterset += [ "uri.value LIKE ? ESCAPE \"\\\"" ]
314
additional_args += [ filter["uri"] ]
315
if "tags" in filter:
316
if not hasattr(filter["tags"], "__iter__"):
317
raise TypeError, "Expected a container type, found %s" % \
318
type(filter["tags"])
319
for tag in filter["tags"]:
320
filterset += [ "(tags == \"%s\" OR tags LIKE \"%s, %%\" OR "
321
"tags LIKE \"%%, %s, %%\" OR tags LIKE \"%%, %s\")" \
322
% (tag, tag, tag, tag) ]
323
if "mimetypes" in filter and len(filter["mimetypes"]):
324
filterset += [ "(" + " OR ".join(
325
["main_item.mimetype LIKE ? ESCAPE \"\\\""] * \
326
len(filter["mimetypes"])) + ")" ]
327
additional_args += filter["mimetypes"]
328
if "source" in filter:
329
filterset += [ "main_item.source_id IN (SELECT id "
330
" FROM source WHERE value IN (%s))" % \
331
",".join("?" * len(filter["source"])) ]
332
additional_args += filter["source"]
333
if "content" in filter:
334
filterset += [ "main_item.content_id IN (SELECT id "
335
" FROM content WHERE value IN (%s))" % \
336
",".join("?" * len(filter["content"])) ]
337
additional_args += filter["content"]
338
if "application" in filter:
339
filterset += [ "event.app_id IN (SELECT item_id "
340
" FROM app WHERE info IN (%s))" % \
341
",".join("?" * len(filter["application"])) ]
342
additional_args += filter["application"]
343
if "bookmarked" in filter:
344
if filter["bookmarked"]:
345
# Only get bookmarked items
346
filterset += [ "bookmark == 1" ]
347
else:
348
# Only get items that aren't bookmarked
349
filterset += [ "bookmark == 0" ]
350
if filterset:
351
expressions += [ "(" + " AND ".join(filterset) + ")" ]
352
353
if expressions:
354
expressions = ("AND (" + " OR ".join(expressions) + ")")
355
else:
356
expressions = ""
357
358
preexpressions = ""
359
additional_orderby = ""
360
361
if mode in ("item", "mostused"):
362
preexpressions += ", MAX(event.start)"
363
expressions += " GROUP BY event.subject_id"
364
if mode == "mostused":
365
additional_orderby += " COUNT(event.rowid) DESC,"
366
elif return_mode == 2:
367
preexpressions += " , COUNT(event.app_id) AS app_count"
368
expressions += " GROUP BY event.app_id"
369
additional_orderby += " app_count DESC,"
370
371
args = [ Content.BOOKMARK.id, Content.TAG.id, min, max ]
372
args += additional_args
373
args += [ limit or sys.maxint ]
374
375
events = self.store.execute("""
376
SELECT uri.value, event.start, main_item.id, content.value,
377
"" AS use, source.value, main_item.origin, main_item.text,
378
main_item.mimetype, main_item.icon,
379
(SELECT info
380
FROM app
381
WHERE app.item_id = event.app_id
382
) AS app,
383
(SELECT COUNT(id)
384
FROM item
385
INNER JOIN annotation ON annotation.item_id = item.id
386
WHERE annotation.subject_id = main_item.id AND
387
item.content_id = ?) AS bookmark,
388
(SELECT group_concat(item.text, ", ")
389
FROM item
390
INNER JOIN annotation ON annotation.item_id = item.id
391
WHERE annotation.subject_id = main_item.id AND
392
item.content_id = ?
393
) AS tags
394
%s
395
FROM item main_item
396
INNER JOIN event ON (main_item.id = event.subject_id)
397
INNER JOIN uri ON (uri.id = main_item.id)
398
INNER JOIN content ON (content.id == main_item.content_id)
399
INNER JOIN source ON (source.id == main_item.source_id)
400
WHERE event.start >= ? AND event.start <= ? %s
401
ORDER BY %s event.start %s LIMIT ?
402
""" % (preexpressions, expressions, additional_orderby,
403
"ASC" if sorting_asc else "DESC"), args).get_all()
404
405
if return_mode == 0:
406
result = map(EventDict.convert_result_to_dict, events)
407
408
time2 = time.time()
409
log.debug("Fetched %s items in %.5f s." % (len(result), time2 - time1))
410
elif return_mode == 1:
411
# We could change the query above to "SELECT COUNT(*) FROM (...)",
412
# where "..." is the complete query converted into a temporary
413
# table, and get the result directly but there isn't enough of
414
# a speed gain in doing that as that it'd be worth doing.
415
result = len(events)
416
elif return_mode == 2:
417
return [(event[10], event[13]) for event in events]
418
419
return result
420
421
def _update_item(self, item):
422
"""
423
Updates an item already in the database.
424
425
If the item has tags, then the tags will also be updated.
426
"""
427
428
#FIXME Delete all tags of the ITEM
429
self._delete_item(item["uri"])
430
self.store.commit()
431
self.store.flush()
432
self.insert_event(item, True, True)
433
self.store.commit()
434
self.store.flush()
435
436
def update_items(self, items):
437
map(self._update_item, items)
438
return items
439
440
def _delete_item(self, uri):
441
442
uri_id = self.store.execute("SELECT id FROM URI WHERE value=?", (uri,)).get_one()
443
uri_id = uri_id[0]
444
annotation_ids = self.store.execute(
445
"SELECT item_id FROM Annotation WHERE subject_id=?", (uri_id,)).get_all()
446
if len(annotation_ids) > 0:
447
for anno in annotation_ids[0]:
448
self.store.execute("DELETE FROM Annotation WHERE subject_id=?",
449
(uri_id,), noresult=True)
450
self.store.execute("DELETE FROM Item WHERE id=?",
451
(anno,), noresult=True)
452
self.store.execute("DELETE FROM Item WHERE id=?",
453
(uri_id,), noresult=True)
454
455
def delete_items(self, items):
456
map(self._delete_item, items)
457
return items
458
459
def get_types(self):
460
"""
461
Returns a list of all different types in the database.
462
"""
463
contents = self.store.find(Content)
464
return [content.value for content in contents]
465
466
def get_tags(self, min_timestamp=0, max_timestamp=0, limit=0, name_filter=""):
467
"""
468
Returns a list containing tuples with the name and the number of
469
occurencies of the tags matching `name_filter', or all existing
470
tags in case it's empty, sorted from most used to least used. `limit'
471
can base used to limit the amount of results.
472
473
Use `min_timestamp' and `max_timestamp' to limit the time frames you
474
want to consider.
475
"""
476
477
return self.store.execute("""
478
SELECT item.text, (SELECT COUNT(rowid) FROM annotation
479
WHERE annotation.item_id = item.id) AS amount
480
FROM item
481
WHERE item.id IN (SELECT annotation.item_id FROM annotation
482
INNER JOIN event ON (event.subject_id = annotation.subject_id)
483
WHERE event.start >= ? AND event.start <= ?)
484
AND item.content_id = ? AND item.text LIKE ? ESCAPE "\\"
485
ORDER BY amount DESC LIMIT ?
486
""", (min_timestamp, max_timestamp or sys.maxint, Content.TAG.id,
487
name_filter or "%", limit or sys.maxint)).get_all()
488
489
def get_last_insertion_date(self, application):
490
"""
491
Returns the timestamp of the last item which was inserted
492
related to the given application. If there is no such record,
493
0 is returned.
494
"""
495
496
query = self.store.execute("""
497
SELECT start FROM event
498
WHERE app_id = (SELECT item_id FROM app WHERE info = ?)
499
ORDER BY start DESC LIMIT 1
500
""", (application,)).get_one()
501
return query[0] if query else 0
502
503
_engine = None
504
def get_default_engine():
505
global _engine
506
if not _engine:
507
_engine = ZeitgeistEngine(get_default_store())
508
return _engine
Loggerhead is a web-based interface for Breezy
Version: 2.0.1