« back to all changes in this revision
Viewing changes to zeitgeist/engine/engine.py
- browse files at revision 762
- compare with another revision
- download diff
- download tarball
- view history from revision 762
- Committer: Siegfried-Angel Gevatter Pujals
- Date: 2009-06-10 13:43:44 UTC
- mfrom: (760.1.29 zeitgeist)
- Revision ID: rainct@ubuntu.com-20090610134344-ncpc7agtb7c09tlg
Merge and cleanup the files a bit (FYI, there is «bzr mv» which doesn't destroy the history of renamed files :).
- files added:
- test
- files renamed:
- zeitgeist/loggers/datasources/zeitgeist_evolution.py => zeitgeist/loggers/datasources/zeitgeist_evolution.py.in
- zeitgeist/loggers/datasources/zeitgeist_recent.py => zeitgeist/loggers/datasources/zeitgeist_recent.py.in
- zeitgeist/loggers/datasources/zeitgeist_tomboy.py => zeitgeist/loggers/datasources/zeitgeist_tomboy.py.in
- zeitgeist/loggers/datasources/zeitgeist_twitter.py => zeitgeist/loggers/datasources/zeitgeist_twitter.py.in
- files modified:
- .bzrignore
added
removed
zeitgeist/engine/engine.py
3
3
import time
4
4
import sys
5
5
import os
6
import gc
6
7
import shutil
7
8
import sqlite3
8
9
import gettext
9
10
import gobject
10
11
from xdg import BaseDirectory
11
12
13
import traceback
14
from random import randint
15
12
16
from zeitgeist import config
13
17
from zeitgeist.shared.zeitgeist_shared import *
18
from zeitgeist.engine.base import *
14
19
15
20
class ZeitgeistEngine(gobject.GObject):
16
21
_salt = 0
22
17
23
def __init__(self):
18
24
19
25
gobject.GObject.__init__(self)
20
26
self.reload_callback = None
21
27
'''
22
28
path = BaseDirectory.save_config_path("zeitgeist")
23
29
database = os.path.join(path, "zeitgeist.sqlite")
24
30
self.connection = self._get_database(database)
25
31
self.cursor = self.connection.cursor()
32
'''
26
33
self._apps = set()
27
34
28
def _result2data(self, result, timestamp=0, start=0, end=sys.maxint, app="", usage=""):
29
30
res = self.cursor.execute(
31
"""SELECT tagid FROM tags WHERE uri = ?""",(result[0],)
32
).fetchall()
33
34
tags = ",".join([unicode(tag[0]) for tag in res]) or ""
35
def _result2data(self, result, timestamp=0, app="", usage=""):
36
37
'''
38
Get Tags
39
'''
40
# FIXME: Get tag
41
tags = ""
42
43
#FIXME: Get bookmark
44
bookmark = False
35
45
36
46
return (
37
47
timestamp,
38
result[0], # uri
39
result[1], # name
40
result[6] or "N/A", # type
41
result[5] or "N/A", # mimetype
48
store.find(URI.value, URI.id == result.id), # uri
49
result.text, # name
50
store.find(URI.value, URI.id == Item.id) or "N/A", # type
51
result.mimetype, # mimetype
42
52
tags, # tags
43
result[2] or "", # comment
53
"", # comment
44
54
usage, # use
45
result[4], # bookmark
46
result[3], # icon
55
bookmark, # bookmark
56
result.icon, # icon
47
57
app, # app
48
58
)
49
59
50
def get_count_for_item(self, uri, start=0, end=sys.maxint):
51
# Emulate optional arguments for D-Bus
52
if not end:
53
end = sys.maxint
54
55
query = "SELECT COUNT(uri) FROM timetable where start >=? and end<=? and uri=?"
56
result = self.cursor.execute(query,(start, end, uri,)).fetchone()
57
return result[0]
58
59
60
def _ensure_item(self, item, uri_only=False):
60
61
"""
61
62
Takes either a Data object or an URI for an item in the
102
103
else:
103
104
return connection
104
105
106
def _next_salt(self):
107
self._salt += 1
108
return self._salt
109
105
110
def get_last_timestamp(self, uri=None):
106
111
"""
107
112
Gets the timestamp of the most recent item in the database. If
110
115
111
116
Returns 0 if there are no items in the database.
112
117
"""
113
114
if uri:
115
filter = "WHERE uri=\"%s\"" % uri
116
else:
117
filter = ""
118
119
query = "SELECT * FROM timetable %s LIMIT 1" % filter
120
121
result = self.cursor.execute(query).fetchone()
122
if result is None:
123
return 0
124
else:
125
return result[0]
118
return 0
126
119
127
def insert_item(self, item):
120
def insert_item(self, ritem, commit=True):
128
121
"""
129
122
Inserts an item into the database. Returns True on success,
130
123
False otherwise (for example, if the item already is in the
131
124
database).
132
125
"""
133
126
127
if not ritem.has_key("uri") or not ritem["uri"]:
128
print >> sys.stderr, "Discarding item without a URI: %s" % ritem
129
return False
130
if not ritem.has_key("content") or not ritem["content"]:
131
print >> sys.stderr, "Discarding item without a Content type: %s" % ritem
132
return False
133
if not ritem.has_key("source") or not ritem["source"]:
134
print >> sys.stderr, "Discarding item without a Source type: %s" % ritem
135
return False
136
134
137
try:
135
# Insert into timetable
136
self.cursor.execute('INSERT INTO timetable VALUES (?,?,?,?,?,?)',
137
(item["timestamp"],
138
None,
139
item["uri"],
140
item["use"],
141
"%d-%s" % (item["timestamp"], item["uri"]),
142
item["app"]
143
))
144
145
# Insert into data, if it isn't there yet
146
try:
147
self.cursor.execute('INSERT INTO data VALUES (?,?,?,?,?,?,?)',
148
(item["uri"],
149
item["name"],
150
item["comment"],
151
item["icon"],
152
0,
153
item["mimetype"],
154
item["type"]))
155
except sqlite3.IntegrityError:
156
pass
157
158
try:
159
# Add tags into the database
160
for tag in (tag.strip() for tag in item["tags"].split(",") if tag.strip()):
161
self.cursor.execute('INSERT INTO tags VALUES (?,?)',
162
(tag.capitalize(), item["uri"]))
163
except Exception, ex:
164
print "Error inserting tags: %s" % ex
165
166
if not item["app"] in self._apps:
167
self._apps.add(item["app"])
168
try:
169
# Add app into the database
170
self.cursor.execute('INSERT INTO app VALUES (?,?)',
171
(item["app"],0))
172
except Exception, ex:
173
print "Error inserting application: %s" % ex
174
175
except sqlite3.IntegrityError, ex:
176
return False
177
178
else:
179
return True
138
# The item may already exist in the db,
139
# so only create it if necessary
140
item = Item.lookup_or_create(ritem["uri"])
141
item.content = Content.lookup_or_create(ritem["content"])
142
item.source = Source.lookup_or_create(ritem["source"])
143
item.text = unicode(ritem["text"])
144
item.mimetype = unicode(ritem["mimetype"])
145
item.icon = unicode(ritem["icon"])
146
item.origin = ritem["origin"]
147
except sqlite3.IntegrityError, ex:
148
traceback.print_exc()
149
print >> sys.stderr, "Failed to insert item:\n%s" % ritem
150
print >> sys.stderr, "Error was: %s" % ex
151
return False
152
153
# Store a new event for this
154
try:
155
e_uri = "zeitgeist://event/"+ritem["use"]+"/"+str(item.id)+"/"+str(ritem["timestamp"]) + "#" + str(self._next_salt())
156
e = Event.lookup_or_create(e_uri)
157
e.subject = item
158
e.start = ritem["timestamp"]
159
e.item.text = u"Activity"
160
e.item.source_id = Source.USER_ACTIVITY.id
161
e.item.content = Content.lookup_or_create(ritem["use"])
162
163
#FIXME: Lots of info from the applications, try to sort them out here properly
164
165
app_info = resolve_dot_desktop(ritem["app"])
166
167
e.app = App.lookup_or_create(ritem["app"])
168
#print app_info
169
e.app.item.text = unicode(app_info["name"])
170
e.app.item.content = Content.lookup_or_create(app_info["type"])
171
e.app.item.source = Source.lookup_or_create(app_info["exec"])
172
e.app.item.icon = unicode(app_info["icon"])
173
e.app.info = unicode(ritem["app"]) # FIXME: App constructor could parse out appliction name from .desktop file
174
if app_info.has_key("categories") and app_info["categories"].strip() != "":
175
# Iterate over non-empty strings only
176
for tag in filter(lambda t : bool(t), app_info["categories"].split(";")):
177
print "TAG:", tag
178
a_uri = "zeitgeist://tag/%s" % tag
179
a = Annotation.lookup_or_create(a_uri)
180
a.subject = e.app.item
181
a.item.text = tag
182
a.item.source_id = Source.APPLICATION.id
183
a.item.content_id = Content.TAG.id
184
185
186
except sqlite3.IntegrityError, ex:
187
traceback.print_exc()
188
print >> sys.stderr, "Failed to insert event, '%s':\n%s" % (e_uri, ritem)
189
print >> sys.stderr, "Error was: %s" % ex
190
return False
191
192
# Extract tags
193
if ritem.has_key("tags") and ritem["tags"].strip() != "":
194
# Iterate over non-empty strings only
195
for tag in filter(lambda t : bool(t), ritem["tags"].split(";")):
196
print "TAG:", tag
197
a_uri = "zeitgeist://tag/%s" % tag
198
a = Annotation.lookup_or_create(a_uri)
199
a.subject = item
200
a.item.text = tag
201
a.item.source_id = Source.USER_ACTIVITY.id
202
a.item.content_id = Content.TAG.id
203
204
# Extract bookmarks
205
if ritem.has_key("bookmark") and ritem["bookmark"]:
206
print "BOOKMARK:", ritem["uri"]
207
a_uri = "zeitgeist://bookmark/%s" % ritem["uri"]
208
a = Annotation.lookup_or_create(a_uri)
209
a.subject = item
210
a.item.text = u"Bookmark"
211
a.item.source_id = Source.USER_ACTIVITY.id
212
a.item.content_id = Content.BOOKMARK.id
213
214
if commit:
215
store.flush()
216
217
return True
180
218
181
219
def insert_items(self, items):
182
220
"""
185
223
"""
186
224
amount_items = 0
187
225
for item in items:
188
if self.insert_item(item):
226
if self.insert_item(item, commit=False):
189
227
amount_items += 1
190
191
self.connection.commit()
228
else:
229
print >> sys.stderr, "Error inserting %s" % item["uri"]
230
231
store.commit()
232
gc.collect()
233
print "DONE"
234
print "got items"
235
192
236
return amount_items
193
237
194
238
def get_item(self, uri):
195
239
"""Returns basic information about the indicated URI."""
196
240
197
item = self.cursor.execute(
198
"""SELECT * FROM data WHERE data.uri=?""", (uri,)).fetchone()
241
id = store.find(URI, URI.value == uri)
242
item = store.find(Item, Item)
199
243
200
244
if item:
201
245
return self._result2data(item)
207
251
may be used to filter on tags.
208
252
"""
209
253
func = self._result2data
210
254
pack = []
211
255
# Emulate optional arguments for the D-Bus interface
212
256
if not max: max = sys.maxint
213
257
214
# Get a list of all tags
215
if tags:
216
tagsql = []
217
for tag in tags.split(","):
218
tagsql.append("""(data.uri LIKE '%%%s%%'
219
OR data.name LIKE '%%%s%%' OR '%s' in
220
(SELECT tags.tagid FROM tags
221
WHERE tags.uri=data.uri))""" % (tag, tag, tag))
222
condition = "(" + " AND ".join(tagsql) + ")"
223
else:
224
condition = "1"
225
226
if mimetypes:
227
condition += " AND data.mimetype IN (%s)" % \
228
",".join(("\"%s\"" % mime for mime in mimetypes.split(",")))
229
230
# Loop over all items in the timetable table which are between min and max
231
query = """
232
SELECT start, data.uri, app, usage FROM timetable
233
JOIN data ON (data.uri=timetable.uri)
234
WHERE usage != 'linked' AND start >= ? AND start <= ?
235
AND %s
236
ORDER BY start ASC
237
""" % condition
238
#print query
239
res = self.cursor.execute(query, (str(min), str(max))).fetchall()
240
pack = []
241
for start, uri, app, usage in res:
242
# Retrieve the item from the data table
243
# TODO: Integrate this into the previous SQL query.
244
item = self.cursor.execute(
245
"""SELECT * FROM data WHERE data.uri=?""", (uri,)).fetchone()
258
for event in store.find(Event, Event.start >= min, Event.start <= max):
259
start = event.start
260
261
usage_id = store.find(URI, URI.id == event.item_id).one()
262
usage = store.find(Item.content_id, Item.id == usage_id.id).one()
263
usage = store.find(Content.value, Content.id == usage).one()
264
265
item = store.find(Item, Item.id == event.subject_id).one()
266
267
app= ""
246
268
247
269
if item:
248
item = func(item, timestamp = start, usage=usage, app=app)
249
print item
250
pack.append(item)
270
pack.append(func(item, timestamp = start, usage=usage, app=app))
251
271
return pack
252
272
253
273
def update_item(self, item):
283
303
self.connection.commit()
284
304
285
305
def delete_item(self, item):
286
item_uri = self._ensure_item(item, uri_only=True)
287
self.cursor.execute('DELETE FROM data where uri=?', (item_uri,))
288
self.cursor.execute('DELETE FROM tags where uri=?', (item_uri,))
289
self.connection.commit()
306
pass
290
307
291
308
def _get_tags(self, order_by, count, min, max):
292
309
"""
293
310
Private class used to retrive a list of tags according to a
294
311
desired condition (eg., most used tags, recently used tags...).
295
312
"""
296
297
# We simulate optional values in D-Bus; reset the defaults
298
if not count: count = 20
299
if not max: max = sys.maxint
300
301
# TODO: This is awful.
302
303
uris = []
304
tags = []
305
306
# Get uri's in in time interval sorted desc by time
307
query = """SELECT uri
308
FROM timetable
309
WHERE usage != 'linked'
310
AND start >= ?
311
AND start <= ?
312
ORDER BY %s DESC""" % order_by
313
314
for uri in self.cursor.execute(query, (str(int(min)), str(int(max)))).fetchall():
315
316
# Retrieve the item from the data table:
317
uri = uri[0]
318
319
if uris.count(uri) <= 0 and len(tags) < count:
320
uris.append(uri)
321
uri = self.cursor.execute(
322
"SELECT * FROM data WHERE uri=?", (uri,)).fetchone()
323
324
if uri:
325
res = self.cursor.execute(
326
"""SELECT tagid FROM tags WHERE uri = ?""",
327
(uri[0],)).fetchall()
328
329
for tag in res:
330
tag = unicode(tag[0])
331
if tags.count(tag) <= 0:
332
if len(tags) < count:
333
tags.append(tag)
334
335
if len(tags) == count:
336
break
337
338
return tags
313
return []
339
314
340
315
def get_all_tags(self):
341
316
"""
342
317
Returns a list containing the name of all tags.
343
318
"""
344
345
for tag in self.cursor.execute(
346
"SELECT DISTINCT(tagid) FROM tags").fetchall():
347
yield unicode(tag[0])
319
return []
348
320
349
321
def get_types(self):
350
322
"""
372
344
return self._get_tags("uri", count, min, max)
373
345
374
346
def get_min_timestamp_for_tag(self, tag):
375
res = self.cursor.execute("""
376
SELECT
377
(SELECT min(start) FROM timetable WHERE uri=tags.uri)
378
AS value
379
FROM tags WHERE tagid=?
380
ORDER BY value ASC LIMIT 1
381
""", (tag,)).fetchall()
382
if res:
383
return res[0][0]
384
else:
385
347
return None
386
348
387
349
def get_max_timestamp_for_tag(self, tag):
388
res = self.cursor.execute("""
389
SELECT
390
(SELECT max(start) FROM timetable WHERE uri=tags.uri)
391
AS value
392
FROM tags WHERE tagid=?
393
ORDER BY value DESC LIMIT 1
394
""", (tag,)).fetchall()
395
if res:
396
return res[0][0]
397
else:
398
350
return None
399
351
400
352
def get_timestamps_for_tag(self, tag):
401
402
begin = self.get_min_timestamp_for_tag(tag)
403
end = self.get_max_timestamp_for_tag(tag)
404
405
if begin and end:
406
if end - begin > 86400:
407
# TODO: Why do we do this?
408
end = end + 86400
409
else:
410
begin = end = 0
411
412
return (begin, end)
353
pass
413
354
414
355
def get_items_related_by_tags(self, item):
415
416
356
# TODO: Is one matching tag enough or should more/all of them
417
357
# match?
418
for tag in self._ensure_item(item)[4]:
419
res = self.cursor.execute('SELECT uri FROM tags WHERE tagid=? GROUP BY uri ORDER BY COUNT(uri) DESC', (tag,)).fetchall()
420
for raw in res:
421
item = self.cursor.execute("SELECT * FROM data WHERE uri=?", (raw[0],)).fetchone()
422
if item:
423
yield self._result2data(item)
358
pass
424
359
425
360
def get_related_items(self, uri):
426
427
print uri
428
print "\n\n\n"
429
list = []
430
dict = {}
431
radius = 120 #radius is half a day
432
433
'''
434
Create pins to create neighbourhoods around
435
'''
436
results = self.cursor.execute( "SELECT start FROM timetable WHERE uri=? ORDER BY start",
437
( uri, ) ).fetchall()
438
pins = [r[0] for r in results]
439
440
'''
441
Determine neighbourhoods
442
'''
443
nbhs = {}
444
for p in pins:
445
446
start = p - radius
447
end = p + radius
448
449
info = {"start":start, "pin":p, "end":end, "uri":uri}
450
451
try:
452
start = ( p + pins[pins.index( p ) - 1] ) / 2
453
print + "+ " + start + " +"
454
if p - start < radius:
455
info["start"] = start
456
except Exception, ex:
457
pass
458
459
try:
460
end = ( p + pins[pins.index( p ) + 1] ) / 2
461
if end - p < radius:
462
info["end"] = end
463
except Exception, ex:
464
pass
465
466
nbh = []
467
for i in self.get_items(info["start"],info["end"]):
468
nbh.append(i)
469
nbhs[p] = nbh
470
471
self.compare_nbhs(nbhs)
472
473
'''
474
get items for each nbh and store in a list in a hash'
475
'''
476
for item in self.common_items.values():
477
if item[1] >= 3:
478
print len(nbhs)
479
yield item[0]
361
pass
480
362
481
482
363
def compare_nbhs(self,nbhs):
483
self.common_items = {}
484
for nbh in nbhs.values():
485
try:
486
for item in nbh:
487
for tempnbh in nbhs.values():
488
for tempitem in tempnbh:
489
if tempitem[2] == item[2]:
490
if self.common_items.has_key(item[2]):
491
self.common_items[item[2]][1] +=1
492
else:
493
self.common_items[item[2]] = [item,1]
494
495
except Exception, ex:
496
print "\n"
497
print ex
498
print "\n"
499
364
pass
500
365
501
366
def get_items_with_mimetype(self, mimetype, min=0, max=sys.maxint, tags=""):
502
return self.get_items(min, max, tags, mimetype)
367
pass
503
368
504
369
def get_uris_for_timestamp(self, timestamp):
505
return [x[0] for x in
506
self.cursor.execute("SELECT uri FROM timetable WHERE start=?",
507
(timestamp,)).fetchall()]
370
pass
508
371
509
372
def get_bookmarks(self):
510
for item in self.cursor.execute("SELECT * FROM data WHERE boomark=1").fetchall():
511
yield self._result2data(item, timestamp = -1)
373
return []
512
374
513
375
engine = ZeitgeistEngine()
Loggerhead is a web-based interface for Breezy
Version: 2.0.1