1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
|
# Copyright (C) 2011 Canonical
#
# Authors:
# Matthew McGowan
# Michael Vogt
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; version 3.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
import logging
import time
import threading
import xapian
from gi.repository import GObject
from softwarecenter.enums import (SortMethods,
XapianValues,
NonAppVisibility,
DEFAULT_SEARCH_LIMIT)
from softwarecenter.db.database import (
SearchQuery, LocaleSorter, TopRatedSorter)
from softwarecenter.distro import get_distro
from softwarecenter.utils import ExecutionTime
LOG=logging.getLogger(__name__)
class AppEnquire(GObject.GObject):
"""
A interface to enquire data from a xapian database.
It can combined with any xapian querry and with
a generic filter function (that can filter on data not
available in xapian)
"""
# signal emited
__gsignals__ = {"query-complete" : (GObject.SIGNAL_RUN_FIRST,
GObject.TYPE_NONE,
()),
}
def __init__(self, cache, db):
"""
Init a AppEnquire object
:Parameters:
- `cache`: apt cache (for stuff like the overlay icon)
- `db`: a xapian.Database that contians the applications
"""
GObject.GObject.__init__(self)
self.cache = cache
self.db = db
self.distro = get_distro()
self.search_query = SearchQuery(None)
self.nonblocking_load = True
self.sortmode = SortMethods.UNSORTED
self.nonapps_visible = NonAppVisibility.MAYBE_VISIBLE
self.limit = DEFAULT_SEARCH_LIMIT
self.filter = None
self.exact = False
self.nr_pkgs = 0
self.nr_apps = 0
self._matches = []
self.match_docids = set()
def __len__(self):
return len(self._matches)
@property
def matches(self):
""" return the list of matches as xapian.MSetItem """
return self._matches
def _threaded_perform_search(self):
self._perform_search_complete = False
# generate a name and ensure we never have two threads
# with the same name
names = [thread.name for thread in threading.enumerate()]
for i in range(threading.active_count()+1, 0, -1):
thread_name = 'ThreadedQuery-%s' % i
if not thread_name in names:
break
# create and start it
t = threading.Thread(
target=self._blocking_perform_search, name=thread_name)
t.start()
# don't block the UI while the thread is running
context = GObject.main_context_default()
while not self._perform_search_complete:
time.sleep(0.02) # 50 fps
while context.pending():
context.iteration()
t.join()
# call the query-complete callback
self.emit("query-complete")
def _get_estimate_nr_apps_and_nr_pkgs(self, enquire, q, xfilter):
# filter out docs of pkgs of which there exists a doc of the app
enquire.set_query(xapian.Query(xapian.Query.OP_AND,
q, xapian.Query("ATapplication")))
try:
tmp_matches = enquire.get_mset(0, len(self.db), None, xfilter)
except Exception:
LOG.exception("_get_estimate_nr_apps_and_nr_pkgs failed")
return (0, 0)
nr_apps = tmp_matches.get_matches_estimated()
enquire.set_query(xapian.Query(xapian.Query.OP_AND_NOT,
q, xapian.Query("XD")))
tmp_matches = enquire.get_mset(0, len(self.db), None, xfilter)
nr_pkgs = tmp_matches.get_matches_estimated() - nr_apps
return (nr_apps, nr_pkgs)
def _blocking_perform_search(self):
# WARNING this call may run in a thread, so its *not*
# allowed to touch gtk, otherwise hell breaks loose
# performance only: this is only needed to avoid the
# python __call__ overhead for each item if we can avoid it
# use a unique instance of both enquire and xapian database
# so concurrent queries dont result in an inconsistent database
# an alternative would be to serialise queries
enquire = xapian.Enquire(self.db.xapiandb)
if self.filter and self.filter.required:
xfilter = self.filter
else:
xfilter = None
# go over the queries
self.nr_apps, self.nr_pkgs = 0, 0
_matches = self._matches
match_docids = self.match_docids
for q in self.search_query:
LOG.debug("initial query: '%s'" % q)
# for searches we may want to disable show/hide
terms = [term for term in q]
exact_pkgname_query = (len(terms) == 1 and
terms[0].startswith("XP"))
with ExecutionTime("calculate nr_apps and nr_pkgs: "):
nr_apps, nr_pkgs = self._get_estimate_nr_apps_and_nr_pkgs(enquire, q, xfilter)
self.nr_apps += nr_apps
self.nr_pkgs += nr_pkgs
# only show apps by default (unless in always visible mode)
if self.nonapps_visible != NonAppVisibility.ALWAYS_VISIBLE:
if not exact_pkgname_query:
q = xapian.Query(xapian.Query.OP_AND,
xapian.Query("ATapplication"),
q)
LOG.debug("nearly completely filtered query: '%s'" % q)
# filter out docs of pkgs of which there exists a doc of the app
# FIXME: make this configurable again?
enquire.set_query(xapian.Query(xapian.Query.OP_AND_NOT,
q, xapian.Query("XD")))
# sort results
# cataloged time - what's new category
if self.sortmode == SortMethods.BY_CATALOGED_TIME:
if (self.db._axi_values and
"catalogedtime" in self.db._axi_values):
enquire.set_sort_by_value(
self.db._axi_values["catalogedtime"], reverse=True)
else:
LOG.warning("no catelogedtime in axi")
elif self.sortmode == SortMethods.BY_TOP_RATED:
from softwarecenter.backend.reviews import get_review_loader
review_loader = get_review_loader(self.cache, self.db)
sorter = TopRatedSorter(self.db, review_loader)
enquire.set_sort_by_key(sorter, reverse=True)
# search ranking - when searching
elif self.sortmode == SortMethods.BY_SEARCH_RANKING:
#enquire.set_sort_by_value(XapianValues.POPCON)
# use the default enquire.set_sort_by_relevance()
pass
# display name - all categories / channels
elif (self.db._axi_values and
"display_name" in self.db._axi_values):
enquire.set_sort_by_key(LocaleSorter(self.db), reverse=False)
# fallback to pkgname - if needed?
# fallback to pkgname - if needed?
else:
enquire.set_sort_by_value_then_relevance(
XapianValues.PKGNAME, False)
#~ try:
if self.limit == 0:
matches = enquire.get_mset(0, len(self.db), None, xfilter)
else:
matches = enquire.get_mset(0, self.limit, None, xfilter)
LOG.debug("found ~%i matches" % matches.get_matches_estimated())
#~ except:
#~ logging.exception("get_mset")
#~ matches = []
# promote exact matches to a "app", this will make the
# show/hide technical items work correctly
if exact_pkgname_query and len(matches) == 1:
self.nr_apps += 1
self.nr_pkgs -= 2
# add matches, but don't duplicate docids
with ExecutionTime("append new matches to existing ones:"):
for match in matches:
if not match.docid in match_docids:
_matches.append(match)
match_docids.add(match.docid)
# if we have no results, try forcing pkgs to be displayed
# if not NonAppVisibility.NEVER_VISIBLE is set
if (not _matches and
self.nonapps_visible not in (NonAppVisibility.ALWAYS_VISIBLE,
NonAppVisibility.NEVER_VISIBLE)):
self.nonapps_visible = NonAppVisibility.ALWAYS_VISIBLE
self._blocking_perform_search()
# wake up the UI if run in a search thread
self._perform_search_complete = True
return
def get_estimated_matches_count(self, query):
with ExecutionTime("estimate item count for query: '%s'" % query):
enquire = xapian.Enquire(self.db.xapiandb)
enquire.set_query(query)
# no performance difference between the two
#tmp_matches = enquire.get_mset(0, 1, None, None)
#nr_pkgs = tmp_matches.get_matches_estimated()
tmp_matches = enquire.get_mset(0, len(self.db), None, None)
nr_pkgs = len(tmp_matches)
return nr_pkgs
def set_query(self, search_query,
limit=DEFAULT_SEARCH_LIMIT,
sortmode=SortMethods.UNSORTED,
filter=None,
exact=False,
nonapps_visible=NonAppVisibility.MAYBE_VISIBLE,
nonblocking_load=True,
persistent_duplicate_filter=False):
"""
Set a new query
:Parameters:
- `search_query`: a single search as a xapian.Query or a list
- `limit`: how many items the search should return (0 == unlimited)
- `sortmode`: sort the result
- `filter`: filter functions that can be used to filter the
data further. A python function that gets a pkgname
- `exact`: If true, indexes of queries without matches will be
maintained in the store (useful to show e.g. a row
with "??? not found")
- `nonapps_visible`: decide whether adding non apps in the model or not.
Can be NonAppVisibility.ALWAYS_VISIBLE/NonAppVisibility.MAYBE_VISIBLE
/NonAppVisibility.NEVER_VISIBLE
(NonAppVisibility.MAYBE_VISIBLE will return non apps result
if no matching apps is found)
- `nonblocking_load`: set to False to execute the query inside the current
thread. Defaults to True to allow the search to be
performed without blocking the UI.
- 'persistent_duplicate_filter': if True allows filtering of duplicate
matches across multiple queries
"""
self.search_query = SearchQuery(search_query)
self.limit = limit
self.sortmode = sortmode
# make a copy for good measure
if filter:
self.filter = filter.copy()
else:
self.filter = None
self.exact = exact
self.nonblocking_load = nonblocking_load
self.nonapps_visible = nonapps_visible
# no search query means "all"
if not search_query:
self.search_query = SearchQuery(xapian.Query(""))
self.sortmode = SortMethods.BY_ALPHABET
self.limit = 0
# flush old query matches
self._matches = []
if not persistent_duplicate_filter:
self.match_docids = set()
# we support single and list search_queries,
# if list we append them one by one
with ExecutionTime("populate model from query: '%s' (threaded: %s)" % (
" ; ".join([str(q) for q in self.search_query]),
self.nonblocking_load), with_traceback=False):
if self.nonblocking_load:
self._threaded_perform_search()
else:
self._blocking_perform_search()
return True
# def get_pkgnames(self):
# xdb = self.db.xapiandb
# pkgnames = []
# for m in self.matches:
# doc = xdb.get_document(m.docid)
# pkgnames.append(doc.get_value(XapianValues.PKGNAME) or doc.get_data())
# return pkgnames
# def get_applications(self):
# apps = []
# for pkgname in self.get_pkgnames():
# apps.append(Application(pkgname=pkgname))
# return apps
def get_docids(self):
""" get the docids of the current matches """
xdb = self.db.xapiandb
return [xdb.get_document(m.docid).get_docid() for m in self._matches]
def get_documents(self):
""" get the xapian.Document objects of the current matches """
xdb = self.db.xapiandb
return [xdb.get_document(m.docid) for m in self._matches]
|