← Back to branch summary

~gary-lasker/software-center/recommendations-opt-out

« back to all changes in this revision

Viewing changes to softwarecenter/backend/reviews/__init__.py

  • Committer: Gary Lasker
  • Date: 2012-03-18 22:14:47 UTC
  • mfrom: (2844.2.30 tweaks)
  • Revision ID: gary.lasker@canonical.com-20120318221447-7ihopj3074ginitu
trunkify

Show diffs side-by-side

added added

removed removed

Lines of Context:
softwarecenter/backend/reviews/__init__.py
36
36
# py3 compat
37
37
try:
38
38
    import cPickle as pickle
39
 
    pickle # pyflakes
 
39
    pickle  # pyflakes
40
40
except ImportError:
41
41
    import pickle
42
42
 
58
58
 
59
59
LOG = logging.getLogger(__name__)
60
60
 
 
61
 
61
62
class ReviewStats(object):
62
63
    def __init__(self, app):
63
64
        self.app = app
64
65
        self.ratings_average = None
65
66
        self.ratings_total = 0
66
 
        self.rating_spread = [0,0,0,0,0]
 
67
        self.rating_spread = [0, 0, 0, 0, 0]
67
68
        self.dampened_rating = 3.00
 
69
 
68
70
    def __repr__(self):
69
 
        return ("<ReviewStats '%s' ratings_average='%s' ratings_total='%s'" 
70
 
                " rating_spread='%s' dampened_rating='%s'>" % 
71
 
                (self.app, self.ratings_average, self.ratings_total, 
 
71
        return ("<ReviewStats '%s' ratings_average='%s' ratings_total='%s'"
 
72
                " rating_spread='%s' dampened_rating='%s'>" %
 
73
                (self.app, self.ratings_average, self.ratings_total,
72
74
                self.rating_spread, self.dampened_rating))
73
 
    
 
75
 
74
76
 
75
77
class UsefulnessCache(object):
76
78
 
77
79
    USEFULNESS_CACHE = {}
78
 
    
 
80
 
79
81
    def __init__(self, try_server=False):
80
82
        fname = "usefulness.p"
81
83
        self.USEFULNESS_CACHE_FILE = os.path.join(SOFTWARE_CENTER_CACHE_DIR,
82
 
                                                    fname)
83
 
        
 
84
                                                  fname)
 
85
 
84
86
        self._retrieve_votes_from_cache()
85
 
        #Only try to get votes from the server if required, otherwise just use cache
 
87
        # Only try to get votes from the server if required, otherwise
 
88
        # just use cache
86
89
        if try_server:
87
90
            self._retrieve_votes_from_server()
88
 
    
 
91
 
89
92
    def _retrieve_votes_from_cache(self):
90
93
        if os.path.exists(self.USEFULNESS_CACHE_FILE):
91
94
            try:
92
 
                self.USEFULNESS_CACHE = pickle.load(open(self.USEFULNESS_CACHE_FILE))
 
95
                self.USEFULNESS_CACHE = pickle.load(
 
96
                    open(self.USEFULNESS_CACHE_FILE))
93
97
            except:
94
98
                LOG.exception("usefulness cache load fallback failure")
95
 
                os.rename(self.USEFULNESS_CACHE_FILE, self.USEFULNESS_CACHE_FILE+".fail")
96
 
        return
97
 
    
 
99
                os.rename(self.USEFULNESS_CACHE_FILE,
 
100
                    self.USEFULNESS_CACHE_FILE + ".fail")
 
101
 
98
102
    def _retrieve_votes_from_server(self):
99
103
        LOG.debug("_retrieve_votes_from_server started")
100
104
        user = get_person_from_config()
101
 
        
 
105
 
102
106
        if not user:
103
 
            LOG.warn("Could not get usefulness from server, no username in config file")
 
107
            LOG.warn("Could not get usefulness from server, no username "
 
108
                "in config file")
104
109
            return False
105
 
        
 
110
 
106
111
        # run the command and add watcher
107
112
        spawn_helper = SpawnHelper()
108
113
        spawn_helper.connect("data-available", self._on_usefulness_data)
116
121
        for result in results:
117
122
            self.USEFULNESS_CACHE[str(result['review_id'])] = result['useful']
118
123
        if not self.save_usefulness_cache_file():
119
 
            LOG.warn("Read usefulness results from server but failed to write to cache")
120
 
    
 
124
            LOG.warn("Read usefulness results from server but failed to "
 
125
                "write to cache")
 
126
 
121
127
    def save_usefulness_cache_file(self):
122
128
        """write the dict out to cache file"""
123
129
        cachedir = SOFTWARE_CENTER_CACHE_DIR
129
135
            return True
130
136
        except:
131
137
            return False
132
 
    
 
138
 
133
139
    def add_usefulness_vote(self, review_id, useful):
134
 
        """pass a review id and useful boolean vote and save it into the dict, then try to save to cache file"""
 
140
        """pass a review id and useful boolean vote and save it into the
 
141
           dict, then try to save to cache file
 
142
        """
135
143
        self.USEFULNESS_CACHE[str(review_id)] = useful
136
144
        if self.save_usefulness_cache_file():
137
145
            return True
138
146
        return False
139
 
    
 
147
 
140
148
    def check_for_usefulness(self, review_id):
141
 
        """pass a review id and get a True/False useful back or None if the review_id is not in the dict"""
 
149
        """pass a review id and get a True/False useful back or None if the
 
150
           review_id is not in the dict
 
151
        """
142
152
        return self.USEFULNESS_CACHE.get(str(review_id))
143
 
    
144
 
    
 
153
 
145
154
 
146
155
class Review(object):
147
156
    """A individual review object """
163
172
        self.version = ""
164
173
        self.usefulness_total = 0
165
174
        self.usefulness_favorable = 0
166
 
        # this will be set if tryint to submit usefulness for this review failed
 
175
        # this will be set if tryint to submit usefulness for this review
 
176
        # failed
167
177
        self.usefulness_submit_error = False
168
178
        self.delete_error = False
169
179
        self.modify_error = False
 
180
 
170
181
    def __repr__(self):
171
182
        return "[Review id=%s review_text='%s' reviewer_username='%s']" % (
172
183
            self.id, self.review_text, self.reviewer_username)
 
184
 
173
185
    def __cmp__(self, other):
174
186
        # first compare version, high version number first
175
187
        vc = upstream_version_compare(self.version, other.version)
176
188
        if vc != 0:
177
189
            return vc
178
190
        # then wilson score
179
 
        uc = cmp(wilson_score(self.usefulness_favorable, 
 
191
        uc = cmp(wilson_score(self.usefulness_favorable,
180
192
                              self.usefulness_total),
181
193
                 wilson_score(other.usefulness_favorable,
182
194
                              other.usefulness_total))
184
196
            return uc
185
197
        # last is date
186
198
        t1 = datetime.datetime.strptime(self.date_created, '%Y-%m-%d %H:%M:%S')
187
 
        t2 = datetime.datetime.strptime(other.date_created, '%Y-%m-%d %H:%M:%S')
 
199
        t2 = datetime.datetime.strptime(other.date_created,
 
200
            '%Y-%m-%d %H:%M:%S')
188
201
        return cmp(t1, t2)
189
 
        
 
202
 
190
203
    @classmethod
191
204
    def from_piston_mini_client(cls, other):
192
205
        """ converts the rnrclieent reviews we get into
209
222
            setattr(review, k, v)
210
223
        return review
211
224
 
 
225
 
212
226
class ReviewLoader(GObject.GObject):
213
227
    """A loader that returns a review object list"""
214
228
 
215
229
    __gsignals__ = {
216
 
        "refresh-review-stats-finished" : (GObject.SIGNAL_RUN_LAST,
217
 
                                           GObject.TYPE_NONE, 
218
 
                                           (GObject.TYPE_PYOBJECT,),
219
 
                                          ),
 
230
        "refresh-review-stats-finished": (GObject.SIGNAL_RUN_LAST,
 
231
                                          GObject.TYPE_NONE,
 
232
                                          (GObject.TYPE_PYOBJECT,),
 
233
                                         ),
220
234
    }
221
235
 
222
236
    # cache the ReviewStats
236
250
        self.REVIEW_STATS_CACHE_FILE = os.path.join(SOFTWARE_CENTER_CACHE_DIR,
237
251
                                                    fname)
238
252
        self.REVIEW_STATS_BSDDB_FILE = "%s__%s.%s.db" % (
239
 
            self.REVIEW_STATS_CACHE_FILE, 
240
 
            bdb.DB_VERSION_MAJOR, 
 
253
            self.REVIEW_STATS_CACHE_FILE,
 
254
            bdb.DB_VERSION_MAJOR,
241
255
            bdb.DB_VERSION_MINOR)
242
256
 
243
257
        self.language = get_language()
244
258
        if os.path.exists(self.REVIEW_STATS_CACHE_FILE):
245
259
            try:
246
 
                self.REVIEW_STATS_CACHE = pickle.load(open(self.REVIEW_STATS_CACHE_FILE))
 
260
                self.REVIEW_STATS_CACHE = pickle.load(
 
261
                    open(self.REVIEW_STATS_CACHE_FILE))
247
262
                self._cache_version_old = self._missing_histogram_in_cache()
248
263
            except:
249
264
                LOG.exception("review stats cache load failure")
250
 
                os.rename(self.REVIEW_STATS_CACHE_FILE, self.REVIEW_STATS_CACHE_FILE+".fail")
251
 
    
 
265
                os.rename(self.REVIEW_STATS_CACHE_FILE,
 
266
                    self.REVIEW_STATS_CACHE_FILE + ".fail")
 
267
 
252
268
    def _missing_histogram_in_cache(self):
253
269
        '''iterate through review stats to see if it has been fully reloaded
254
270
           with new histogram data from server update'''
260
276
 
261
277
    def get_reviews(self, application, callback, page=1, language=None,
262
278
                    sort=0, relaxed=False):
263
 
        """run callback f(app, review_list) 
 
279
        """run callback f(app, review_list)
264
280
           with list of review objects for the given
265
281
           db.database.Application object
266
282
        """
282
298
                return self.REVIEW_STATS_CACHE[application]
283
299
        except ValueError:
284
300
            pass
285
 
        return None
286
301
 
287
302
    def refresh_review_stats(self, callback):
288
303
        """ get the review statists and call callback when its there """
322
337
        """ write out the full REVIEWS_STATS_CACHE as a pickle """
323
338
        pickle.dump(self.REVIEW_STATS_CACHE,
324
339
                      open(self.REVIEW_STATS_CACHE_FILE, "w"))
325
 
                                       
 
340
 
326
341
    def _dump_bsddbm_for_unity(self, outfile, outdir):
327
342
        """ write out the subset that unity needs of the REVIEW_STATS_CACHE
328
343
            as a C friendly (using struct) bsddb
330
345
        env = bdb.DBEnv()
331
346
        if not os.path.exists(outdir):
332
347
            os.makedirs(outdir)
333
 
        env.open (outdir,
334
 
                  bdb.DB_CREATE | bdb.DB_INIT_CDB | bdb.DB_INIT_MPOOL |
335
 
                  bdb.DB_NOMMAP, # be gentle on e.g. nfs mounts
336
 
                  0600)
337
 
        db = bdb.DB (env)
338
 
        db.open (outfile,
339
 
                 dbtype=bdb.DB_HASH,
340
 
                 mode=0600,
341
 
                 flags=bdb.DB_CREATE)
 
348
        env.open(outdir,
 
349
                 bdb.DB_CREATE | bdb.DB_INIT_CDB | bdb.DB_INIT_MPOOL |
 
350
                 bdb.DB_NOMMAP,  # be gentle on e.g. nfs mounts
 
351
                 0600)
 
352
        db = bdb.DB(env)
 
353
        db.open(outfile,
 
354
                dbtype=bdb.DB_HASH,
 
355
                mode=0600,
 
356
                flags=bdb.DB_CREATE)
342
357
        for (app, stats) in self.REVIEW_STATS_CACHE.iteritems():
343
358
            # pkgname is ascii by policy, so its fine to use str() here
344
 
            db[str(app.pkgname)] = struct.pack('iii', 
 
359
            db[str(app.pkgname)] = struct.pack('iii',
345
360
                                               stats.ratings_average or 0,
346
361
                                               stats.ratings_total,
347
362
                                               stats.dampened_rating)
348
 
        db.close ()
349
 
        env.close ()
350
 
    
 
363
        db.close()
 
364
        env.close()
 
365
 
351
366
    def get_top_rated_apps(self, quantity=12, category=None):
352
367
        """Returns a list of the packages with the highest 'rating' based on
353
368
           the dampened rating calculated from the ReviewStats rating spread.
354
369
           Also optionally takes a category (string) to filter by"""
355
370
 
356
371
        cache = self.REVIEW_STATS_CACHE
357
 
        
 
372
 
358
373
        if category:
359
374
            applist = self._get_apps_for_category(category)
360
375
            cache = self._filter_cache_with_applist(cache, applist)
361
 
        
 
376
 
362
377
        #create a list of tuples with (Application,dampened_rating)
363
378
        dr_list = []
364
379
        for item in cache.items():
365
 
            if hasattr(item[1],'dampened_rating'):
 
380
            if hasattr(item[1], 'dampened_rating'):
366
381
                dr_list.append((item[0], item[1].dampened_rating))
367
382
            else:
368
383
                dr_list.append((item[0], 3.00))
369
 
        
 
384
 
370
385
        #sorted the list descending by dampened rating
371
386
        sorted_dr_list = sorted(dr_list, key=operator.itemgetter(1),
372
387
                                reverse=True)
373
 
        
 
388
 
374
389
        #return the quantity requested or as much as we can
375
390
        if quantity < len(sorted_dr_list):
376
391
            return_qty = quantity
377
392
        else:
378
393
            return_qty = len(sorted_dr_list)
379
 
        
 
394
 
380
395
        top_rated = []
381
 
        for i in range (0,return_qty):
 
396
        for i in range(0, return_qty):
382
397
            top_rated.append(sorted_dr_list[i][0])
383
 
        
 
398
 
384
399
        return top_rated
385
 
    
 
400
 
386
401
    def _filter_cache_with_applist(self, cache, applist):
387
402
        """Take the review cache and filter it to only include the apps that
388
403
           also appear in the applist passed in"""
391
406
            if key.pkgname in applist:
392
407
                filtered_cache[key] = cache[key]
393
408
        return filtered_cache
394
 
    
 
409
 
395
410
    def _get_apps_for_category(self, category):
396
411
        query = get_query_for_category(self.db, category)
397
412
        if not query:
398
413
            LOG.warn("_get_apps_for_category: received invalid category")
399
414
            return []
400
 
        
 
415
 
401
416
        pathname = os.path.join(XAPIAN_BASE_PATH, "xapian")
402
417
        db = StoreDatabase(pathname, self.cache)
403
418
        db.open()
404
419
        docs = db.get_docs_from_query(query)
405
 
        
 
420
 
406
421
        #from the db docs, return a list of pkgnames
407
422
        applist = []
408
423
        for doc in docs:
409
424
            applist.append(db.get_pkgname(doc))
410
425
        return applist
411
426
 
412
 
    def spawn_write_new_review_ui(self, translated_app, version, iconname, 
 
427
    def spawn_write_new_review_ui(self, translated_app, version, iconname,
413
428
                                  origin, parent_xid, datadir, callback):
414
429
        """Spawn the UI for writing a new review and adds it automatically
415
430
        to the reviews DB.
424
439
        """
425
440
        pass
426
441
 
427
 
    def spawn_submit_usefulness_ui(self, review_id, is_useful, parent_xid, datadir, callback):
 
442
    def spawn_submit_usefulness_ui(self, review_id, is_useful, parent_xid,
 
443
        datadir, callback):
428
444
        """Spawn a helper to submit a usefulness vote."""
429
445
        pass
430
446
 
432
448
        """Spawn a helper to delete a review."""
433
449
        pass
434
450
 
435
 
    def spawn_modify_review_ui(self, parent_xid, iconname, datadir, review_id, callback):
 
451
    def spawn_modify_review_ui(self, parent_xid, iconname, datadir, review_id,
 
452
        callback):
436
453
        """Spawn a helper to modify a review."""
437
454
        pass
438
455
 
439
456
 
440
457
class ReviewLoaderFake(ReviewLoader):
441
458
 
442
 
    USERS = ["Joe Doll", "John Foo", "Cat Lala", "Foo Grumpf", "Bar Tender", "Baz Lightyear"]
 
459
    USERS = ["Joe Doll", "John Foo", "Cat Lala", "Foo Grumpf", "Bar Tender",
 
460
        "Baz Lightyear"]
443
461
    SUMMARIES = ["Cool", "Medium", "Bad", "Too difficult"]
444
462
    IPSUM = "no ipsum\n\nstill no ipsum"
445
463
 
447
465
        ReviewLoader.__init__(self, cache, db)
448
466
        self._review_stats_cache = {}
449
467
        self._reviews_cache = {}
 
468
 
450
469
    def _random_person(self):
451
470
        return random.choice(self.USERS)
 
471
 
452
472
    def _random_text(self):
453
473
        return random.choice(self.LOREM.split("\n\n"))
 
474
 
454
475
    def _random_summary(self):
455
476
        return random.choice(self.SUMMARIES)
 
477
 
456
478
    def get_reviews(self, application, callback, page=1, language=None,
457
479
        sort=0, relaxed=False):
458
480
        if not application in self._review_stats_cache:
462
484
            reviews = []
463
485
            for i in range(0, stats.ratings_total):
464
486
                review = Review(application)
465
 
                review.id = random.randint(1,50000)
 
487
                review.id = random.randint(1, 50000)
466
488
                # FIXME: instead of random, try to match the avg_rating
467
 
                review.rating = random.randint(1,5)
 
489
                review.rating = random.randint(1, 5)
468
490
                review.summary = self._random_summary()
469
491
                review.date_created = time.strftime("%Y-%m-%d %H:%M:%S")
470
492
                review.reviewer_username = self._random_person()
471
 
                review.review_text = self._random_text().replace("\n","")
 
493
                review.review_text = self._random_text().replace("\n", "")
472
494
                review.usefulness_total = random.randint(1, 20)
473
495
                review.usefulness_favorable = random.randint(1, 20)
474
496
                reviews.append(review)
475
497
            self._reviews_cache[application] = reviews
476
498
        reviews = self._reviews_cache[application]
477
499
        callback(application, reviews)
 
500
 
478
501
    def get_review_stats(self, application):
479
502
        if not application in self._review_stats_cache:
480
503
            stat = ReviewStats(application)
481
 
            stat.ratings_average = random.randint(1,5)
482
 
            stat.ratings_total = random.randint(1,20)
 
504
            stat.ratings_average = random.randint(1, 5)
 
505
            stat.ratings_total = random.randint(1, 20)
483
506
            self._review_stats_cache[application] = stat
484
507
        return self._review_stats_cache[application]
 
508
 
485
509
    def refresh_review_stats(self, callback):
486
510
        review_stats = []
487
511
        callback(review_stats)
488
512
 
 
513
 
489
514
class ReviewLoaderFortune(ReviewLoaderFake):
490
515
    def __init__(self, cache, db):
491
516
        ReviewLoaderFake.__init__(self, cache, db)
492
517
        self.LOREM = ""
493
518
        for i in range(10):
494
 
            out = subprocess.Popen(["fortune"], stdout=subprocess.PIPE).communicate()[0]
 
519
            out = subprocess.Popen(["fortune"],
 
520
                stdout=subprocess.PIPE).communicate()[0]
495
521
            self.LOREM += "\n\n%s" % out
496
522
 
 
523
 
497
524
class ReviewLoaderTechspeak(ReviewLoaderFake):
498
525
    """ a test review loader that does not do any network io
499
526
        and returns random review texts
500
527
    """
501
 
    LOREM=u"""This package is using cloud based technology that will
 
528
    LOREM = u"""This package is using cloud based technology that will
502
529
make it suitable in a distributed environment where soup and xml-rpc
503
530
are used. The backend is written in C++ but the frontend code will
504
531
utilize dynamic languages lika LUA to provide a execution environment
547
574
your finger tips. This has limitless possibilities and will permeate
548
575
every facet of your life.  Believe the hype."""
549
576
 
 
577
 
550
578
class ReviewLoaderIpsum(ReviewLoaderFake):
551
579
    """ a test review loader that does not do any network io
552
580
        and returns random lorem ipsum review texts
554
582
    #This text is under public domain
555
583
    #Lorem ipsum
556
584
    #Cicero
557
 
    LOREM=u"""lorem ipsum "dolor" äöü sit amet consetetur sadipscing elitr sed diam nonumy
 
585
    LOREM = u"""lorem ipsum "dolor" äöü sit amet consetetur sadipscing elitr
 
586
sed diam nonumy
558
587
eirmod tempor invidunt ut labore et dolore magna aliquyam erat sed diam
559
588
voluptua at vero eos et accusam et justo duo dolores et ea rebum stet clita
560
589
kasd gubergren no sea takimata sanctus est lorem ipsum dolor sit amet lorem
632
661
        callback(application, [])
633
662
 
634
663
    def get_review_stats(self, application):
635
 
        return None
 
664
        pass
636
665
 
637
666
    def refresh_review_stats(self, callback):
638
667
        review_stats = []
640
669
 
641
670
 
642
671
review_loader = None
 
672
 
 
673
 
643
674
def get_review_loader(cache, db=None):
644
 
    """ 
 
675
    """
645
676
    factory that returns a reviews loader singelton
646
677
    """
647
678
    global review_loader
666
697
    def callback(app, reviews):
667
698
        print "app callback:"
668
699
        print app, reviews
 
700
 
669
701
    def stats_callback(stats):
670
702
        print "stats callback:"
671
703
        print stats
675
707
    cache = get_pkg_info()
676
708
    cache.open()
677
709
 
678
 
    db = StoreDatabase(XAPIAN_BASE_PATH+"/xapian", cache)
 
710
    db = StoreDatabase(XAPIAN_BASE_PATH + "/xapian", cache)
679
711
    db.open()
680
712
 
681
713
    # rnrclient loader
682
714
    app = Application("ACE", "unace")
683
715
    #app = Application("", "2vcard")
684
716
 
685
 
    from softwarecenter.backend.reviews.rnr import ReviewLoaderSpawningRNRClient
 
717
    from softwarecenter.backend.reviews.rnr import (
 
718
        ReviewLoaderSpawningRNRClient
 
719
    )
686
720
    loader = ReviewLoaderSpawningRNRClient(cache, db)
687
721
    print loader.refresh_review_stats(stats_callback)
688
722
    print loader.get_reviews(app, callback)
694
728
    main.run()
695
729
 
696
730
    # default loader
697
 
    app = Application("","2vcard")
 
731
    app = Application("", "2vcard")
698
732
    loader = get_review_loader(cache, db)
699
733
    loader.refresh_review_stats(stats_callback)
700
734
    loader.get_reviews(app, callback)