3
from tempfile import mkstemp, gettempdir
4
from datetime import datetime
7
from miro import config
8
from miro import feedparser
10
from miro import dialogs
11
from miro import database
12
from miro import storedatabase
13
from miro import feedparserutil
14
from miro.plat import resources
15
from miro.item import Item
16
from miro.feed import validate_feed_url, normalize_feed_url, Feed
18
from miro.test.framework import MiroTestCase, EventLoopTest
23
class AcceptScrapeTestDelegate:
27
def run_dialog(self, dialog):
29
if not isinstance(dialog, dialogs.ChoiceDialog):
30
raise AssertionError("Only expected ChoiceDialogs")
31
if not dialog.title.startswith("Channel is not compatible"):
32
raise AssertionError("Only expected scrape dialogs")
33
dialog.choice = dialogs.BUTTON_YES
34
dialog.callback(dialog)
36
class FeedURLValidationTest(MiroTestCase):
37
def test_positive(self):
38
for testurl in [u"http://foo.bar.com/",
39
u"https://foo.bar.com/",
41
self.assertEqual(validate_feed_url(testurl), True)
43
def test_negative(self):
44
for testurl in [u"feed://foo.bar.com/",
45
u"http://foo.bar.com",
47
u"https:foo.bar.com/",
49
u"http:/foo.bar.com/",
50
u"https:/foo.bar.com/",
51
u"feed:/foo.bar.com/",
52
u"http:///foo.bar.com/",
53
u"https:///foo.bar.com/",
54
u"feed:///foo.bar.com/",
58
u"crap://foo.bar.com",
59
u"crap:///foo.bar.com",
61
self.assertEqual(validate_feed_url(testurl), False)
63
# FIXME - add tests for all the other kinds of urls that
64
# validate_feed_url handles.
66
class FeedURLNormalizationTest(MiroTestCase):
68
for i, o in [(u"http://foo.bar.com", u"http://foo.bar.com/"),
69
(u"https://foo.bar.com", u"https://foo.bar.com/"),
70
(u"feed://foo.bar.com", u"http://foo.bar.com/")
72
self.assertEqual(normalize_feed_url(i), o)
74
def test_garbage(self):
75
for i, o in [(u"http:foo.bar.com", u"http://foo.bar.com/"),
76
(u"https:foo.bar.com", u"https://foo.bar.com/"),
77
(u"feed:foo.bar.com", u"http://foo.bar.com/"),
78
(u"http:/foo.bar.com", u"http://foo.bar.com/"),
79
(u"https:/foo.bar.com", u"https://foo.bar.com/"),
80
(u"feed:/foo.bar.com", u"http://foo.bar.com/"),
81
(u"http:///foo.bar.com", u"http://foo.bar.com/"),
82
(u"https:///foo.bar.com", u"https://foo.bar.com/"),
83
(u"feed:///foo.bar.com", u"http://foo.bar.com/"),
84
(u"foo.bar.com", u"http://foo.bar.com/"),
85
(u"http://foo.bar.com:80", u"http://foo.bar.com:80/"),
87
self.assertEquals(normalize_feed_url(i), o)
89
# FIXME - add tests for all the other kinds of feeds that
90
# normalize_feed_url handles.
92
class FeedTestCase(EventLoopTest):
94
EventLoopTest.setUp(self)
95
self.filename = self.make_temp_path()
97
def write_file(self, content):
98
self.url = u'file://%s' % self.filename
99
handle = file(self.filename,"wb")
100
# RSS 2.0 example feed
101
# http://cyber.law.harvard.edu/blogs/gems/tech/rss2sample.xml
102
handle.write(content)
105
def update_feed(self, feed):
107
self.processThreads()
111
feed = Feed(self.url)
112
self.update_feed(feed)
115
class SimpleFeedTestCase(FeedTestCase):
117
FeedTestCase.setUp(self)
119
# http://cyber.law.harvard.edu/blogs/gems/tech/rss2sample.xml
121
# this rss feed has no enclosures.
122
self.write_file("""<?xml version="1.0"?>
125
<title>Liftoff News</title>
126
<link>http://liftoff.msfc.nasa.gov/</link>
127
<description>Liftoff to Space Exploration.</description>
128
<language>en-us</language>
129
<pubDate>Tue, 10 Jun 2003 04:00:00 GMT</pubDate>
131
<lastBuildDate>Tue, 10 Jun 2003 09:41:01 GMT</lastBuildDate>
132
<docs>http://blogs.law.harvard.edu/tech/rss</docs>
133
<generator>Weblog Editor 2.0</generator>
134
<managingEditor>editor@example.com</managingEditor>
135
<webMaster>webmaster@example.com</webMaster>
137
<title>Star City</title>
138
<link>http://liftoff.msfc.nasa.gov/news/2003/news-starcity.mov</link>
139
<description>How do Americans get ready to work with Russians aboard the International Space Station? They take a crash course in culture, language and protocol at Russia's <a href="http://howe.iki.rssi.ru/GCTC/gctc_e.htm">Star City</a>.</description>
140
<pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
141
<guid>http://liftoff.msfc.nasa.gov/2003/06/03.html#item573</guid>
144
<description>Sky watchers in Europe, Asia, and parts of Alaska and Canada will experience a <a href="http://science.nasa.gov/headlines/y2003/30may_solareclipse.htm">partial eclipse of the Sun</a> on Saturday, May 31st.</description>
145
<pubDate>Fri, 30 May 2003 11:06:42 GMT</pubDate>
146
<guid>http://liftoff.msfc.nasa.gov/2003/05/30.html#item572</guid>
149
<title>The Engine That Does More</title>
150
<link>http://liftoff.msfc.nasa.gov/news/2003/news-VASIMR.asp</link>
151
<description>Before man travels to Mars, NASA hopes to design new engines that will let us fly through the Solar System more quickly. The proposed VASIMR engine would do that.</description>
152
<pubDate>Tue, 27 May 2003 08:37:32 GMT</pubDate>
153
<guid>http://liftoff.msfc.nasa.gov/2003/05/27.html#item571</guid>
156
<title>Astronauts' Dirty Laundry</title>
157
<link>http://liftoff.msfc.nasa.gov/news/2003/news-laundry.asp</link>
158
<description>Compared to earlier spacecraft, the International Space Station has many luxuries, but laundry facilities are not one of them. Instead, astronauts have other options.</description>
159
<pubDate>Tue, 20 May 2003 08:56:02 GMT</pubDate>
160
<guid>http://liftoff.msfc.nasa.gov/2003/05/20.html#item570</guid>
166
dialogs.delegate = AcceptScrapeTestDelegate()
167
my_feed = self.make_feed()
169
# the feed has no enclosures, but we now insert enclosures into it.
170
# thus it should not cause a dialog to pop up and ask the user if they
172
self.assertEqual(dialogs.delegate.calls, 0)
173
# the Feed, plus the 1 item that is a video
174
items = list(Item.make_view())
175
self.assertEqual(len(items), 1)
177
# make sure that re-updating doesn't re-create the items
179
items = list(Item.make_view())
180
self.assertEqual(len(items), 1)
183
class MultiFeedExpireTest(FeedTestCase):
184
def write_files(self, subfeed_count, feed_item_count):
188
content = self.make_feed_content(feed_item_count)
189
for i in xrange(subfeed_count):
190
filename = self.make_temp_path()
191
open(filename, 'wb').write(content)
192
all_urls.append(u"file://%s" % filename)
193
self.filenames.append(filename)
195
self.url = u'dtv:multi:' + ','.join(all_urls) + "," + 'testquery'
197
def rewrite_files(self, feed_item_count):
198
content = self.make_feed_content(feed_item_count)
199
for filename in self.filenames:
200
open(filename, 'wb').write(content)
202
def make_feed_content(self, entry_count):
203
# make a feed with a new item and parse it
207
items.append("""<?xml version="1.0"?>
210
<title>Downhill Battle Pics</title>
211
<link>http://downhillbattle.org/</link>
212
<description>Downhill Battle is a non-profit organization working to support participatory culture and build a fairer music industry.</description>
213
<pubDate>Wed, 16 Mar 2005 12:03:42 EST</pubDate>
216
for x in range(entry_count):
220
<title>Bumper Sticker</title>
222
<enclosure url="http://downhillbattle.org/key/gallery/%s.mpg" />
223
<description>I'm a musician and I support filesharing.</description>
225
""" % (counter, counter))
230
return "".join(items)
232
def test_multi_feed_expire(self):
233
# test what happens when a RSSMultiFeed has feeds that
234
# reference the same item, and they are truncated at the same
237
self.write_files(5, 10) # 5 feeds containing 10 identical items
238
self.feed = self.make_feed()
239
config.set(prefs.TRUNCATE_CHANNEL_AFTER_X_ITEMS, 4)
240
config.set(prefs.MAX_OLD_ITEMS_DEFAULT, 5)
241
self.rewrite_files(1) # now only 5 items in each feed
242
self.update_feed(self.feed)
244
class EnclosureFeedTestCase(FeedTestCase):
246
FeedTestCase.setUp(self)
247
self.write_file("""<?xml version="1.0"?>
250
<title>Downhill Battle Pics</title>
251
<link>http://downhillbattle.org/</link>
252
<description>Downhill Battle is a non-profit organization working to support participatory culture and build a fairer music industry.</description>
253
<pubDate>Wed, 16 Mar 2005 12:03:42 EST</pubDate>
255
<title>Bumper Sticker</title>
256
<enclosure url="http://downhillbattle.org/key/gallery/chriscA.mpg" />
257
<description>I'm a musician and I support filesharing.</description>
261
<title>T-shirt</title>
262
<enclosure url="http://downhillbattle.org/key/gallery/payola_tshirt.mpg" />
265
<enclosure url="http://downhillbattle.org/key/gallery/chriscE.mpg" />
266
<description>Flyer in Yucaipa, CA</description>
269
<enclosure url="http://downhillbattle.org/key/gallery/jalabel_nov28.mpg" />
272
<enclosure url="http://downhillbattle.org/key/gallery/jalabel_nov28.jpg" />
279
my_feed = self.make_feed()
280
items = list(Item.make_view())
281
self.assertEqual(len(items), 4)
282
# make sure that re-updating doesn't re-create the items
284
items = list(Item.make_view())
285
self.assertEqual(len(items), 4)
288
class OldItemExpireTest(FeedTestCase):
289
# Test that old items expire when the feed gets too big
291
FeedTestCase.setUp(self)
293
self.write_new_feed()
294
self.feed = self.make_feed()
295
config.set(prefs.TRUNCATE_CHANNEL_AFTER_X_ITEMS, 4)
296
config.set(prefs.MAX_OLD_ITEMS_DEFAULT, 20)
298
def write_new_feed(self, entryCount=2):
299
# make a feed with a new item and parse it
302
items.append("""<?xml version="1.0"?>
305
<title>Downhill Battle Pics</title>
306
<link>http://downhillbattle.org/</link>
307
<description>Downhill Battle is a non-profit organization working to support participatory culture and build a fairer music industry.</description>
308
<pubDate>Wed, 16 Mar 2005 12:03:42 EST</pubDate>
311
for x in range(entryCount):
315
<title>Bumper Sticker</title>
317
<enclosure url="http://downhillbattle.org/key/gallery/%s.mpg" />
318
<description>I'm a musician and I support filesharing.</description>
320
""" % (self.counter, self.counter))
325
self.write_file("\n".join(items))
327
def check_guids(self, *ids):
329
for i in Item.make_view():
330
actual.add(i.get_rss_id())
331
correct = set(['guid-%d' % i for i in ids])
332
self.assertEquals(actual, correct)
334
def parse_new_feed(self, entryCount=2):
335
self.write_new_feed(entryCount)
336
self.update_feed(self.feed)
338
def test_simple_overflow(self):
339
self.assertEqual(Item.make_view().count(), 2)
340
self.parse_new_feed()
341
self.assertEqual(Item.make_view().count(), 4)
342
self.parse_new_feed()
343
self.assertEqual(Item.make_view().count(), 4)
344
self.check_guids(3, 4, 5, 6)
346
def test_overflow_with_downloads(self):
347
items = list(Item.make_view())
348
items[0]._downloader = FakeDownloader()
349
items[1]._downloader = FakeDownloader()
350
self.assertEqual(len(items), 2)
351
self.parse_new_feed()
352
self.parse_new_feed()
353
self.check_guids(1, 2, 5, 6)
355
def test_overflow_still_in_feed(self):
356
config.set(prefs.TRUNCATE_CHANNEL_AFTER_X_ITEMS, 0)
357
self.parse_new_feed(6)
358
self.check_guids(3, 4, 5, 6, 7, 8)
360
def test_overflow_with_replacement(self):
361
# Keep item with guid-2 in the feed.
362
config.set(prefs.TRUNCATE_CHANNEL_AFTER_X_ITEMS, 0)
364
self.parse_new_feed(5)
365
self.check_guids(2, 3, 4, 5, 6)
367
def test_overflow_with_max_old_items(self):
368
config.set(prefs.TRUNCATE_CHANNEL_AFTER_X_ITEMS, 1000) # don't bother
369
self.assertEqual(Item.make_view().count(), 2)
370
self.parse_new_feed()
371
self.assertEquals(Item.make_view().count(), 4)
372
self.parse_new_feed()
373
self.feed.setMaxOldItems(4)
374
self.feed.actualFeed.clean_old_items()
375
while self.feed.actualFeed.updating:
376
self.processThreads()
379
self.assertEquals(Item.make_view().count(), 6)
380
self.feed.setMaxOldItems(2)
381
self.feed.actualFeed.clean_old_items()
382
while self.feed.actualFeed.updating:
383
self.processThreads()
386
self.assertEquals(Item.make_view().count(), 4)
387
self.check_guids(3, 4, 5, 6)
389
def test_overflow_with_global_max_old_items(self):
390
config.set(prefs.TRUNCATE_CHANNEL_AFTER_X_ITEMS, 1000) # don't bother
391
self.assertEqual(Item.make_view().count(), 2)
392
self.parse_new_feed()
393
self.assertEquals(Item.make_view().count(), 4)
394
self.parse_new_feed()
395
config.set(prefs.MAX_OLD_ITEMS_DEFAULT, 4)
396
self.feed.actualFeed.clean_old_items()
397
while self.feed.actualFeed.updating:
398
self.processThreads()
401
self.assertEquals(Item.make_view().count(), 6)
402
config.set(prefs.MAX_OLD_ITEMS_DEFAULT, 2)
403
self.feed.actualFeed.clean_old_items()
404
while self.feed.actualFeed.updating:
405
self.processThreads()
408
self.assertEquals(Item.make_view().count(), 4)
409
self.check_guids(3, 4, 5, 6)
411
class FeedParserAttributesTestCase(FeedTestCase):
412
"""Test that we save/restore attributes from feedparser correctly.
414
We don't store feedparser dicts in the database. This test case is
415
checking if the values we to the database are the same as the original
416
ones from feedparser.
419
FeedTestCase.setUp(self)
420
self.tempdb = os.path.join(gettempdir(), 'democracy-temp-db')
421
if os.path.exists(self.tempdb):
422
os.remove(self.tempdb)
423
self.reload_database(self.tempdb)
425
self.parsed_feed = feedparser.parse(self.filename)
427
self.save_then_restore_db()
430
self.runPendingIdles()
431
self.shutdown_database()
432
os.remove(self.tempdb)
433
FeedTestCase.tearDown(self)
435
def save_then_restore_db(self):
436
self.reload_database(self.tempdb)
437
self.feed = Feed.make_view().get_singleton()
438
self.item = Item.make_view().get_singleton()
440
def write_feed(self):
441
self.write_file("""<?xml version="1.0"?>
444
<title>Downhill Battle Pics</title>
445
<link>http://downhillbattle.org/</link>
446
<description>Downhill Battle is a non-profit organization working to support participatory culture and build a fairer music industry.</description>
447
<pubDate>Wed, 16 Mar 2005 12:03:42 EST</pubDate>
450
<title>Bumper Sticker</title>
451
<link>http://downhillbattle.org/item</link>
452
<comments>http://downhillbattle.org/item/comments</comments>
453
<creativeCommons:license>http://www.creativecommons.org/licenses/by-nd/1.0</creativeCommons:license>
454
<guid>guid-1234</guid>
455
<enclosure url="http://downhillbattle.org/key/gallery/movie.mpg"
459
<description>I'm a musician and I support filesharing.</description>
460
<pubDate>Fri, 18 Mar 2005 12:03:42 EST</pubDate>
461
<media:thumbnail url="%(thumburl)s" />
462
<dtv:paymentlink url="http://www.example.com/payment.html" />
467
""" % {'thumburl': resources.url("testdata/democracy-now-unicode-bug.xml")})
470
def test_attributes(self):
471
entry = self.parsed_feed.entries[0]
472
self.assertEquals(self.item.get_rss_id(), entry.id)
473
self.assertEquals(self.item.get_thumbnail_url(), entry.thumbnail['url'])
474
self.assertEquals(self.item.get_title(), entry.title)
475
self.assertEquals(self.item.get_raw_description(), entry.description)
476
self.assertEquals(self.item.get_link(), entry.link)
477
self.assertEquals(self.item.get_payment_link(), entry.payment_url)
478
self.assertEquals(self.item.get_license(), entry.license)
479
self.assertEquals(self.item.get_comments_link(), entry.comments)
481
enclosure = entry.enclosures[0]
482
self.assertEquals(self.item.get_url(), enclosure.url)
483
self.assertEquals(self.item.get_size(), int(enclosure.length))
484
self.assertEquals(self.item.get_format(), '.mpeg')
486
def test_remove_rssid(self):
487
self.item.remove_rss_id()
488
self.save_then_restore_db()
489
self.assertEquals(self.item.get_rss_id(), None)
491
def test_change_title(self):
492
entry = self.parsed_feed.entries[0]
493
self.item.set_title(u"new title")
494
self.save_then_restore_db()
495
self.assertEquals(self.item.get_title(), "new title")
497
if __name__ == "__main__":
added
removed
portable/test/feedtest.py
