← Back to branch summary

~ubuntu-branches/ubuntu/karmic/calibre/karmic

« back to all changes in this revision

Viewing changes to src/calibre/ebooks/lrf/web/profiles/chr_mon.py

  • Committer: Bazaar Package Importer
  • Author(s): Martin Pitt
  • Date: 2009-07-30 12:49:41 UTC
  • mto: This revision was merged to the branch mainline in revision 13.
  • Revision ID: james.westby@ubuntu.com-20090730124941-kviipg9ypwgppulc
Tags: upstream-0.6.3+dfsg
Import upstream version 0.6.3+dfsg

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/calibre/ebooks/lrf/web/profiles/chr_mon.py
1
 
 
2
 
import re, time
3
 
from calibre.ebooks.lrf.web.profiles import DefaultProfile
4
 
from calibre.ebooks.BeautifulSoup import BeautifulSoup
5
 
 
6
 
class ChristianScienceMonitor(DefaultProfile):
7
 
 
8
 
    title = 'Christian Science Monitor'
9
 
    max_recursions = 2
10
 
    max_articles_per_feed = 20
11
 
    no_stylesheets = True
12
 
    
13
 
  
14
 
 
15
 
    
16
 
    preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in 
17
 
        [
18
 
        (r'<body.*?<div id="story"', lambda match : '<body><div id="story"'),
19
 
        (r'<div class="pubdate">.*?</div>', lambda m: ''),
20
 
        (r'Full HTML version of this story which may include photos, graphics, and related links.*</body>',
21
 
              lambda match : '</body>'),
22
 
        ]]
23
 
     
24
 
 
25
 
    def parse_feeds(self):
26
 
        soup = BeautifulSoup(self.browser.open('http://www.csmonitor.com/textedition'))
27
 
        articles = {}
28
 
        feed = []
29
 
        for tag in soup.findAll(['h2', 'p']):
30
 
            if tag.name == 'h2':
31
 
                title = self.tag_to_string(tag)
32
 
                feed = [] 
33
 
                articles[title] = feed
34
 
            elif tag.has_key('class') and tag['class'] == 'story':
35
 
                a = tag.find('a')
36
 
                if a is not None and a.has_key('href'):
37
 
                    feed.append({
38
 
                         'title': self.tag_to_string(a),
39
 
                         'url'  : 'http://www.csmonitor.com'+a['href'],
40
 
                         'date' : time.strftime('%d %b'),
41
 
                         'content' : '',
42
 
                         })
43
 
                    a.extract()
44
 
                    feed[-1]['description'] = self.tag_to_string(tag).strip()
45
 
        return articles
46