← Back to branch summary

~facundo/encuentro/trunk

« back to all changes in this revision

Viewing changes to server/get_conect_episodes.py

  • Committer: Facundo Batista
  • Date: 2013-06-01 18:55:04 UTC
  • mto: This revision was merged to the branch mainline in revision 178.
  • Revision ID: facundo@taniquetil.com.ar-20130601185504-i3ygczru97g9b45m
https://launchpad.net/bugs/1063498
Better logging.

Show diffs side-by-side

added added

removed removed

Lines of Context:
server/get_conect_episodes.py
1
1
# -*- coding: utf8 -*-
2
2
 
3
 
# Copyright 2012 Facundo Batista
 
3
# Copyright 2013 Facundo Batista
4
4
#
5
5
# This program is free software: you can redistribute it and/or modify it
6
6
# under the terms of the GNU General Public License version 3, as published
18
18
 
19
19
"""Main server process to get all info from Conectate web site."""
20
20
 
 
21
import logging
 
22
import sys
21
23
import urllib2
22
24
 
23
25
# we execute this script from inside the directory; pylint: disable=W0403
24
26
import helpers
25
27
import scrapers_conect
 
28
import srv_logger
26
29
 
27
30
 
28
31
# different channels from where read content
54
57
    "&pagina=%(page_number)d&modo=Lista"
55
58
)
56
59
 
 
60
logger = logging.getLogger("Conectate")
57
61
 
58
62
episodes_cache = helpers.Cache("episodes_cache_conect.pickle")
59
63
 
60
64
 
61
 
@helpers.retryable
 
65
@helpers.retryable(logger)
62
66
def _search(url):
63
67
    """Search each page."""
64
68
    u = urllib2.urlopen(url)
69
73
 
70
74
def do_search(channel_id, emis_id):
71
75
    """Search the web site."""
72
 
    print "Searching channel=%d  emission=%d" % (channel_id, emis_id)
 
76
    logger.info("Searching channel=%d  emission=%d", channel_id, emis_id)
73
77
    all_items = []
74
78
    page = 1
75
79
    while True:
76
 
        print "    page", page
 
80
        logger.info("    page %s", page)
77
81
        d = dict(channel_id=channel_id, emission_id=emis_id, page_number=page)
78
82
        url = URL_SEARCH % d
79
83
        items = _search(url)
80
84
        if not items:
81
85
            # done, return collected info
82
 
            print "      done:", len(all_items)
 
86
            logger.info("      done: %d", len(all_items))
83
87
            return all_items
84
88
 
85
89
        # store and go for next page
87
91
        page += 1
88
92
 
89
93
 
90
 
@helpers.retryable
 
94
@helpers.retryable(logger)
91
95
def get_from_series(i, url):
92
96
    """Get the episodes from an url page."""
93
 
    print "Get from series:", i, url
 
97
    logger.info("Get from series: %s %r", i, url)
94
98
    u = urllib2.urlopen(url)
95
99
    page = u.read()
96
100
    results = scrapers_conect.scrap_series(page)
97
 
    print "   ", len(results)
 
101
    logger.info("   %d", len(results))
98
102
    return results
99
103
 
100
104
 
101
 
@helpers.retryable
 
105
@helpers.retryable(logger)
102
106
def get_episode_info(i, url):
103
107
    """Get the info from an episode."""
104
 
    print "Get episode info:", i, url
 
108
    logger.info("Get episode info: %s %r", i, url)
105
109
    try:
106
110
        info = episodes_cache.get(url)
107
 
        print "    cached!"
 
111
        logger.info("    cached!")
108
112
    except KeyError:
109
113
        u = urllib2.urlopen(url)
110
114
        page = u.read()
111
115
        info = scrapers_conect.scrap_video(page)
112
116
        episodes_cache.set(url, info)
113
 
        print "    ok"
 
117
        logger.info("    ok")
114
118
    return info
115
119
 
116
120
 
125
129
                for i, (_, url) in enumerate(results):
126
130
                    episodes.extend(get_from_series(i, URL_BASE + url))
127
131
                results = episodes
128
 
                print "Series collected:", len(results)
 
132
                logger.info("Series collected: %d", len(results))
129
133
 
130
134
            # inform each
131
135
            for i, (title, url) in enumerate(results):
152
156
    helpers.save_file("conectar-v03", all_data)
153
157
 
154
158
 
155
 
main()
 
159
if __name__ == '__main__':
 
160
    shy = len(sys.argv) > 1 and sys.argv[1] == '--shy'
 
161
    srv_logger.setup_log(shy)
 
162
    main()