← Back to branch summary

~facundo/encuentro/trunk

« back to all changes in this revision

Viewing changes to encuentro/network.py

  • Committer: facundo at com
  • Date: 2012-12-17 02:39:03 UTC
  • Revision ID: facundo@taniquetil.com.ar-20121217023903-ofg583jcnu3q5lli
Bacua integration in the client.

Show diffs side-by-side

added added

removed removed

Lines of Context:
encuentro/network.py
24
24
import sys
25
25
import time
26
26
import urllib
 
27
import urlparse
27
28
 
28
29
if sys.platform == 'win32':
29
30
    # multiprocessing path mangling is not useful for how we are
41
42
from encuentro import platform
42
43
 
43
44
from twisted.internet import defer, reactor
 
45
from twisted.web.client import HTTPDownloader, PartialDownloadError
44
46
 
45
47
URL_BASE = "http://conectate.gov.ar"
46
48
URL_AUTH = ("http://conectate.gov.ar/educar-portal-video-web/"
199
201
        return d
200
202
 
201
203
 
202
 
class Downloader(object):
203
 
    """Episode downloader."""
 
204
class BaseDownloader(object):
 
205
    """Base episode downloader."""
204
206
 
205
207
    def __init__(self, config):
206
208
        self.config = config
 
209
 
 
210
    def shutdown(self):
 
211
        """Quit the download."""
 
212
        return self._shutdown()
 
213
 
 
214
    def cancel(self):
 
215
        """Cancel a download."""
 
216
        return self._cancel()
 
217
 
 
218
    def _setup_target(self, channel, section, title, extension):
 
219
        """Set up the target file to download."""
 
220
        # build where to save it
 
221
        downloaddir = self.config.get('downloaddir', '')
 
222
        channel = platform.sanitize(channel)
 
223
        section = platform.sanitize(section)
 
224
        title = platform.sanitize(title)
 
225
        fname = os.path.join(downloaddir, channel, section, title + extension)
 
226
 
 
227
        # if the directory doesn't exist, create it
 
228
        dirsecc = os.path.dirname(fname)
 
229
        if not os.path.exists(dirsecc):
 
230
            os.makedirs(dirsecc)
 
231
 
 
232
        tempf = fname + str(time.time())
 
233
        return fname, tempf
 
234
 
 
235
    def download(self, channel, section, title, url, cb_progress):
 
236
        """Download an episode."""
 
237
        return self._download(channel, section, title, url, cb_progress)
 
238
 
 
239
 
 
240
class ConectarDownloader(BaseDownloader):
 
241
    """Episode downloader for Conectar site."""
 
242
 
 
243
    def __init__(self, config):
 
244
        super(ConectarDownloader, self).__init__(config)
207
245
        self._prev_progress = None
208
246
        self.browser_quit = set()
209
 
        logger.info("Downloader inited")
210
247
        self.cancelled = False
 
248
        logger.info("Conectar downloader inited")
211
249
 
212
 
    def shutdown(self):
 
250
    def _shutdown(self):
213
251
        """Quit the download."""
214
252
        for bquit in self.browser_quit:
215
253
            bquit.set()
216
 
        logger.info("Downloader shutdown finished")
 
254
        logger.info("Conectar downloader shutdown finished")
217
255
 
218
 
    def cancel(self):
 
256
    def _cancel(self):
219
257
        """Cancel a download."""
220
258
        self.cancelled = True
 
259
        logger.info("Conectar downloader cancelled")
221
260
 
222
261
    @defer.inlineCallbacks
223
 
    def download(self, canal, seccion, titulo, url, cb_progress):
 
262
    def _download(self, canal, seccion, titulo, url, cb_progress):
224
263
        """Descarga una emisión a disco."""
225
264
        self.cancelled = False
226
265
 
237
276
        brow.start()
238
277
 
239
278
        # build where to save it
240
 
        downloaddir = self.config.get('downloaddir', '')
241
 
        canal = platform.sanitize(canal)
242
 
        seccion = platform.sanitize(seccion)
243
 
        titulo = platform.sanitize(titulo)
244
 
        fname = os.path.join(downloaddir, canal, seccion, titulo + u".avi")
245
 
 
246
 
        # ver si esa seccion existe, sino crearla
247
 
        dirsecc = os.path.dirname(fname)
248
 
        if not os.path.exists(dirsecc):
249
 
            os.makedirs(dirsecc)
250
 
 
251
 
        # descargamos en un temporal
252
 
        tempf = fname + str(time.time())
 
279
        fname, tempf = self._setup_target(canal, seccion, titulo, u".avi")
253
280
        logger.debug("Downloading to temporal file %r", tempf)
254
281
        qoutput.put(tempf)
255
282
 
284
311
        defer.returnValue(fname)
285
312
 
286
313
 
 
314
class GenericDownloader(BaseDownloader):
 
315
    """Episode downloader for a generic site that works with urllib2."""
 
316
 
 
317
    headers = {
 
318
        'User-Agent': 'Mozilla/5.0',
 
319
        'Accept': '*/*',
 
320
    }
 
321
 
 
322
    def __init__(self, config):
 
323
        super(GenericDownloader, self).__init__(config)
 
324
        self._prev_progress = None
 
325
        self.downloader = None
 
326
        logger.info("Generic downloader inited")
 
327
 
 
328
    def _shutdown(self):
 
329
        """Quit the download."""
 
330
        logger.info("Generic downloader shutdown finished")
 
331
 
 
332
    def _cancel(self):
 
333
        """Cancel a download."""
 
334
        if self.downloader is not None:
 
335
            self.downloader.cancel()
 
336
            logger.info("Generic downloader cancelled")
 
337
 
 
338
    def _parse_url(self, url):
 
339
        """Return host and port from the URL."""
 
340
        urlparts = urlparse.urlparse(url)
 
341
        if urlparts.port is None:
 
342
            if urlparts.scheme == 'http':
 
343
                port = 80
 
344
            elif urlparts.scheme == 'http':
 
345
                port = 80
 
346
            else:
 
347
                raise ValueError("Unknown schema when guessing port: " +
 
348
                                 repr(urlparts.scheme))
 
349
        else:
 
350
            port = int(urlparts.port)
 
351
        return urlparts.hostname, port
 
352
 
 
353
    @defer.inlineCallbacks
 
354
    def _download(self, canal, seccion, titulo, url, cb_progress):
 
355
        """Download an episode to disk."""
 
356
        url = str(url)
 
357
        logger.info("Download episode %r", url)
 
358
 
 
359
        def report(dloaded, total):
 
360
            """Report download."""
 
361
            size_mb = total // 1024 ** 2
 
362
            perc = dloaded * 100.0 / total
 
363
            m = "%.1f%% (de %d MB)" % (perc, size_mb)
 
364
            if m != self._prev_progress:
 
365
                cb_progress(m)
 
366
                self._prev_progress = m
 
367
 
 
368
        class ReportingDownloader(HTTPDownloader):
 
369
            """Customize HTTPDownloader to also report and can be cancelled."""
 
370
            def __init__(self, *args, **kwrgs):
 
371
                self.content_length = None
 
372
                self.downloaded = 0
 
373
                self.connected_client = None
 
374
                self.cancelled = False
 
375
                HTTPDownloader.__init__(self, *args, **kwrgs)
 
376
 
 
377
            def gotHeaders(self, headers):
 
378
                """Got headers."""
 
379
                clength = headers.get("content-length", [None])[0]
 
380
                self.content_length = int(clength)
 
381
                HTTPDownloader.gotHeaders(self, headers)
 
382
 
 
383
            def pagePart(self, data):
 
384
                """Got part of content."""
 
385
                self.downloaded += len(data)
 
386
                report(self.downloaded, self.content_length)
 
387
                HTTPDownloader.pagePart(self, data)
 
388
 
 
389
            def cancel(self):
 
390
                """Cancel."""
 
391
                self.cancelled = True
 
392
                if self.connected_client is not None:
 
393
                    self.connected_client.stopProducing()
 
394
 
 
395
            def buildProtocol(self, addr):
 
396
                """Store the protocol built."""
 
397
                p = HTTPDownloader.buildProtocol(self, addr)
 
398
                self.connected_client = p
 
399
                if self.cancelled:
 
400
                    p.stopProducing()
 
401
                return p
 
402
 
 
403
        # build where to save it
 
404
        fname, tempf = self._setup_target(canal, seccion, titulo, u".mp4")
 
405
        logger.debug("Downloading to temporal file %r", tempf)
 
406
 
 
407
        self.downloader = ReportingDownloader(url, tempf, headers=self.headers)
 
408
        host, port = self._parse_url(url)
 
409
        reactor.connectTCP(host, port, self.downloader)
 
410
        try:
 
411
            yield self.downloader.deferred
 
412
        except PartialDownloadError:
 
413
            if self.downloader.cancelled:
 
414
                raise CancelledError
 
415
            else:
 
416
                raise
 
417
 
 
418
        # rename to final name and end
 
419
        logger.info("Downloading done, renaming temp to %r", fname)
 
420
        os.rename(tempf, fname)
 
421
        defer.returnValue(fname)
 
422
 
 
423
 
 
424
# this is the entry point to get the downloaders for each type
 
425
all_downloaders = {
 
426
    None: ConectarDownloader,
 
427
    'conectar': ConectarDownloader,
 
428
    'generic': GenericDownloader,
 
429
}
 
430
 
 
431
 
287
432
if __name__ == "__main__":
288
433
    h = logging.StreamHandler()
289
434
    h.setLevel(logging.DEBUG)
297
442
    test_config = dict(user="lxpdvtnvrqdoa@mailinator.com",
298
443
                       password="descargas", downloaddir='.')
299
444
 
300
 
    url_episode = "http://conectate.gov.ar/educar-portal-video-web/module/"\
301
 
                  "detalleRecurso/DetalleRecurso.do?modulo=masVotados&"\
302
 
                  "recursoPadreId=50001&idRecurso=50004"
 
445
#    url_episode = "http://conectate.gov.ar/educar-portal-video-web/module/"\
 
446
#                  "detalleRecurso/DetalleRecurso.do?modulo=masVotados&"\
 
447
#                  "recursoPadreId=50001&idRecurso=50004"
 
448
    url_episode = "http://backend.bacua.gob.ar/video.php?v=_173fb17c"
303
449
 
304
450
    @defer.inlineCallbacks
305
451
    def download():
306
452
        """Download."""
307
 
        downloader = Downloader(test_config)
308
 
#        reactor.callLater(10, downloader.cancel)
 
453
#        downloader = ConectarDownloader(test_config)
 
454
        downloader = GenericDownloader(test_config)
 
455
#        reactor.callLater(5, downloader.cancel)
309
456
        try:
310
457
            fname = yield downloader.download("test-ej-canal", "secc", "tit",
311
458
                                              url_episode, show)