~jstys-z/helioviewer.org/client5

« back to all changes in this revision

Viewing changes to install/hvpull/downloader/urllib.py

  • Committer: Keith Hughitt
  • Date: 2012-04-23 16:02:25 UTC
  • mto: This revision was merged to the branch mainline in revision 732.
  • Revision ID: keith.hughitt@nasa.gov-20120423160225-xzoh82ejf37c8yr7
Incorporated HVPull code

Show diffs side-by-side

added added

removed removed

Lines of Context:
 
1
"""urllib2-based file downloader"""
 
2
import os
 
3
import logging
 
4
import threading
 
5
from urllib2 import urlopen, Request # URLError, HTTPError
 
6
 
 
7
class URLLibDownloader(threading.Thread):
 
8
    def __init__(self, image_archive, working_dir, base_url, queue):
 
9
        """Creates a new URLLibDownloader"""
 
10
        threading.Thread.__init__(self)
 
11
        
 
12
        self.shutdown_requested = False
 
13
        
 
14
        self.image_archive = image_archive
 
15
        self.working_dir = working_dir
 
16
        self.base_url = base_url
 
17
        self.queue = queue
 
18
        self.quarantine = os.path.join(self.working_dir, 'quarantine')
 
19
        
 
20
        if not os.path.exists(self.quarantine):
 
21
            os.makedirs(self.quarantine)
 
22
            
 
23
    def stop(self):
 
24
        self.shutdown_requested = True
 
25
    
 
26
    def run(self):
 
27
        """Downloads the file at the specified URL"""
 
28
        while not self.shutdown_requested:
 
29
            url = self.queue.get()
 
30
            
 
31
            # Location to save file to
 
32
            filepath = os.path.join(self.image_archive, 
 
33
                                    url.replace(self.base_url, ""))
 
34
            
 
35
            # Create sub-directory if it does not already exist
 
36
            if not os.path.exists(os.path.dirname(filepath)):
 
37
                os.makedirs(os.path.dirname(filepath))
 
38
    
 
39
            # TODO: should urlretrieve be used instead?
 
40
            remote_file = urlopen(Request(url))
 
41
            
 
42
            logging.info("Downloading " + os.path.basename(filepath))
 
43
            
 
44
            # Open our local file for writing
 
45
            local_file = open(filepath, "wb")
 
46
            
 
47
            #Write to our local file
 
48
            local_file.write(remote_file.read())
 
49
            local_file.close()
 
50
            
 
51
            self.queue.task_done()
 
52
    
 
 
b'\\ No newline at end of file'