1
"""urllib2-based file downloader"""
5
from urllib2 import urlopen, Request # URLError, HTTPError
7
class URLLibDownloader(threading.Thread):
8
def __init__(self, image_archive, working_dir, base_url, queue):
9
"""Creates a new URLLibDownloader"""
10
threading.Thread.__init__(self)
12
self.shutdown_requested = False
14
self.image_archive = image_archive
15
self.working_dir = working_dir
16
self.base_url = base_url
18
self.quarantine = os.path.join(self.working_dir, 'quarantine')
20
if not os.path.exists(self.quarantine):
21
os.makedirs(self.quarantine)
24
self.shutdown_requested = True
27
"""Downloads the file at the specified URL"""
28
while not self.shutdown_requested:
29
url = self.queue.get()
31
# Location to save file to
32
filepath = os.path.join(self.image_archive,
33
url.replace(self.base_url, ""))
35
# Create sub-directory if it does not already exist
36
if not os.path.exists(os.path.dirname(filepath)):
37
os.makedirs(os.path.dirname(filepath))
39
# TODO: should urlretrieve be used instead?
40
remote_file = urlopen(Request(url))
42
logging.info("Downloading " + os.path.basename(filepath))
44
# Open our local file for writing
45
local_file = open(filepath, "wb")
47
#Write to our local file
48
local_file.write(remote_file.read())
51
self.queue.task_done()
b'\\ No newline at end of file'