130
by Corey Bryant
[corey.bryant,r=trivial] Sync charm-helpers. |
1 |
# Copyright 2014-2015 Canonical Limited.
|
2 |
#
|
|
3 |
# This file is part of charm-helpers.
|
|
4 |
#
|
|
5 |
# charm-helpers is free software: you can redistribute it and/or modify
|
|
6 |
# it under the terms of the GNU Lesser General Public License version 3 as
|
|
7 |
# published by the Free Software Foundation.
|
|
8 |
#
|
|
9 |
# charm-helpers is distributed in the hope that it will be useful,
|
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12 |
# GNU Lesser General Public License for more details.
|
|
13 |
#
|
|
14 |
# You should have received a copy of the GNU Lesser General Public License
|
|
15 |
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
16 |
||
17 |
import os |
|
18 |
import hashlib |
|
19 |
import re |
|
20 |
||
21 |
from charmhelpers.fetch import ( |
|
22 |
BaseFetchHandler, |
|
23 |
UnhandledSource
|
|
24 |
)
|
|
25 |
from charmhelpers.payload.archive import ( |
|
26 |
get_archive_handler, |
|
27 |
extract, |
|
28 |
)
|
|
29 |
from charmhelpers.core.host import mkdir, check_hash |
|
30 |
||
31 |
import six |
|
32 |
if six.PY3: |
|
33 |
from urllib.request import ( |
|
34 |
build_opener, install_opener, urlopen, urlretrieve, |
|
35 |
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, |
|
36 |
)
|
|
37 |
from urllib.parse import urlparse, urlunparse, parse_qs |
|
38 |
from urllib.error import URLError |
|
39 |
else: |
|
40 |
from urllib import urlretrieve |
|
41 |
from urllib2 import ( |
|
42 |
build_opener, install_opener, urlopen, |
|
43 |
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, |
|
44 |
URLError
|
|
45 |
)
|
|
46 |
from urlparse import urlparse, urlunparse, parse_qs |
|
47 |
||
48 |
||
49 |
def splituser(host): |
|
50 |
'''urllib.splituser(), but six's support of this seems broken'''
|
|
51 |
_userprog = re.compile('^(.*)@(.*)$') |
|
52 |
match = _userprog.match(host) |
|
53 |
if match: |
|
54 |
return match.group(1, 2) |
|
55 |
return None, host |
|
56 |
||
57 |
||
58 |
def splitpasswd(user): |
|
59 |
'''urllib.splitpasswd(), but six's support of this is missing'''
|
|
60 |
_passwdprog = re.compile('^([^:]*):(.*)$', re.S) |
|
61 |
match = _passwdprog.match(user) |
|
62 |
if match: |
|
63 |
return match.group(1, 2) |
|
64 |
return user, None |
|
65 |
||
66 |
||
67 |
class ArchiveUrlFetchHandler(BaseFetchHandler): |
|
68 |
"""
|
|
69 |
Handler to download archive files from arbitrary URLs.
|
|
70 |
||
71 |
Can fetch from http, https, ftp, and file URLs.
|
|
72 |
||
73 |
Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files.
|
|
74 |
||
75 |
Installs the contents of the archive in $CHARM_DIR/fetched/.
|
|
76 |
"""
|
|
77 |
def can_handle(self, source): |
|
78 |
url_parts = self.parse_url(source) |
|
79 |
if url_parts.scheme not in ('http', 'https', 'ftp', 'file'): |
|
80 |
# XXX: Why is this returning a boolean and a string? It's
|
|
81 |
# doomed to fail since "bool(can_handle('foo://'))" will be True.
|
|
82 |
return "Wrong source type" |
|
83 |
if get_archive_handler(self.base_url(source)): |
|
84 |
return True |
|
85 |
return False |
|
86 |
||
87 |
def download(self, source, dest): |
|
88 |
"""
|
|
89 |
Download an archive file.
|
|
90 |
||
91 |
:param str source: URL pointing to an archive file.
|
|
92 |
:param str dest: Local path location to download archive file to.
|
|
93 |
"""
|
|
94 |
# propogate all exceptions
|
|
95 |
# URLError, OSError, etc
|
|
96 |
proto, netloc, path, params, query, fragment = urlparse(source) |
|
97 |
if proto in ('http', 'https'): |
|
98 |
auth, barehost = splituser(netloc) |
|
99 |
if auth is not None: |
|
100 |
source = urlunparse((proto, barehost, path, params, query, fragment)) |
|
101 |
username, password = splitpasswd(auth) |
|
102 |
passman = HTTPPasswordMgrWithDefaultRealm() |
|
103 |
# Realm is set to None in add_password to force the username and password
|
|
104 |
# to be used whatever the realm
|
|
105 |
passman.add_password(None, source, username, password) |
|
106 |
authhandler = HTTPBasicAuthHandler(passman) |
|
107 |
opener = build_opener(authhandler) |
|
108 |
install_opener(opener) |
|
109 |
response = urlopen(source) |
|
110 |
try: |
|
111 |
with open(dest, 'wb') as dest_file: |
|
112 |
dest_file.write(response.read()) |
|
113 |
except Exception as e: |
|
114 |
if os.path.isfile(dest): |
|
115 |
os.unlink(dest) |
|
116 |
raise e |
|
117 |
||
118 |
# Mandatory file validation via Sha1 or MD5 hashing.
|
|
119 |
def download_and_validate(self, url, hashsum, validate="sha1"): |
|
120 |
tempfile, headers = urlretrieve(url) |
|
121 |
check_hash(tempfile, hashsum, validate) |
|
122 |
return tempfile |
|
123 |
||
124 |
def install(self, source, dest=None, checksum=None, hash_type='sha1'): |
|
125 |
"""
|
|
126 |
Download and install an archive file, with optional checksum validation.
|
|
127 |
||
128 |
The checksum can also be given on the `source` URL's fragment.
|
|
129 |
For example::
|
|
130 |
||
131 |
handler.install('http://example.com/file.tgz#sha1=deadbeef')
|
|
132 |
||
133 |
:param str source: URL pointing to an archive file.
|
|
134 |
:param str dest: Local destination path to install to. If not given,
|
|
135 |
installs to `$CHARM_DIR/archives/archive_file_name`.
|
|
136 |
:param str checksum: If given, validate the archive file after download.
|
|
137 |
:param str hash_type: Algorithm used to generate `checksum`.
|
|
138 |
Can be any hash alrgorithm supported by :mod:`hashlib`,
|
|
139 |
such as md5, sha1, sha256, sha512, etc.
|
|
140 |
||
141 |
"""
|
|
142 |
url_parts = self.parse_url(source) |
|
143 |
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') |
|
144 |
if not os.path.exists(dest_dir): |
|
145 |
mkdir(dest_dir, perms=0o755) |
|
146 |
dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path)) |
|
147 |
try: |
|
148 |
self.download(source, dld_file) |
|
149 |
except URLError as e: |
|
150 |
raise UnhandledSource(e.reason) |
|
151 |
except OSError as e: |
|
152 |
raise UnhandledSource(e.strerror) |
|
153 |
options = parse_qs(url_parts.fragment) |
|
154 |
for key, value in options.items(): |
|
155 |
if not six.PY3: |
|
156 |
algorithms = hashlib.algorithms |
|
157 |
else: |
|
158 |
algorithms = hashlib.algorithms_available |
|
159 |
if key in algorithms: |
|
160 |
if len(value) != 1: |
|
161 |
raise TypeError( |
|
162 |
"Expected 1 hash value, not %d" % len(value)) |
|
163 |
expected = value[0] |
|
164 |
check_hash(dld_file, expected, key) |
|
165 |
if checksum: |
|
166 |
check_hash(dld_file, checksum, hash_type) |
|
167 |
return extract(dld_file, dest) |