1
# Copyright 2014-2015 Canonical Limited.
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
7
# http://www.apache.org/licenses/LICENSE-2.0
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
19
from charmhelpers.fetch import (
23
from charmhelpers.payload.archive import (
27
from charmhelpers.core.host import mkdir, check_hash
31
from urllib.request import (
32
build_opener, install_opener, urlopen, urlretrieve,
33
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
35
from urllib.parse import urlparse, urlunparse, parse_qs
36
from urllib.error import URLError
38
from urllib import urlretrieve
40
build_opener, install_opener, urlopen,
41
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
44
from urlparse import urlparse, urlunparse, parse_qs
48
'''urllib.splituser(), but six's support of this seems broken'''
49
_userprog = re.compile('^(.*)@(.*)$')
50
match = _userprog.match(host)
52
return match.group(1, 2)
56
def splitpasswd(user):
57
'''urllib.splitpasswd(), but six's support of this is missing'''
58
_passwdprog = re.compile('^([^:]*):(.*)$', re.S)
59
match = _passwdprog.match(user)
61
return match.group(1, 2)
65
class ArchiveUrlFetchHandler(BaseFetchHandler):
67
Handler to download archive files from arbitrary URLs.
69
Can fetch from http, https, ftp, and file URLs.
71
Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files.
73
Installs the contents of the archive in $CHARM_DIR/fetched/.
75
def can_handle(self, source):
76
url_parts = self.parse_url(source)
77
if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
78
# XXX: Why is this returning a boolean and a string? It's
79
# doomed to fail since "bool(can_handle('foo://'))" will be True.
80
return "Wrong source type"
81
if get_archive_handler(self.base_url(source)):
85
def download(self, source, dest):
87
Download an archive file.
89
:param str source: URL pointing to an archive file.
90
:param str dest: Local path location to download archive file to.
92
# propogate all exceptions
93
# URLError, OSError, etc
94
proto, netloc, path, params, query, fragment = urlparse(source)
95
if proto in ('http', 'https'):
96
auth, barehost = splituser(netloc)
98
source = urlunparse((proto, barehost, path, params, query, fragment))
99
username, password = splitpasswd(auth)
100
passman = HTTPPasswordMgrWithDefaultRealm()
101
# Realm is set to None in add_password to force the username and password
102
# to be used whatever the realm
103
passman.add_password(None, source, username, password)
104
authhandler = HTTPBasicAuthHandler(passman)
105
opener = build_opener(authhandler)
106
install_opener(opener)
107
response = urlopen(source)
109
with open(dest, 'wb') as dest_file:
110
dest_file.write(response.read())
111
except Exception as e:
112
if os.path.isfile(dest):
116
# Mandatory file validation via Sha1 or MD5 hashing.
117
def download_and_validate(self, url, hashsum, validate="sha1"):
118
tempfile, headers = urlretrieve(url)
119
check_hash(tempfile, hashsum, validate)
122
def install(self, source, dest=None, checksum=None, hash_type='sha1'):
124
Download and install an archive file, with optional checksum validation.
126
The checksum can also be given on the `source` URL's fragment.
129
handler.install('http://example.com/file.tgz#sha1=deadbeef')
131
:param str source: URL pointing to an archive file.
132
:param str dest: Local destination path to install to. If not given,
133
installs to `$CHARM_DIR/archives/archive_file_name`.
134
:param str checksum: If given, validate the archive file after download.
135
:param str hash_type: Algorithm used to generate `checksum`.
136
Can be any hash alrgorithm supported by :mod:`hashlib`,
137
such as md5, sha1, sha256, sha512, etc.
140
url_parts = self.parse_url(source)
141
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
142
if not os.path.exists(dest_dir):
143
mkdir(dest_dir, perms=0o755)
144
dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
146
self.download(source, dld_file)
147
except URLError as e:
148
raise UnhandledSource(e.reason)
150
raise UnhandledSource(e.strerror)
151
options = parse_qs(url_parts.fragment)
152
for key, value in options.items():
154
algorithms = hashlib.algorithms
156
algorithms = hashlib.algorithms_available
157
if key in algorithms:
160
"Expected 1 hash value, not %d" % len(value))
162
check_hash(dld_file, expected, key)
164
check_hash(dld_file, checksum, hash_type)
165
return extract(dld_file, dest)
added
removed
hooks/charmhelpers/fetch/archiveurl.py
