« back to all changes in this revision
Viewing changes to snapcraft/repo.py
- browse files at revision 186
- compare with another revision
- download diff
- download tarball
- view history from revision 186
- Committer: Snappy Tarmac
- Author(s): Ted Gould, Ted Gould, Sergio Schvezov, Michael Vogt
- Date: 2015-09-21 16:43:00 UTC
- mfrom: (183.1.8 snapcraft)
- Revision ID: snappy_tarmac-20150921164300-igplcvyohcre6bl7
Improving stage-package handling by snappy-dev approved by mvo
added
removed
snapcraft/repo.py
14
14
# You should have received a copy of the GNU General Public License
15
15
# along with this program. If not, see <http://www.gnu.org/licenses/>.
16
16
17
import apt
18
17
import glob
19
18
import itertools
20
19
import os
20
import string
21
21
import subprocess
22
import urllib
23
import urllib.request
24
25
import apt
26
from xml.etree import ElementTree
27
28
_DEFAULT_SOURCES = '''deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid main restricted
29
deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid-updates main restricted
30
deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid universe
31
deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid-updates universe
32
deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid multiverse
33
deb http://${mirror}archive.ubuntu.com/ubuntu/ vivid-updates multiverse
34
deb http://security.ubuntu.com/ubuntu vivid-security main restricted
35
deb http://security.ubuntu.com/ubuntu vivid-security universe
36
deb http://security.ubuntu.com/ubuntu vivid-security multiverse
37
'''
38
_GEOIP_SERVER = "http://geoip.ubuntu.com/lookup"
22
39
23
40
24
41
class PackageNotFoundError(Exception):
43
60
44
61
class Ubuntu:
45
62
46
def __init__(self, download_dir, recommends=False):
47
self.apt_cache = apt.Cache()
48
self.manifest_dep_names = self._manifest_dep_names()
63
def __init__(self, rootdir, recommends=False, sources=_DEFAULT_SOURCES):
64
sources = sources or _DEFAULT_SOURCES
65
self.downloaddir = os.path.join(rootdir, 'download')
66
self.rootdir = rootdir
67
self.apt_cache = _setup_apt_cache(rootdir, sources)
49
68
self.recommends = recommends
50
self.download_dir = download_dir
51
69
52
70
def get(self, package_names):
53
# TODO cleanup download_dir for clean gets and unpacks
54
self.all_dep_names = set()
55
56
all_package_names = self._compute_deps(package_names)
57
58
for pkg in all_package_names:
59
self.apt_cache[pkg].candidate.fetch_binary(destdir=self.download_dir)
60
61
return all_package_names
62
63
def unpack(self, root_dir):
64
pkgs_abs_path = glob.glob(os.path.join(self.download_dir, '*.deb'))
71
os.makedirs(self.downloaddir, exist_ok=True)
72
73
manifest_dep_names = self._manifest_dep_names()
74
75
for name in package_names:
76
try:
77
self.apt_cache[name].mark_install()
78
except KeyError:
79
raise PackageNotFoundError(name)
80
81
for pkg in self.apt_cache:
82
# those should be already on each system, it also prevents
83
# diving into downloading libc6
84
if (pkg.candidate.priority in 'essential' and
85
pkg.name not in package_names):
86
print('Skipping priority essential/imporant %s' % pkg.name)
87
continue
88
if (pkg.name in manifest_dep_names and pkg.name not in package_names):
89
print('Skipping blacklisted from manifest package %s' % pkg.name)
90
continue
91
if pkg.marked_install:
92
pkg.candidate.fetch_binary(destdir=self.downloaddir)
93
94
def unpack(self, rootdir):
95
pkgs_abs_path = glob.glob(os.path.join(self.downloaddir, '*.deb'))
65
96
for pkg in pkgs_abs_path:
66
97
# TODO needs elegance and error control
67
98
try:
68
subprocess.check_call(['dpkg-deb', '--extract', pkg, root_dir])
99
subprocess.check_call(['dpkg-deb', '--extract', pkg, rootdir])
69
100
except subprocess.CalledProcessError:
70
101
raise UnpackError(pkg)
71
102
72
_fix_symlinks(root_dir)
103
_fix_symlinks(rootdir)
73
104
74
105
def _manifest_dep_names(self):
75
106
manifest_dep_names = set()
82
113
83
114
return manifest_dep_names
84
115
85
def _compute_deps(self, package_names):
86
self._add_deps(package_names)
87
88
for pkg in package_names:
89
if pkg not in self.all_dep_names:
90
raise PackageNotFoundError(pkg)
91
92
return sorted(self.all_dep_names)
93
94
def _add_deps(self, package_names):
95
def add_deps(packages):
96
for pkg in packages:
97
# Remove the :any in packages
98
# TODO support multiarch
99
pkg = pkg.rsplit(':', 1)[0]
100
if pkg in self.all_dep_names:
101
continue
102
if pkg in self.manifest_dep_names and pkg not in package_names:
103
continue
104
deps = set()
105
try:
106
candidate_pkg = self.apt_cache[pkg].candidate
107
except KeyError:
108
raise PackageNotFoundError(pkg)
109
deps = candidate_pkg.dependencies
110
if self.recommends:
111
deps += candidate_pkg.recommends
112
self.all_dep_names.add(pkg)
113
add_deps([x[0].name for x in deps])
114
add_deps(package_names)
116
117
def get_geoip_country_code_prefix():
118
try:
119
with urllib.request.urlopen(_GEOIP_SERVER) as f:
120
xml_data = f.read()
121
et = ElementTree.fromstring(xml_data)
122
cc = et.find("CountryCode")
123
if cc is None:
124
return ""
125
return cc.text.lower() + "."
126
except (ElementTree.ParseError, urllib.error.URLError):
127
pass
128
return ""
129
130
131
def _setup_apt_cache(rootdir, sources):
132
os.makedirs(os.path.join(rootdir, 'etc', 'apt'), exist_ok=True)
133
srcfile = os.path.join(rootdir, 'etc', 'apt', 'sources.list')
134
with open(srcfile, 'w') as f:
135
mirror_prefix = get_geoip_country_code_prefix()
136
sources_list = string.Template(sources).substitute(
137
{"mirror": mirror_prefix})
138
f.write(sources_list)
139
progress = apt.progress.text.AcquireProgress()
140
apt_cache = apt.Cache(rootdir=rootdir, memonly=True)
141
apt_cache.update(fetch_progress=progress, sources_list=srcfile)
142
apt_cache.open()
143
144
return apt_cache
115
145
116
146
117
147
def _fix_symlinks(debdir):
Loggerhead is a web-based interface for Breezy
Version: 2.0.1