1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
|
import sys
import os
import warnings
warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning)
import apt
from DistUpgradeConfigParser import DistUpgradeConfig
from UpgradeTestBackend import UpgradeTestBackend
import tempfile
import subprocess
import shutil
import logging
import glob
import ConfigParser
class UpgradeTestBackendChroot(UpgradeTestBackend):
diverts = ["/usr/sbin/mkinitrd",
"/sbin/modprobe",
"/usr/sbin/invoke-rc.d",
# install-info has a locking problem quite often
"/usr/sbin/install-info",
"/sbin/start-stop-daemon"]
def __init__(self, profile, basefiledir):
UpgradeTestBackend.__init__(self, profile, basefiledir)
self.tarball = None
def _umount(self, chrootdir):
umount_list = []
for line in open("/proc/mounts"):
(dev, mnt, fs, options, d, p) = line.split()
if mnt.startswith(chrootdir):
umount_list.append(mnt)
# now sort and umount by reverse length (to ensure
# we umount /sys/fs/binfmt_misc before /sys)
umount_list.sort(key=len)
umount_list.reverse()
# do the list
for mpoint in umount_list:
print "Umount '%s'" % mpoint
os.system("umount %s" % mpoint)
def _runInChroot(self, chrootdir, command, cmd_options=[]):
print "runing: ",command
pid = os.fork()
if pid == 0:
os.chroot(chrootdir)
os.system("mount -t devpts devpts /dev/pts")
os.system("mount -t sysfs sysfs /sys")
os.system("mount -t proc proc /proc")
os.system("mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc")
os.execv(command[0], command)
else:
print "Parent: waiting for %s" % pid
(id, exitstatus) = os.waitpid(pid, 0)
self._umount(chrootdir)
return exitstatus
def installPackages(self, pkgs):
#self._runApt(tmpdir, "install", pkgs)
raise NotImplementedException
def _runApt(self, tmpdir, command, cmd_options=[]):
ret = self._runInChroot(tmpdir,
["/usr/bin/apt-get", command]+self.apt_options+cmd_options)
return ret
def _tryRandomPkgInstall(self, amount):
" install 'amount' packages randomly "
self._runApt(tmpdir,"install",["python2.4-apt", "python-apt"])
shutil.copy("%s/randomInst.py",tmpdir+"/tmp")
ret = subprocess.call(["chroot",tmpdir,"/tmp/randomInst.py","%s" % amount])
def _cacheDebs(self, tmpdir):
# see if the debs should be cached
if self.cachedir:
print "Caching debs"
for f in glob.glob(tmpdir+"/var/cache/apt/archives/*.deb"):
if not os.path.exists(self.cachedir+"/"+os.path.basename(f)):
try:
shutil.copy(f,self.cachedir)
except IOError, e:
print "Can't copy '%s' (%s)" % (f,e)
def _getTmpDir(self):
tmpdir = self.config.getWithDefault("NonInteractive","Tempdir",None)
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
else:
if os.path.exists(tmpdir):
self._umount(tmpdir)
shutil.rmtree(tmpdir)
os.makedirs(tmpdir)
return tmpdir
def bootstrap(self,outfile=None):
" bootstaps a pristine fromDist tarball"
if not outfile:
outfile = os.path.dirname(self.profile) + "/dist-upgrade-%s.tar.gz" % self.fromDist
outfile = os.path.abspath(outfile)
self.tarball = outfile
# don't bootstrap twice if this is something we can cache
try:
if (self.config.getboolean("NonInteractive","CacheTarball") and
os.path.exists(self.tarball) ):
return True
except ConfigParser.NoOptionError:
pass
# bootstrap!
tmpdir = self._getTmpDir()
print "tmpdir is %s" % tmpdir
print "bootstraping to %s" % outfile
ret = subprocess.call(["debootstrap", self.fromDist, tmpdir, self.config.get("NonInteractive","Mirror")])
print "debootstrap returned: %s" % ret
if ret != 0:
return False
print "diverting"
self._dpkgDivert(tmpdir)
# create some minimal device node
print "Creating some devices"
os.system("(cd %s/dev ; echo $PWD; ./MAKEDEV null)" % tmpdir)
#self._runInChroot(tmpdir, ["/bin/mknod","/dev/null","c","1","3"])
# set a hostname
shutil.copy("/etc/hostname","%s/etc/hostanme" % tmpdir)
# copy the stuff from toChroot/
os.chdir("toChroot/")
for (dirpath, dirnames, filenames) in os.walk("."):
for name in filenames:
if not os.path.exists(os.path.join(tmpdir,dirpath,name)):
shutil.copy(os.path.join(dirpath,name), os.path.join(tmpdir,dirpath,name))
os.chdir("..")
# write new sources.list
if (self.config.has_option("NonInteractive","Components") and
self.config.has_option("NonInteractive","Pockets")):
shutil.copy(self.getSourcesListFile(), tmpdir+"/etc/apt/sources.list")
print open(tmpdir+"/etc/apt/sources.list","r").read()
# move the cache debs
self._populateWithCachedDebs(tmpdir)
print "Updating the chroot"
ret = self._runApt(tmpdir,"update")
print "apt update returned %s" % ret
if ret != 0:
return False
# run it three times to work around network issues
for i in range(3):
ret = self._runApt(tmpdir,"dist-upgrade")
print "apt dist-upgrade returned %s" % ret
if ret != 0:
return False
print "installing basepkg"
ret = self._runApt(tmpdir,"install", [self.config.get("NonInteractive","BasePkg")])
print "apt returned %s" % ret
if ret != 0:
return False
CMAX = 4000
pkgs = self.config.getListFromFile("NonInteractive","AdditionalPkgs")
while(len(pkgs)) > 0:
print "installing additonal: %s" % pkgs[:CMAX]
ret= self._runApt(tmpdir,"install",pkgs[:CMAX])
print "apt(2) returned: %s" % ret
if ret != 0:
self._cacheDebs(tmpdir)
return False
pkgs = pkgs[CMAX+1:]
if self.config.has_option("NonInteractive","PostBootstrapScript"):
script = self.config.get("NonInteractive","PostBootstrapScript")
if os.path.exists(script):
shutil.copy(script, os.path.join(tmpdir,"tmp"))
self._runInChroot(tmpdir,[os.path.join("/tmp",script)])
else:
print "WARNING: %s not found" % script
try:
amount = self.config.get("NonInteractive","RandomPkgInstall")
self._tryRandomPkgInstall(amount)
except ConfigParser.NoOptionError:
pass
print "Caching debs"
self._cacheDebs(tmpdir)
print "Cleaning chroot"
ret = self._runApt(tmpdir,"clean")
if ret != 0:
return False
print "building tarball: '%s'" % outfile
os.chdir(tmpdir)
ret = subprocess.call(["tar","czf",outfile,"."])
print "tar returned %s" % ret
print "Removing chroot"
shutil.rmtree(tmpdir)
return True
def _populateWithCachedDebs(self, tmpdir):
# now populate with hardlinks for the debs
if self.cachedir:
print "Linking cached debs into chroot"
for f in glob.glob(self.cachedir+"/*.deb"):
try:
os.link(f, tmpdir+"/var/cache/apt/archives/%s" % os.path.basename(f))
except OSError, e:
print "Can't link: %s (%s)" % (f,e)
return True
def upgrade(self, tarball=None):
if not tarball:
tarball = self.tarball
assert(tarball != None)
print "runing upgrade on: %s" % tarball
tmpdir = self._unpackToTmpdir(tarball)
if not tmpdir:
print "Error extracting tarball"
#self._runApt(tmpdir, "install",["apache2"])
self._populateWithCachedDebs(tmpdir)
# copy itself to the chroot (resolve symlinks)
targettmpdir = os.path.join(tmpdir,"tmp","dist-upgrade")
if not os.path.exists(targettmpdir):
os.mkdir(targettmpdir)
for f in glob.glob("%s/*" % os.path.join(self.basefilesdir,"DistUpgrade")):
if not os.path.isdir(f):
shutil.copy(f, targettmpdir)
# copy the profile
if os.path.exists(self.profile):
print "Copying '%s' to '%s' " % (self.profile,targettmpdir)
shutil.copy(self.profile, targettmpdir)
# copy the .cfg and .list stuff from there too
for f in glob.glob("%s/*.cfg" % (os.path.dirname(self.profile))):
shutil.copy(f, targettmpdir)
for f in glob.glob("%s/*.list" % (os.path.dirname(self.profile))):
shutil.copy(f, targettmpdir)
# run it
pid = os.fork()
if pid == 0:
os.chroot(tmpdir)
os.chdir("/tmp/dist-upgrade")
os.system("mount -t devpts devpts /dev/pts")
os.system("mount -t sysfs sysfs /sys")
os.system("mount -t proc proc /proc")
os.system("mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc")
os.execl("/tmp/dist-upgrade/dist-upgrade.py",
"/tmp/dist-upgrade/dist-upgrade.py")
else:
print "Parent: waiting for %s" % pid
(id, exitstatus) = os.waitpid(pid, 0)
print "Child exited (%s, %s): %s" % (id, exitstatus, os.WEXITSTATUS(exitstatus))
# copy the result
for f in glob.glob(tmpdir+"/var/log/dist-upgrade/*"):
print "copying result to: ", self.resultdir
shutil.copy(f, self.resultdir)
# cache debs and cleanup
self._cacheDebs(tmpdir)
self._umount(tmpdir)
shutil.rmtree(tmpdir)
return (exitstatus == 0)
def _unpackToTmpdir(self, baseTarBall):
# unpack the tarball
tmpdir = self._getTmpDir()
os.chdir(tmpdir)
ret = subprocess.call(["tar","xzf",baseTarBall])
if ret != 0:
return None
return tmpdir
def _dpkgDivert(self, tmpdir):
for d in self.diverts:
cmd = ["chroot",tmpdir,
"dpkg-divert","--add","--local",
"--divert",d+".thereal",
"--rename",d]
ret = subprocess.call(cmd)
shutil.copy(tmpdir+"/bin/true",tmpdir+d)
def test(self):
# FIXME: add some sanity testing here
return True
if __name__ == "__main__":
if len(sys.argv) > 1:
profilename = sys.argv[1]
else:
profilename = "default"
chroot = Chroot(profilename)
tarball = "%s/tarball/dist-upgrade-%s.tar.gz" % (os.getcwd(),profilename)
if not os.path.exists(tarball):
print "No existing tarball found, creating a new one"
chroot.bootstrap(tarball)
chroot.upgrade(tarball)
#tmpdir = chroot._unpackToTmpdir(tarball)
#chroot._dpkgDivert(tmpdir)
#print tmpdir
|