← Back to branch summary

~ubuntu-core-dev/update-manager/main

« back to all changes in this revision

Viewing changes to AutoUpgradeTester/UpgradeTestBackendEC2.py

  • Committer: Michael Terry
  • Date: 2012-05-15 19:43:43 UTC
  • mto: This revision was merged to the branch mainline in revision 2428.
  • Revision ID: michael.terry@canonical.com-20120515194343-tocvo0awttmggie1
update several strings to match the software updater spec; most notably, rename from Update Manager to Software Updater

Show diffs side-by-side

added added

removed removed

Lines of Context:
AutoUpgradeTester/UpgradeTestBackendEC2.py
 
1
# ec2 backend
 
2
 
 
3
from __future__ import absolute_import, print_function
 
4
 
 
5
from .UpgradeTestBackendSSH import UpgradeTestBackendSSH
 
6
from .UpgradeTestBackend import UpgradeTestBackend
 
7
 
 
8
from DistUpgrade.sourceslist import SourcesList
 
9
 
 
10
from boto.ec2.connection import EC2Connection
 
11
 
 
12
try:
 
13
    import configparser
 
14
except ImportError:
 
15
    import ConfigParser as configparser
 
16
import os
 
17
import sys
 
18
import glob
 
19
import time
 
20
import atexit
 
21
import apt_pkg
 
22
 
 
23
 
 
24
# images created with EC2
 
25
class NoCredentialsFoundException(Exception):
 
26
    pass
 
27
 
 
28
class OptionError(Exception):
 
29
    pass
 
30
 
 
31
 
 
32
# Step to perform for a ec2 upgrade test
 
33
#
 
34
# 1. conn = EC2Connect()
 
35
# 2. reservation = conn.run_instances(image_id = image, security_groups = groups, key_name = key)
 
36
# 3. wait for instance.state == 'running':
 
37
#    instance.update()
 
38
# 4. ssh -i <key> root@instance.dns_name <command>
 
39
 
 
40
 
 
41
# TODO
 
42
#
 
43
# Using ebs (elastic block storage) and snapshots for the test
 
44
# 1. ec2-create-volume -s 80 -z us-east-1a
 
45
#    (check with ec2-describe-instance that its actually in 
 
46
#     the right zone)
 
47
# 2. ec2-attach-volume vol-7bd23de2 -i i-3325ad4 -d /dev/sdh
 
48
#    (do not name it anything but sd*)
 
49
# 3. mount/use the thing inside the instance
 
50
#
 
51
#
 
52
# Other useful things:
 
53
# - sda1: root fs
 
54
# - sda2: free space (~140G)
 
55
# - sda3: swapspace  (~1G)
 
56
 
 
57
class UpgradeTestBackendEC2(UpgradeTestBackendSSH):
 
58
    " EC2 backend "
 
59
 
 
60
    def __init__(self, profile):
 
61
        UpgradeTestBackend.__init__(self, profile)
 
62
        self.profiledir = os.path.abspath(os.path.dirname(profile))
 
63
        # ami base name (e.g .ami-44bb5c2d)
 
64
        self.ec2ami = self.config.get("EC2","AMI")
 
65
        self.ssh_key = self.config.get("EC2","SSHKey")
 
66
        try:
 
67
            self.access_key_id = (os.getenv("AWS_ACCESS_KEY_ID") or
 
68
                                  self.config.get("EC2","access_key_id"))
 
69
            self.secret_access_key = (os.getenv("AWS_SECRET_ACCESS_KEY") or
 
70
                                      self.config.get("EC2","secret_access_key"))
 
71
        except configparser.NoOptionError:
 
72
            print("Either export AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY or")
 
73
            print("set access_key_id and secret_access_key in the profile config")
 
74
            print("file.")
 
75
            sys.exit(1)
 
76
        self._conn = EC2Connection(self.access_key_id, self.secret_access_key)
 
77
        self.ubuntu_official_ami = False
 
78
        if self.config.has_option("EC2","UbuntuOfficialAMI"):
 
79
            self.ubuntu_official_ami = self.config.getboolean("EC2","UbuntuOfficialAMI")
 
80
        
 
81
        try:
 
82
            self.security_groups = self.config.getlist("EC2","SecurityGroups")
 
83
        except configparser.NoOptionError:
 
84
            self.security_groups = []
 
85
 
 
86
        if self.ssh_key.startswith("./"):
 
87
            self.ssh_key = self.profiledir + self.ssh_key[1:]
 
88
        self.ssh_port = "22"
 
89
        self.instance = None
 
90
 
 
91
        # the public name of the instance, e.g. 
 
92
        #  ec2-174-129-152-83.compute-1.amazonaws.com
 
93
        self.ssh_hostname = ""
 
94
        # the instance name (e.g. i-3325ad4)
 
95
        self.ec2instance = ""
 
96
        if (self.config.has_option("NonInteractive","RealReboot") and
 
97
            self.config.getboolean("NonInteractive","RealReboot")):
 
98
            raise OptionError("NonInteractive/RealReboot option must be set to False for the ec2 upgrader")
 
99
        atexit.register(self._cleanup)
 
100
 
 
101
    def _cleanup(self):
 
102
        print("_cleanup(): stopping running instance")
 
103
        if self.instance:
 
104
            self.instance.stop()
 
105
 
 
106
    def _enableRootLogin(self):
 
107
        command = ["sudo", 
 
108
                   "sed", "-i", "-e", "'s,\(.*\)\(ssh-rsa.*\),\\2,'", 
 
109
                   "/root/.ssh/authorized_keys"]
 
110
        ret = self._runInImageAsUser("ubuntu", command)
 
111
        return (ret == 0)
 
112
 
 
113
    def bootstrap(self, force=False):
 
114
        print("bootstrap()")
 
115
 
 
116
        print("Building new image based on '%s'" % self.ec2ami)
 
117
 
 
118
        # get common vars
 
119
        basepkg = self.config.get("NonInteractive","BasePkg")
 
120
 
 
121
        # start the VM
 
122
        self.start_instance()
 
123
 
 
124
        # prepare the sources.list (needed because a AMI may have any
 
125
        # sources.list)
 
126
        sources = self.getSourcesListFile()
 
127
        ret = self._copyToImage(sources.name, "/etc/apt/sources.list")
 
128
        assert(ret == 0)
 
129
 
 
130
        # install some useful stuff
 
131
        ret = self._runInImage(["apt-get","update"])
 
132
        assert(ret == 0)
 
133
        # FIXME: instead of this retrying (for network errors with 
 
134
        #        proxies) we should have a self._runAptInImage() 
 
135
        for i in range(3):
 
136
            ret = self._runInImage(["DEBIAN_FRONTEND=noninteractive","apt-get","install", "--allow-unauthenticated", "-y",basepkg])
 
137
        assert(ret == 0)
 
138
 
 
139
        CMAX = 4000
 
140
        pkgs =  self.config.getListFromFile("NonInteractive","AdditionalPkgs")
 
141
        while(len(pkgs)) > 0:
 
142
            print("installing additional: %s" % pkgs[:CMAX])
 
143
            ret= self._runInImage(["DEBIAN_FRONTEND=noninteractive","apt-get","install","--reinstall", "--allow-unauthenticated", "-y"]+pkgs[:CMAX])
 
144
            print("apt(2) returned: %s" % ret)
 
145
            if ret != 0:
 
146
                #self._cacheDebs(tmpdir)
 
147
                print("apt returned an error, stopping")
 
148
                self.stop_instance()
 
149
                return False
 
150
            pkgs = pkgs[CMAX+1:]
 
151
 
 
152
        if self.config.has_option("NonInteractive","PostBootstrapScript"):
 
153
            script = self.config.get("NonInteractive","PostBootstrapScript")
 
154
            print("have PostBootstrapScript: %s" % script)
 
155
            if os.path.exists(script):
 
156
                self._runInImage(["mkdir","/upgrade-tester"])
 
157
                self._copyToImage(script, "/upgrade-tester")
 
158
                print("running script: %s" % os.path.join("/tmp", script))
 
159
                self._runInImage([os.path.join("/upgrade-tester",script)])
 
160
            else:
 
161
                print("WARNING: %s not found" % script)
 
162
 
 
163
        if self.config.getWithDefault("NonInteractive",
 
164
                                      "UpgradeFromDistOnBootstrap", False):
 
165
            print("running apt-get upgrade in from dist (after bootstrap)")
 
166
            for i in range(3):
 
167
                ret = self._runInImage(["DEBIAN_FRONTEND=noninteractive","apt-get","--allow-unauthenticated", "-y","dist-upgrade"])
 
168
            assert(ret == 0)
 
169
 
 
170
        print("Cleaning image")
 
171
        ret = self._runInImage(["apt-get","clean"])
 
172
        assert(ret == 0)
 
173
 
 
174
        # done with the bootstrap
 
175
 
 
176
        # FIXME: idealy we would reboot here, but its less important
 
177
        #        because we can't get a new kernel anyway in ec2 (yet)
 
178
        # - the reboot thing is *not* yet reliable!
 
179
        #self.reboot_instance()
 
180
 
 
181
        # FIXME: support for caching/snapshoting the base image here
 
182
        
 
183
        return True
 
184
 
 
185
    def start_instance(self):
 
186
        print("Starting ec2 instance and wait until it's available")
 
187
 
 
188
        # start the instance
 
189
        reservation = self._conn.run_instances(image_id=self.ec2ami,
 
190
                                               security_groups=self.security_groups,
 
191
                                               key_name=self.ssh_key[:-4].split("/")[-1])
 
192
        self.instance = reservation.instances[0]
 
193
        while self.instance.state == "pending":
 
194
                print("Waiting for instance %s to come up..." % self.instance.id)
 
195
                time.sleep(10)
 
196
                self.instance.update()
 
197
 
 
198
        print("It's up: hostname =", self.instance.dns_name)
 
199
        self.ssh_hostname = self.instance.dns_name
 
200
        self.ec2instance = self.instance.id
 
201
 
 
202
        # now sping until ssh comes up in the instance
 
203
        if self.ubuntu_official_ami:
 
204
            user = "ubuntu"
 
205
        else:
 
206
            user = "root"
 
207
        for i in range(900):
 
208
            time.sleep(1)
 
209
            if self.ping(user):
 
210
                print("instance available via ssh ping")
 
211
                break
 
212
        else:
 
213
            print("Could not connect to instance after 900s, exiting")
 
214
            return False
 
215
        # re-enable root login if needed
 
216
        if self.ubuntu_official_ami:
 
217
            print("Re-enabling root login... ",)
 
218
            ret = self._enableRootLogin()
 
219
            if ret:
 
220
                print("Done!")
 
221
            else:
 
222
                print("Oops, failed to enable root login...")
 
223
            assert (ret == True)
 
224
            # the official image seems to run a update on startup?!?
 
225
            print("waiting for the official image to leave apt alone")
 
226
            time.sleep(10)
 
227
        return True
 
228
    
 
229
    def reboot_instance(self):
 
230
        " reboot a ec2 instance and wait until its available again "
 
231
        self.instance.reboot()
 
232
        # FIMXE: find a better way to know when the instance is 
 
233
        #        down - maybe with "-v" ?
 
234
        time.sleep(5)
 
235
        while True:
 
236
            if self._runInImage(["/bin/true"]) == 0:
 
237
                print("instance rebooted")
 
238
                break
 
239
 
 
240
    def stop_instance(self):
 
241
        " permanently stop a instance (it can never be started again "
 
242
        # terminates are final - all data is lost
 
243
        self.instance.stop()
 
244
        # wait until its down
 
245
        while True:
 
246
            if self._runInImage(["/bin/true"]) != 0:
 
247
                print("instance stopped")
 
248
                break
 
249
 
 
250
    def upgrade(self):
 
251
        print("upgrade()")
 
252
 
 
253
        # clean from any leftover pyc files
 
254
        for f in glob.glob("%s/*.pyc" %  self.upgradefilesdir):
 
255
            os.unlink(f)
 
256
 
 
257
        print("Starting for upgrade")
 
258
 
 
259
        assert(self.ec2instance)
 
260
        assert(self.ssh_hostname)
 
261
 
 
262
        # copy the profile
 
263
        if os.path.exists(self.profile):
 
264
            print("Copying '%s' to image overrides" % self.profile)
 
265
            self._runInImage(["mkdir","-p","/etc/update-manager/release-upgrades.d"])
 
266
            self._copyToImage(self.profile, "/etc/update-manager/release-upgrades.d/")
 
267
 
 
268
        # copy test repo sources.list (if needed)
 
269
        test_repo = self.config.getWithDefault("NonInteractive","AddRepo","")
 
270
        if test_repo:
 
271
            test_repo = os.path.join(os.path.dirname(self.profile), test_repo)
 
272
            self._copyToImage(test_repo, "/etc/apt/sources.list.d")
 
273
            sourcelist = self.getSourcesListFile()
 
274
            apt_pkg.config.set("Dir::Etc", os.path.dirname(sourcelist.name))
 
275
            apt_pkg.config.set("Dir::Etc::sourcelist", 
 
276
                               os.path.basename(sourcelist.name))
 
277
            sources = SourcesList(matcherPath=".")
 
278
            sources.load(test_repo)
 
279
            # add the uri to the list of valid mirros in the image
 
280
            for entry in sources.list:
 
281
                if (not (entry.invalid or entry.disabled) and
 
282
                    entry.type == "deb"):
 
283
                    print("adding %s to mirrors" % entry.uri)
 
284
                    self._runInImage(["echo '%s' >> /upgrade-tester/mirrors.cfg" % entry.uri])
 
285
 
 
286
        # check if we have a bzr checkout dir to run against or
 
287
        # if we should just run the normal upgrader
 
288
        if (os.path.exists(self.upgradefilesdir) and
 
289
            self.config.getWithDefault("NonInteractive", 
 
290
                                       "UseUpgraderFromBzr", 
 
291
                                       True)):
 
292
            self._copyUpgraderFilesFromBzrCheckout()
 
293
            ret = self._runBzrCheckoutUpgrade()
 
294
        else:
 
295
            ret = self._runInImage(["do-release-upgrade","-d",
 
296
                                    "-f","DistUpgradeViewNonInteractive"])
 
297
        print("dist-upgrade.py returned: %i" % ret)
 
298
        
 
299
 
 
300
        # copy the result
 
301
        print("copying the result")
 
302
        self._copyFromImage("/var/log/dist-upgrade/*",self.resultdir)
 
303
 
 
304
        # stop the machine
 
305
        print("Shutting down the VM")
 
306
        self.stop_instance()
 
307
 
 
308
        return True
 
309
 
 
310
    def test(self):
 
311
        # FIXME: add some tests here to see if the upgrade worked
 
312
        # this should include:
 
313
        # - new kernel is runing (run uname -r in target)
 
314
        # - did it sucessfully rebooted
 
315
        # - is X runing
 
316
        # - generate diff of upgrade vs fresh install
 
317
        # ...
 
318
        return True
 
319
        
 
320
 
 
321
    # compatibility for the auto-install-tester
 
322
    def start(self):
 
323
        self.start_instance()
 
324
    def stop(self):
 
325
        self.stop_instance()
 
326
    def saveVMSnapshot(self):
 
327
        print("saveVMSnapshot not supported yet")
 
328
    def restoreVMSnapshot(self):
 
329
        print("restoreVMSnapshot not supported yet")
 
330
 
 
331
    
 
332
# vim:ts=4:sw=4:et
 
333