1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
#!/usr/bin/env python
# Ubuntu Continuous Integration Engine
# Copyright 2014 Canonical Ltd.
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License version 3, as
# published by the Free Software Foundation.
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranties of
# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
# PURPOSE. See the GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
import os
import subprocess
import tempfile
import time
import shutil
import logging
HERE = os.path.abspath(os.path.dirname(__file__))
BRANCH_ROOT = os.path.abspath(os.path.join(HERE, '..'))
logging.root.setLevel(logging.DEBUG)
def get_basedir():
'''The directory to put the virtualenv under.
Defaults to CI_VENV_LOCATION environment variable or shared memory.
'''
return os.environ.get('CI_VENV_LOCATION') or '/dev/shm'
def find_components():
'''Find all the components.'''
for component_dir in os.listdir(BRANCH_ROOT):
c = os.path.join(BRANCH_ROOT, component_dir)
if os.path.isdir(c):
if 'setup.py' in os.listdir(c):
yield component_dir
def create(path=None, version='python2.7'):
'''Create a virtualenv.'''
if path is None:
path = tempfile.mktemp(prefix='venv-', dir=get_basedir())
with open('/dev/null', 'w') as devnull:
cmd = ['virtualenv', path, '-p', version]
subprocess.check_call(cmd, stdout=devnull)
set_env(path)
return path
def set_env(path):
'''Set environment variables for the virtualenv installed at path.'''
logging.info('Asked to set the env to the virtualenv at %s' % path)
os.environ['VIRTUAL_ENV'] = path
os.environ['PATH'] = '%s/bin:%s' % (path, os.environ['PATH'])
deps = 'lp:~canonical-ci-engineering/uci-engine/deps'
os.environ['CI_DEPS_BRANCH'] = deps
# For OSX, see http://kaspermunck.github.io/2014/03/fixing-clang-error
arch = '-Wno-error=unused-command-line-argument-hard-error-in-future'
os.environ['ARCHFLAGS'] = arch
# virtualenv doesn't like PYTHONHOME.
try:
del os.environ['PYTHONHOME']
except KeyError:
pass
# Install additional dependencies from our pip cache (respecting
# CI_DEPS_LOCAL).
local_deps = os.environ.get('CI_DEPS_LOCAL')
if local_deps:
os.environ['PIP_CACHE'] = os.path.abspath(local_deps)
else:
os.environ['PIP_CACHE'] = os.path.join(os.getcwd(), '.deps')
def prepare(components=None):
'''Install all the dependencies for each component.'''
if not components:
components = [x for x in find_components()]
if 'ci-utils' in components:
components.remove('ci-utils')
components.insert(0, 'ci-utils')
save = os.getcwd()
for component in components:
os.chdir(os.path.join(BRANCH_ROOT, component))
try:
cmd = ['./setup.py', 'develop']
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
msg = '%s setup failed (with code %d):'
logging.error(msg % (component, e.returncode))
for line in e.output.splitlines():
logging.error(line)
raise SystemExit(1)
finally:
os.chdir(save)
def additional_deps():
'''Install addtional dependencies.'''
additional_deps = ['cheetah', 'python-apt', 'jinja2']
for dep in additional_deps:
cache_file = 'file://%s' % os.environ['PIP_CACHE']
cmd = ['pip', 'install', '--no-index', '-f', cache_file, dep]
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
msg = 'Calling `%s` failed (with code %d:'
logging.error(msg % (e.cmd, e.returncode))
for line in e.output.splitlines():
logging.error(line)
def install():
'''Set up a virtualenv, install all the component dependencies, then
re-exec this process under the virtualenv to pick up the new python
interpreter.'''
if hasattr(sys, 'real_prefix'):
# Already under virtualenv.
return
if os.environ.get('REEXECED_UNDER_VENV'):
logging.error('Exec loop detected. Quitting.')
sys.exit(1)
logging.info('Creating a virtualenv to run under...')
t = time.time()
try:
path = create()
prepare()
additional_deps()
finally:
taken = time.time() - t
logging.info('virtualenv created in %.2fs.' % taken)
os.environ['REEXECED_UNDER_VENV'] = path
# Re-exec ourselves. execv() requires arg 2 not be empty.
os.execv(sys.argv[0], sys.argv or [''])
def delete():
'''Remove the virtualenv.'''
path = os.environ.get('REEXECED_UNDER_VENV')
if path:
shutil.rmtree(path)
del os.environ['REEXECED_UNDER_VENV']
if __name__ == '__main__':
try:
path = create(os.path.abspath(sys.argv[1]))
except IndexError:
path = create()
prepare()
additional_deps()
print 'virtualenv created at %s' % path
|