1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
|
#!/usr/bin/python
import base64
import json
import os
import os.path
import shutil
import StringIO
import sys
from helpers import (
apt_install,
command,
get_config,
install_extra_repository,
log,
run,
)
from local import (
buildbot_create,
buildbot_reconfig,
config_json,
DictDiffer,
generate_string,
slave_json,
)
REQUIRED_KEYS = [
'buildbot-pkg',
'installdir',
]
SUPPORTED_TRANSPORTS = ['bzr']
bzr = command('bzr')
def check_config(config):
# If all of the required keys are not present in the configuration
# then exit with an error.
if not all(config.get(k) for k in REQUIRED_KEYS):
log('All required items not configured: {}'.format(REQUIRED_KEYS))
return False
# If asked to fetch the config file then it must be a transport we support.
config_transport = config.get('config-transport')
config_url = config.get('config-url')
if config_url and config_transport not in SUPPORTED_TRANSPORTS:
log("{} is an unsupported transport".format(config_transport))
return False
return True
def handle_config_changes(config, diff):
log('Updating buildbot configuration.')
log('Configuration changes seen:')
log(str(diff))
buildbot_pkg = config.get('buildbot-pkg')
extra_repo = config.get('extra-repository')
extra_pkgs = config.get('extra-packages')
restart_required = False
# Add a new repository if it was just added.
if extra_repo and 'extra-repository' in diff.added_or_changed:
install_extra_repository(extra_repo)
restart_required = True
if extra_pkgs and 'extra_packages' not in diff.unchanged:
apt_install(
*(pkg.strip() for pkg in extra_pkgs.split()))
restart_required = True
if 'buildbot-pkg' not in diff.unchanged:
apt_install(buildbot_pkg)
restart_required = True
return restart_required
def configure_buildbot(config, diff):
buildbot_dir = config.get('installdir')
config_file = config.get('config-file')
master_cfg_path = os.path.join(buildbot_dir, 'master.cfg')
config_transport = config.get('config-transport')
config_url = config.get('config-url')
restart_required = False
# Write the buildbot config to disk (fetching it if necessary).
log("CONFIG FILE: {}".format(config_file))
log("ADDED OR CHANGED: {}".format(diff.added_or_changed))
if config_file and 'config-file' in diff.added_or_changed:
buildbot_create(buildbot_dir)
# This file will be moved to master.cfg.original in
# initialize_buildbot().
with open(master_cfg_path, 'w') as f:
base64.decode(StringIO.StringIO(config_file), f)
log('config_file decoded and written.')
restart_required = True
elif (config_transport == 'bzr' and config_url and
'config-transport' not in diff.unchanged and
'config-url' not in diff.unchanged):
# If the branch is private then more setup needs to be done. The
# gpg-agent needs to send the key over and the bzr launchpad-login
# needs to be set.
lp_id = config.get('config-user')
if lp_id:
bzr('launchpad-login', lp_id)
private_key = config.get('config-private-key')
if private_key:
# Set up the .ssh directory.
ssh_dir = os.expanduser('~/.ssh')
os.mkdir(ssh_dir)
os.chmod(ssh_dir, 0700)
with open(os.path.join(ssh_dir, 'lp_key', w)) as f:
f.write(base64.decode(private_key))
bzr('branch', '--use-existing-dir', config_url, buildbot_dir)
log('configuration fetched from {}'.format(config_url))
restart_required = True
return restart_required
def initialize_buildbot(config):
# Initialize the buildbot directory and (re)start buildbot.
buildbot_dir = config.get('installdir')
master_cfg_path = os.path.join(buildbot_dir, 'master.cfg')
shutil.move(master_cfg_path, master_cfg_path + '.original')
shutil.copy(
os.path.join(os.path.dirname(__file__), 'master.cfg'), master_cfg_path)
placeholder_path = os.path.join(buildbot_dir, 'placeholder.json')
if not os.path.exists(placeholder_path):
with open(placeholder_path, 'w') as f:
json.dump(generate_string("temporary-placeholder-"), f)
run('chown', '-R', 'ubuntu:ubuntu', buildbot_dir)
buildbot_reconfig()
def main():
config = get_config()
log(str(config))
if not check_config(config):
log("Configuration not valid.")
sys.exit(1)
prev_config = config_json.get()
diff = DictDiffer(config, prev_config)
if not diff.modified:
log("No configuration changes, exiting.")
sys.exit(0)
# Ensure the install directory exists.
buildbot_dir = config.get('installdir')
if not os.path.exists(buildbot_dir):
os.makedirs(buildbot_dir)
restart_required = (
handle_config_changes(config, diff) or
configure_buildbot(config, diff))
master_cfg_path = os.path.join(buildbot_dir, 'master.cfg')
if restart_required and os.path.exists(master_cfg_path):
initialize_buildbot(config)
else:
log("Configuration changed but didn't require restarting.")
config_json.set(config)
if __name__ == '__main__':
log('CONFIG-CHANGED HOOK:')
main()
|