1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
|
#!/usr/bin/env python3
import argparse
from contextlib import contextmanager
from datetime import datetime
import json
import os
import re
import shutil
import subprocess
import sys
import stat
import textwrap
import tempfile
from confinement.generate import generate_confinement
sys.path.append('./py')
from py.embedded_providers import EmbeddedProvider1PlugInCollection
from py.checkbox_touch import CheckboxTouchApplication
from utils import get_package_from_url_and_extract
from utils import prepare_uris
from utils import rsync_tree
# DEFAULT_PROVIDERS is a list of providers directory names as listed in
# ../providers. NOTE: those are directory names, not provider names as reported
# by manage.py info
DEFAULT_PROVIDERS = [
"2015.com.canonical.certification:qml-tests",
"plainbox-provider-ubuntu-touch",
]
@contextmanager
def chdir(path):
previous_path = os.getcwd()
os.chdir(path)
yield
os.chdir(previous_path)
def find_executables(path):
res = []
for node in os.listdir(path):
full_path = os.path.join(path, node)
if os.path.isdir(full_path):
res += find_executables(full_path)
else:
st = os.stat(full_path)
mode = st.st_mode
executable = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
if mode & executable:
res.append(full_path)
return res
def get_confined_jobs(path):
collection = EmbeddedProvider1PlugInCollection(path)
# this should be run within provider's directory, so there should be
# only one provider loaded
provider = collection.get_all_plugin_objects()[0]
for job in provider.job_list:
if job.plugin == 'qml' and 'confined' in job.get_flag_set():
yield job
def check_libs_present():
"""
Check if paths listed in NECESSARY_PATHS are present.
This is a simple heuristic to check if get-libs prior to building
click package.
"""
NECESSARY_PATHS = [
"lib/arm-linux-gnueabihf/io/thp/pyotherside/libpyothersideplugin.so",
"lib/arm-linux-gnueabihf/libpython3.5m.so.1",
"lib/py/plainbox",
"lib/py/plainbox.egg-info",
"lib/py/xlsxwriter"]
for path in NECESSARY_PATHS:
if not os.path.exists(path):
raise EnvironmentError(
"{} not found!\nHave you run get-libs?".format(path))
def check_versions_match(manifest):
"""Check if version coded in python matches one from manifest.json."""
res = CheckboxTouchApplication().get_version_pair()
version_from_py = res['result']['application_version']
if manifest['version'] != version_from_py:
raise Exception(
("Version mismatch! Version from checkbox_touch.py: {}, "
"version from manifest.json: {}").format(
version_from_py, manifest['version']))
def get_revision_string(path='.'):
# Try getting revno from bazaar vcs
try:
revno = int(subprocess.check_output(
["bzr", "revno", path], stderr=subprocess.STDOUT))
return "bzr r{}".format(revno)
except subprocess.CalledProcessError:
# problem encountered when run bzr revno - falling through
pass
# Try getting version from git
try:
revno = subprocess.check_output(
["git", "-C", path, "show", "-s", "--format=%h", "HEAD"],
stderr=subprocess.STDOUT).decode(sys.stdout.encoding).strip()
return "git {}".format(revno)
except subprocess.CalledProcessError:
# problem encountered when run git revision - falling through
pass
return "Unknown revision"
def main():
parser = argparse.ArgumentParser(
description="Get necessary libs for checkbox-touch")
parser.add_argument("--embedded-providers-path", action='store', type=str,
nargs='*', help="Paths to directories with providers")
parser.add_argument("--install",
action='store_true',
help=("Use adb to push and install click package on"
" the device"))
parser.add_argument("--testplan",
action='store', default="", type=str,
help="Test plan to set as the default one")
parser.add_argument("--provider",
action='append', type=str,
help=("Path to a provider that should be included. "
"--provider might be used multiple times."))
parser.add_argument("-s", action='store', type=str,
help=("Serial number of the device to use when using "
"adb. Overrides ANDROID_SERIAL"))
args = parser.parse_args()
check_libs_present()
# generate setting.json
settings = {
"_comment": "file generated automatically with {0}"
.format(parser.prog),
"revision": get_revision_string(),
"clickBuildDate": str(datetime.now().date()),
"testplan": args.testplan,
"providersDir": "providers"
}
settings_file = open('settings.json', 'w')
settings_file.write(json.dumps(settings, sort_keys=True, indent=4))
settings_file.close()
if args.embedded_providers_path is not None:
for path in args.embedded_providers_path:
rsync_tree(path, 'providers')
if args.provider is not None:
for path in args.provider:
target_base_name = os.path.basename(os.path.normpath(path))
rsync_tree(path, os.path.join('providers', target_base_name))
if args.provider is None and args.embedded_providers_path is None:
# no user specified providers
if not os.path.exists('providers') or os.listdir('providers') == []:
print("No providers specified; using default providers")
for provider in DEFAULT_PROVIDERS:
path = os.path.join(os.getcwd(), '..', 'providers', provider)
target_base_name = os.path.basename(os.path.normpath(path))
rsync_tree(path, os.path.join(
'providers', target_base_name))
if not validate_providers():
sys.exit('Provider validation failed.')
extract_dependencies()
current_manifest = json.loads(open('manifest.json', 'rt').read())
check_versions_match(current_manifest)
hooks = {
'checkbox-touch': current_manifest['hooks']['checkbox-touch']
}
for hook in build_confinement(current_manifest['version']):
hooks.update(hook)
new_manifest = current_manifest.copy()
new_manifest['hooks'] = hooks
with open('manifest.json', 'wt') as f:
f.write(json.dumps(new_manifest, sort_keys=True, indent=4))
build_i18n()
generate_desktop_file()
if not os.path.exists('../build-cbt'):
os.makedirs('../build-cbt')
pkg = build_click()
if args.install:
adb_options = []
if args.s:
adb_options = ['-s', args.s]
install_click(pkg, adb_options)
def validate_providers():
print("Validating providers")
providers_valid = True
for provider_dir in os.listdir('providers'):
try:
subprocess.check_output(
[os.path.join('providers', provider_dir, 'manage.py'),
'validate'], stderr=subprocess.STDOUT)
except OSError as e:
print(e.strerror)
providers_valid = False
except subprocess.CalledProcessError as e:
output_lines = e.output.decode(sys.stdout.encoding).split('\n')
if (len(output_lines) > 0 and
"ImportError: No module named 'plainbox'" in output_lines):
print('Plainbox not found. Install plainbox or run the'
'command from the virtual env.')
return False
else:
print(("Problem encountered when validating '{}'."
" Output:\n{}").format(
provider_dir, e.output.decode(sys.stdout.encoding)))
providers_valid = False
return providers_valid
def build_confinement(checkbox_version):
print("Generating confinement")
checkbox_name = "com.ubuntu.checkbox_checkbox-touch_" + checkbox_version
new_hooks = []
for provider_dir in os.listdir('providers'):
with chdir(os.path.join('providers', provider_dir)):
for job in get_confined_jobs('.'):
print(" job: {}".format(job.id))
qml_file = os.path.relpath(
job.qml_file, 'data')
new_hooks.append(generate_confinement(
provider_dir, job.partial_id, checkbox_name, qml_file))
return new_hooks
def build_i18n():
print("Building i18n")
for provider_dir in os.listdir('providers'):
try:
subprocess.check_output(
[os.path.join('providers', provider_dir, 'manage.py'),
'i18n'], stderr=subprocess.STDOUT)
except (OSError, subprocess.CalledProcessError) as e:
print("Problem encountered while building translations for ",
"provider '{}'.".format(provider_dir))
raise e
def extract_dependencies():
print("Extracting dependencies")
for provider_dir in os.listdir('providers'):
debs = []
debs_file = os.path.join('providers', provider_dir, '_extra_debs')
if os.path.exists(debs_file):
with open(debs_file, 'rt') as f:
for deb in f.read().split():
debs.append(deb)
uris = prepare_uris(debs)
for deb in debs:
with tempfile.TemporaryDirectory() as tmp:
get_package_from_url_and_extract(uris[deb], tmp)
bin_path = os.path.join(tmp, 'usr', 'bin')
if os.path.exists(bin_path):
executables = find_executables(bin_path)
provider_bin = os.path.join(
'providers', provider_dir, 'bin')
if executables:
os.makedirs(provider_bin, exist_ok=True)
for f in executables:
target = os.path.join(
'providers', provider_dir, 'bin',
os.path.split(f)[1])
if os.path.exists(target):
print('Warning! {} already exists and will be'
' skipped'.format(target))
else:
shutil.copyfile(f, target)
shutil.copymode(f, target)
lib_path = os.path.join(tmp, 'usr', 'lib')
if os.path.exists(lib_path):
rsync_tree(lib_path, 'lib', preserve_symlinks=1)
def generate_desktop_file():
template = textwrap.dedent("""
# This file has been generated by build-me script
[Desktop Entry]
Name=Checkbox
Comment=System testing utility for Ubuntu
Exec=qmlscene --settings=settings.json {import_options} $@ checkbox-touch.qml
Icon=checkbox-touch.svg
Terminal=false
Type=Application
X-Ubuntu-Touch=true
X-Ubuntu-Supported-Orientations=portrait
""")
import_opts = ['-I lib/py/plainbox/data/plainbox-qml-modules']
for provider in os.listdir('providers'):
provider_data_dir = os.path.join('providers', provider, 'data')
if os.path.exists(provider_data_dir):
import_opts.append("-I " + provider_data_dir)
with open('checkbox-touch.desktop', 'wt', encoding='utf-8') as f:
f.write(template.format(import_options=" ".join(import_opts)))
def build_click():
print('Building click package')
base_path = os.path.normpath(os.path.join(os.getcwd(), '..'))
click_source_path = os.path.join(base_path, 'checkbox-touch')
out = subprocess.check_output(
['click', 'build', click_source_path],
cwd=os.path.join(base_path, 'build-cbt'))
pkg_name = os.path.join(
base_path, 'build-cbt', re.search(
"'\.\/(.*)\.click'", str(out)).group(1) + '.click')
print('Your click package is available here: {0}'.format(pkg_name))
return pkg_name
def install_click(pkg, adb_options=[]):
print('Pushing to the device')
adb_base = ['adb'] + adb_options
try:
subprocess.check_output(adb_base + ['push', pkg, '/tmp'])
except subprocess.CalledProcessError:
sys.exit('Error ecountered while pushing to the device.')
print('Installing click package')
path = '/tmp/' + os.path.basename(pkg)
try:
subprocess.check_output(adb_base + ['shell', ('pkcon install-local'
' --allow-untrusted -p -y {path}').format(
path=path)])
except subprocess.CalledProcessError:
sys.exit('Error ecountered while installing click package.')
if __name__ == "__main__":
main()
|