~pwlars/ubuntu-test-cases/dashboard-no-fail

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
#!/usr/bin/python

import argparse
import datetime
import logging
import os
import shutil
import subprocess

import yaml

from phabletutils.environment import detect_device

import dashboard
import statsd

EMULATOR = os.environ.get('USE_EMULATOR')
if EMULATOR:
    def fake_detect(serial, device=None):
        log.info('faking detect device for emulator')
        return 'emulator'

    # detect_device doesn't support the emulator
    globals()['detect_device'] = fake_detect
    if 'ANDROID_SERIAL' not in os.environ:
        # we need something here or "serial required" logic fails
        os.environ['ANDROID_SERIAL'] = 'emulator-5554'

log = logging.getLogger()
script_dir = os.path.dirname(__file__)
res_dir = os.path.join(os.getcwd(), 'clientlogs')

dashboard_api = dashboard.API()


class SerialAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        log.info('android serial: %s', values[0])
        os.environ['ANDROID_SERIAL'] = values[0]


class DebugAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        log.setLevel(level=logging.DEBUG)
        log.debug('debug logging enabled')


def _serial_required():
    required = 'ANDROID_SERIAL' not in os.environ
    if required:
        try:
            out = subprocess.check_output(['adb', 'devices'])
            required = (len(out.decode().split('\n')) != 4)
        except subprocess.CalledProcessError as e:
            logging.debug('error getting adb devices: %s', e)
    return required


def _get_parser():
    parser = argparse.ArgumentParser(
        description='Run the complete test-execution-service suite.')

    parser.add_argument('-s', '--serial', action=SerialAction, nargs=1,
                        required=_serial_required(),
                        help='Android serial if more than one device present')
    parser.add_argument('--debug', action=DebugAction, nargs=0,
                        help='''Enable debug logging.''')

    parser.add_argument('--install-url',
                        help='''Flash with image from previous jenkins job.
                        This option will check if the device already has image
                        noted from this URL and will skip provisioning. The URL
                        should be the path the job like:
                        http://q-jenkins:8080/job/<your job>/<build number>''')
    parser.add_argument('-p', '--package', action='append',
                        help='Additional packages to install on target.')
    parser.add_argument('-P', '--ppa', action='append',
                        help='Additional PPA to configure on target.')
    parser.add_argument('-a', '--app', action='append',
                        help='Autopilot tests tor run.')
    parser.add_argument('-t', '--test', action='append',
                        help='UTAH tests tor run.')
    parser.add_argument('-r', '--revision', help='Image revision to install.')
    parser.add_argument('-n', '--no-provision', action='store_true',
                        help='Skip provisioning of the target device')
    parser.add_argument('--hooks-dir',
                        help='''A directory containing scripts to be run after
                        the target has been provisioned and before testing.''')
    parser.add_argument('--image-opt',
                        help='Options to pass to phablet-flash')
    parser.add_argument('--image-type', default='touch',
                        help='''Image type being tested. This can be changed
                        to 'touch_sf4p' so that SurfaceFlinger will be used
                        instead of Mir. default=%(default)s''')
    parser.add_argument('--num-workers', type=int, default=1,
                        help='''The total number of workers available for
                        running tests.''')
    parser.add_argument('--worker-idx', type=int, default=0,
                        help='The worker to allocate testing work to.')
    return parser


def _arg_from_env(args, attr, envkey, array):
    val = os.environ.get(envkey, False)
    if val:
        if array:
            setattr(args, attr, val.split())
        else:
            setattr(args, attr, val)
        del os.environ[envkey]


def _merge_env(args):
    '''When run in Jenkins everything comes as environment variables.

    Its makes a much simpler job this way. While command line args are
    much easier for a user.
    '''
    _arg_from_env(args, 'app', 'APPS', True)
    _arg_from_env(args, 'test', 'TESTS', True)
    _arg_from_env(args, 'package', 'PACKAGES', True)
    _arg_from_env(args, 'ppa', 'PPAS', True)
    _arg_from_env(args, 'image_opt', 'IMAGE_OPT', False)
    _arg_from_env(args, 'image_type', 'IMAGE_TYPE', False)
    _arg_from_env(args, 'install_url', 'INSTALL_URL', False)
    _arg_from_env(args, 'revision', 'REVISION', False)
    _arg_from_env(args, 'num_workers', 'workers', False)
    _arg_from_env(args, 'worker_idx', 'worker_idx', False)


def _assert_args(args):
    if args.install_url:
        # this means you shouldn't specify packages, ppas, or image options
        if args.package or args.ppa or args.image_opt:
            msg = 'ERROR: --install-url can\'t be used with ' \
                  '--package, -ppa, or --image_opt'
            print(msg)
            return False

    # don't bother the install_url check, a user might be copy/pasting and
    # doesn't hurt. Its just good to not encourage it.
    _merge_env(args)

    script = os.path.join(script_dir, '../jenkins/testconfig.py')
    if args.package and args.package[0] == 'ALL':
        logging.info('Discovering all required dependencies')
        out = subprocess.check_output(
            [script, 'packages', '-i', args.image_type])
        args.package = [x for x in out.decode().split()]

    if args.app and args.app[0] == 'ALL':
        logging.info('Discovering all autopilot tests')
        out = subprocess.check_output(
            [script, 'apps', '-i', args.image_type,
             '-t', str(args.num_workers), '-w', str(args.worker_idx)])
        args.app = [x for x in out.decode().split()]
        logging.info('Autopilot test list: {}'.format(' '.join(args.app)))

    if args.test and args.test[0].startswith('ALL'):
        logging.info('Discovering all UTAH tests')
        argv = [script, 'utah', '-i', args.image_type,
                '-t', str(args.num_workers), '-w', str(args.worker_idx)]
        if args.test[0] == 'ALL_INCLUDING_AUTOPILOT':
            argv.append('-a')
        out = subprocess.check_output(argv)
        args.test = [x for x in out.decode().split()]
        logging.info('Utah test list: {}'.format(' '.join(args.test)))

    logging.debug('ARGS: %r', args)

    statsd.gauge_it('PACKAGES', args.package)
    statsd.gauge_it('APPS', args.app)
    statsd.gauge_it('TESTS', args.test)

    return True


def _run(args, ignore_error=False):
    try:
        logging.info('Running: %s', ' '.join(args))
        subprocess.check_call(args)
    except subprocess.CalledProcessError:
        if ignore_error:
            logging.error('failed to run %r, continuing', args)
        else:
            exit(1)


def _image_info():
    info = subprocess.check_output(['adb', 'shell', 'sudo',
                                    'system-image-cli', '-i'])
    v_ver = u_ver = d_ver = channel = None
    for line in info.split('\n'):
        if not line.strip():
            continue
        key, val = line.split(':', 1)
        if key == 'version version':
            v_ver = val.strip()
        elif key == 'version ubuntu':
            u_ver = val.strip()
        elif key == 'version device':
            d_ver = val.strip()
        elif key == 'channel':
            channel = val.strip()
    ver = '%s:%s:%s' % (v_ver, u_ver, d_ver)
    # required for the jenkins job's build description
    print('= TOUCH IMAGE VERSION:' + ver)
    return ver, channel


def _assert_image(args):
    log.info('checking if device has proper image ...')
    os.environ['INSTALL_URL'] = args.install_url
    _run([os.path.join(script_dir, 'assert-image')])


def _write_utah(start, end, passed):
    passes = failures = rc = 0
    if passed:
        passes = 1
    else:
        rc = failures = 1

    delta = '%s' % (end - start)
    start = start.strftime('%Y-%m-%d %H:%M:%S')
    data = {
        'name': 'install-and-boot',
        'errors': 0,
        'failures': failures,
        'passes': passes,
        'fetch_errors': 0,
        'uname': 'n/a',
        'media-info': 'n/a',
        'install_type': 'n/a',
        'arch': 'n/a',
        'release': 'n/a',
        'build_number': 'n/a',
        'runlist': 'n/a',
        'ran_at': start,
        'commands': [{
            'cmd_type': 'testcase_test',
            'command': 'provision',
            'returncode': rc,
            'start_time': start,
            'time_delta': delta,
            'stderr': '',
            'stdout': '',
            'testcase': 'boot',
            'testsuite': 'install-and-boot',
        }]
    }
    path = os.path.join(res_dir, 'install-and-boot')
    if not os.path.exists(path):
        os.mkdir(path)
    with open(os.path.join(path, 'utah.yaml'), 'w') as f:
        f.write(yaml.safe_dump(data, default_flow_style=False))


def _post_install_hooks(args):
    if not args.hooks_dir:
        return
    log.info('running post install hooks ...')
    if not os.path.isdir(args.hooks_dir):
        log.warn('hooks directory (%s) does not exist ... skipping',
                 args.hooks_dir)
    for hook in sorted(os.listdir(args.hooks_dir)):
        s = os.stat(os.path.join(args.hooks_dir, hook))
        if s.st_mode & os.path.stat.S_IXUSR == 0:
            log.warn('skipping hook (%s) - not executable', hook)
            continue
        log.info('executing hook: %s', hook)
        hook = os.path.join(args.hooks_dir, hook)
        subprocess.check_call([hook])


def _provision(args):
    log.info('provisioning device ...')
    if args.image_opt:
        log.debug('overriding IMAGE_OPT with: %s', args.image_opt)
        os.environ['IMAGE_OPT'] = args.image_opt

    cargs = [os.path.join(script_dir, 'provision.sh'), '-i', args.image_type]

    if args.package:
        for p in args.package:
            cargs.extend(['-p', p])
    if args.ppa:
        for p in args.ppa:
            cargs.extend(['-P', p])
    if not args.ppa and not args.package:
        # All tests require a writeable system the -p and -P args
        # implicitly create a writable system. so we have to ensure here:
        cargs.append('-w')
    if args.revision:
        cargs.extend(['-r', args.revision])

    with statsd.time_it('provision'):
        start = datetime.datetime.utcnow()
        passed = False
        try:
            _run(cargs)
            _post_install_hooks(args)
            passed = True
        finally:
            end = datetime.datetime.utcnow()
            _write_utah(start, end, passed)


def _test_autopilot(args, build, image):
    if args.app:
        if build:
            os.environ['DASHBOARD_BUILD'] = build
        if image:
            os.environ['DASHBOARD_IMAGE'] = image
        cargs = [os.path.join(script_dir, 'run-autopilot-tests.sh')]
        for app in args.app:
            cargs.extend(['-a', app])
        with statsd.time_it('APPS'):
            _run(cargs)


def _sync_results(build, image, test, fname):
    with open(fname) as f:
        d = yaml.safe_load(f)
        if image and build:
            try:
                dashboard_api.result_syncing(image, build, test, d)
            except:
                log.warn('Dashboard live update failed')


def _test_utah(args, build, image):
    if args.test:
        cargs = [os.path.join(script_dir, 'jenkins.sh')]
        with statsd.time_it('TESTS'):
            for test in args.test:
                os.environ['RESDIR'] = os.path.join(res_dir, test)
                if image and build:
                    try:
                        dashboard_api.result_running(image, build, test)
                    except:
                        log.warn('Dashboard live update failed')
                _run(cargs + ['-a', test, '-p', '/tmp/results'],
                     ignore_error=True)
                fname = os.path.join(res_dir, test, 'utah.yaml')
                _sync_results(build, image, test, fname)


def _image_add(args):
    build_number, channel = _image_info()
    # get the release series (ex. trusty, utopic)
    # this is set in the job environment variable IMAGE_SERIES
    release = os.environ.get('IMAGE_SERIES')
    if release:
        try:
            img = dashboard_api.image_add(build_number, release,
                                          args.image_type,
                                          detect_device(None), 'ubuntu')
        except:
            img = None
            log.warn('Dashboard live update failed')
        return img


def main(args):
    with statsd.time_it('main'):
        if os.path.exists(res_dir):
            logging.info('deleting old result directory: %s', res_dir)
            shutil.rmtree(res_dir)
        os.mkdir(res_dir)

        job_name = os.environ.get('JOB_NAME', '')
        job_number = os.environ.get('BUILD_NUMBER', '')
        try:
            build = dashboard_api.build_add(job_name, job_number)
        except:
            build = None
            log.warn('Dashboard live update failed')

        if args.no_provision:
            logging.info('Skipping the provisioning step as requested')
        elif args.install_url:
            _assert_image(args)
        else:
            _provision(args)

        # TODO - this should be incororated into provision and assert_image
        # so that the status is updated *before* flashing rather than after
        image = _image_add(args)

        if args.test and image and build:
            for x in args.test:
                try:
                    dashboard_api.result_queue(image, build, x)
                except:
                    log.warn('Dashboard live update failed')
        if args.app and image and build:
            for x in args.app:
                try:
                    dashboard_api.result_queue(image, build, x)
                except:
                    log.warn('Dashboard live update failed')

        _test_utah(args, build, image)
        _test_autopilot(args, build, image)

    return 0


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    log.name = 'run-smoke'
    dashboard.log = logging.getLogger('dashboard')

    args = _get_parser().parse_args()
    if not _assert_args(args):
        exit(1)

    exit(main(args))