4
Copyright (C) 2009 Intel Corporation
6
This library is free software; you can redistribute it and/or
7
modify it under the terms of the GNU Lesser General Public
8
License as published by the Free Software Foundation; either
9
version 2.1 of the License, or (at your option) version 3.
11
This library is distributed in the hope that it will be useful,
12
but WITHOUT ANY WARRANTY; without even the implied warranty of
13
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14
Lesser General Public License for more details.
16
You should have received a copy of the GNU Lesser General Public
17
License along with this library; if not, write to the Free Software
18
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21
import sys,os,glob,datetime
28
resultcheck.py: tranverse the test result directory, generate an XML
32
# sort more accurately on sub-second modification times
33
os.stat_float_times(True)
36
def check (resultdir, serverlist,resulturi, srcdir, shellprefix, backenddir):
37
'''Entrypoint, resutldir is the test result directory to be generated,
38
resulturi is the http uri, it will only process corresponding server's
39
test results list in severlist'''
41
servers = serverlist.split(",")
44
result = open("%s/nightly.xml" % resultdir,"w")
45
result.write('''<?xml version="1.0" encoding="utf-8" ?>\n''')
46
result.write('''<nightly-test>\n''')
48
if(os.path.isfile(resultdir+"/output.txt")==False):
49
print "main test output file not exist!"
51
indents,cont = step1(resultdir,result,indents,resultdir,resulturi, shellprefix, srcdir)
53
step2(resultdir,result,servers,indents,srcdir,shellprefix,backenddir)
55
# compare.xsl fails if there is no <client-test> element:
57
result.write('''<client-test/>\n''')
58
result.write('''</nightly-test>\n''')
61
patchsummary = re.compile('^Subject: (?:\[PATCH.*?\] )?(.*)\n')
62
patchauthor = re.compile('^From: (.*?) <.*>\n')
63
def extractPatchSummary(patchfile):
65
for line in open(patchfile):
66
m = patchauthor.match(line)
68
author = m.group(1) + " - "
70
m = patchsummary.match(line)
72
return author + m.group(1)
73
return os.path.basename(patchfile)
75
def step1(resultdir, result, indents, dir, resulturi, shellprefix, srcdir):
76
'''Step1 of the result checking, collect system information and
77
check the preparation steps (fetch, compile)'''
79
# Always keep checking, even if any of the preparation steps failed.
82
input = os.path.join(resultdir, "output.txt")
83
indent =indents[-1]+space
84
indents.append(indent)
86
# include information prepared by GitCopy in runtests.py
87
result.write(indent+'<source-info>\n')
88
files = os.listdir(resultdir)
91
m = re.match('(.*)-source.log', source)
94
result.write(' <source name="%s"><description><![CDATA[%s]]></description>\n' %
95
(name, open(os.path.join(resultdir, source)).read()))
96
result.write(' <patches>\n')
98
if fnmatch.fnmatch(patch, name + '-*.patch'):
99
result.write(' <patch><path>%s</path><summary><![CDATA[%s]]></summary></patch>\n' %
100
( patch, extractPatchSummary(os.path.join(resultdir, patch)) ) )
101
result.write(' </patches>\n')
102
result.write(' </source>\n')
103
result.write(indent+'</source-info>\n')
105
result.write(indent+'''<platform-info>\n''')
106
indent =indents[-1]+space
107
indents.append(indent)
108
result.write(indent+'''<cpuinfo>\n''')
109
fout=subprocess.check_output('cat /proc/cpuinfo|grep "model name" |uniq', shell=True)
110
result.write(indent+fout)
111
result.write(indent+'''</cpuinfo>\n''')
112
result.write(indent+'''<memoryinfo>\n''')
113
fout=subprocess.check_output('cat /proc/meminfo|grep "Mem"', shell=True)
114
for s in fout.split('\n'):
115
result.write(indent+s)
116
result.write(indent+'''</memoryinfo>\n''')
117
result.write(indent+'''<osinfo>\n''')
118
fout=subprocess.check_output('uname -osr'.split())
119
result.write(indent+fout)
120
result.write(indent+'''</osinfo>\n''')
121
if 'schroot' in shellprefix:
122
result.write(indent+'''<chrootinfo>\n''')
123
# Don't make assumption about the schroot invocation. Instead
124
# extract the schroot name from the environment of the shell.
125
name=subprocess.check_output(shellprefix + "sh -c 'echo $SCHROOT_CHROOT_NAME'",
127
info = re.sub(r'schroot .*', 'schroot -i -c ' + name, shellprefix)
128
fout=subprocess.check_output(info, shell=True)
130
for line in fout.split('\n'):
131
m = re.match(r'^\s+(Name|Description)\s+(.*)', line)
133
s.append(indent + m.group(1) + ': ' + m.group(2))
134
result.write('\n'.join(s))
135
result.write(indent+'''</chrootinfo>\n''')
136
result.write(indent+'''<libraryinfo>\n''')
137
libs = ['libsoup-2.4', 'evolution-data-server-1.2', 'glib-2.0','dbus-glib-1']
141
fout=subprocess.check_output(shellprefix+' pkg-config --modversion '+lib +' |grep -v pkg-config',
143
s = s + lib +': '+fout +' '
144
except subprocess.CalledProcessError:
146
result.write(indent+s)
147
result.write(indent+'''</libraryinfo>\n''')
150
result.write(indent+'''</platform-info>\n''')
151
result.write(indent+'''<prepare>\n''')
153
indents.append(indent)
154
tags=['libsynthesis', 'syncevolution', 'activesyncd', 'compile', 'dist', 'distcheck']
155
tagsp={'libsynthesis':'libsynthesis-source',
156
'syncevolution':'syncevolution-source',
157
'activesyncd':'activesyncd-source',
159
'distcheck': 'distcheck',
162
result.write(indent+'''<'''+tagsp[tag])
163
fout=subprocess.check_output('find `dirname '+input+'` -type d -name *'+tag, shell=True)
164
s = fout.rpartition('/')[2].rpartition('\n')[0]
165
result.write(' path ="'+s+'">')
166
'''check the result'''
167
if 0 == os.system("grep -q '^"+tag+": .*: failed' "+input):
168
result.write("failed")
169
elif 0 == os.system ("grep -q '^"+tag+" successful' "+input):
171
elif 0 == os.system("grep -q '^"+tag+".* disabled in configuration$' "+input):
172
result.write("skipped")
174
# Not listed at all? Fail.
175
result.write("failed")
176
result.write('''</'''+tagsp[tag]+'''>\n''')
179
result.write(indent+'''</prepare>\n''')
180
result.write(indent+'''<log-info>\n''')
182
indents.append(indent)
183
result.write(indent+'''<uri>'''+resulturi+'''</uri>\n''')
186
result.write(indent+'''</log-info>\n''')
189
return (indents, cont)
191
def step2(resultdir, result, servers, indents, srcdir, shellprefix, backenddir):
192
'''Step2 of the result checking, for each server listed in
193
servers, tranverse the corresponding result folder, process
194
each log file to decide the status of the testcase'''
195
'''Read the runtime parameter for each server '''
199
for server in servers:
200
cmd+= '-e /^'+server+'/p '
201
print "Analyzing overall result %s" % (resultdir+'/output.txt')
202
cmd = cmd +resultdir+'/output.txt'
203
fout=subprocess.check_output(cmd, shell=True)
204
for line in fout.split('\n'):
205
for server in servers:
206
# Find first line with "foobar successful" or "foobar: <command failure>",
208
if (line.startswith(server + ":") or line.startswith(server + " ")) and server not in params:
209
t = line.partition(server)[2]
210
if(t.startswith(':')):
211
t=t.partition(':')[2]
213
if t != 'skipped: disabled in configuration':
214
print "Result for %s: %s" % (server, t)
217
indent =indents[-1]+space
218
indents.append(indent)
219
'''start of testcase results '''
220
result.write(indent+'''<client-test>\n''')
221
runservers = os.listdir(resultdir)
222
#list source test servers statically, we have no idea how to differenciate
223
#automatically whether the server is a source test or sync test.
224
sourceServers = ['evolution',
229
'evolution-prebuilt-build',
244
#Only process servers listed in the input parameter and in the sourceServer
245
#list and have a result folder
246
for server in servers:
248
for rserver in runservers:
249
for source in sourceServers:
250
if (rserver.find('-')!=-1 and server == rserver.partition('-')[2] and server == source):
254
#put the servers at the front of the servers list, so that we will
256
servers.remove(server)
257
servers.insert (0, server);
258
sourceServersRun = sourceServersRun+1;
261
#process source tests first
264
indents.append(indent)
265
result.write(indent+'''<source>\n''')
268
for server in servers:
270
'''Only process servers listed in the input parametr'''
271
for rserver in runservers:
272
if(rserver.find('-')!= -1 and rserver.partition('-')[2] == server):
276
sourceServersRun = sourceServersRun -1;
277
if (sourceServersRun == -1):
279
'''generate a template which lists all test cases we supply, this helps
280
generate a comparable table and track potential uncontentional skipping
283
oldpath = os.getcwd()
284
# Get list of Client::Sync tests one source at a time (because
285
# the result might depend on CLIENT_TEST_SOURCES and which source
286
# is listed there first) and combine the result for the common
287
# data types (because some tests are only enable for contacts, others
289
# The order of the tests matters, so don't use a hash and start with
290
# a source which has only the common tests enabled. Additional tests
291
# then get added at the end.
292
clientSync = re.compile(r' +Client::Sync::(.*?)::(?:(Suspend|Resend|Retry)::)?([^:]+)')
293
for source in ('file_task', 'file_event', 'file_contact', 'eds_contact', 'eds_event'):
294
cmd = shellprefix + " env LD_LIBRARY_PATH=%s/build-synthesis/src/.libs SYNCEVOLUTION_BACKEND_DIR=%s CLIENT_TEST_PEER_CAN_RESTART=1 CLIENT_TEST_RETRY=t CLIENT_TEST_RESEND=t CLIENT_TEST_SUSPEND=t CLIENT_TEST_SOURCES=%s %s/client-test -h" % (srcdir, backenddir, source, srcdir)
295
fout=subprocess.check_output(cmd, shell=True)
296
for line in fout.split('\n'):
297
m = clientSync.match(line)
300
# special sub-grouping
301
test = m.group(2) + '__' + m.group(3)
304
if test and test not in templates:
305
templates.append(test)
307
indents.append(indent)
308
result.write(indent+'<sync>\n')
309
result.write(indent+space+'<template>')
310
for template in templates:
311
result.write(indent+space+'<'+template+'/>')
312
result.write('</template>\n')
314
indents.append(indent)
315
result.write(indent+'<'+server+' path="' +rserver+'" ')
316
#valgrind check resutls
317
if not params.get(server, None):
318
# Unknown result, treat as failure.
319
result.write('result="1"')
321
m = re.search(r'return code (\d+)', params[server])
323
result.write('result="%s"' % m.group(1))
325
# sort files by creation time, to preserve run order
326
logs = map(lambda file: (os.stat(file).st_mtime, file),
327
glob.glob(resultdir+'/'+rserver+'/*.log'))
329
logs = map(lambda entry: entry[1], logs)
332
if server in ('dbus', 'pim'):
333
# Extract tests and their results from output.txt,
334
# which contains Python unit test output. Example follows.
335
# Note that there can be arbitrary text between the test name
336
# and "ok" resp. "FAIL/ERROR". Therefore failed tests
337
# are identified not by those words but rather by the separate
338
# error reports at the end of the output. Those reports
339
# are split out into separate .log files for easy viewing
340
# via the .html report.
342
# TestDBusServer.testCapabilities - Server.Capabilities() ... ok
343
# TestDBusServer.testGetConfigScheduleWorld - Server.GetConfigScheduleWorld() ... ok
344
# TestDBusServer.testGetConfigsEmpty - Server.GetConfigsEmpty() ... ok
345
# TestDBusServer.testGetConfigsTemplates - Server.GetConfigsTemplates() ... FAIL
346
# TestDBusServer.testInvalidConfig - Server.NoSuchConfig exception ... ok
347
# TestDBusServer.testVersions - Server.GetVersions() ... ok
349
#======================================================================
350
# FAIL: TestDBusServer.testGetConfigsTemplates - Server.GetConfigsTemplates()
351
# ---------------------------------------------------------------------
353
# More recent Python 2.7 produces:
354
# FAIL: testSyncSecondSession (__main__.TestSessionAPIsReal)
356
# first build list of all tests, assuming that they pass
358
test_start = re.compile(r'''^Test(?P<cl>.*)\.test(?P<func>[^ ]*).*ok(?:ay)?$''')
359
# FAIL/ERROR + description of test (old Python)
360
test_fail = re.compile(r'''(?P<type>FAIL|ERROR): Test(?P<cl>.*)\.test(?P<func>[^ ]*)''')
361
# FAIL/ERROR + function name of test (Python 2.7)
362
test_fail_27 = re.compile(r'''(?P<type>FAIL|ERROR): test(?P<func>[^ ]*) \(.*\.(?:Test(?P<cl>.*))\)''')
367
for line in open(rserver + "/output.txt"):
368
m = test_start.search(line)
372
m = test_fail.search(line) or test_fail_27.search(line)
375
# create new (?!) log file
377
func = m.group("func")
378
newname = rserver + "/" + cl + "_" + func + ".log"
381
logfile = open(name, "w")
383
htmlfile.write('</pre></body')
386
htmlfile = open(name + ".html", "w")
387
htmlfile.write('''<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
390
<meta http-equiv="content-type" content="text/html; charset=None">
391
<style type="text/css">
392
td.linenos { background-color: #f0f0f0; padding-right: 10px; }
393
span.lineno { background-color: #f0f0f0; padding: 0 5px 0 5px; }
394
pre { line-height: 125%; }
395
body .hll { background-color: #ffffcc }
396
body { background: #f8f8f8; }
397
span.INFO { background: #c0c0c0 }
398
span.ERROR { background: #e0c0c0 }
399
span.hl { color: #c02020 }
404
if not dbustests.get(cl):
407
# okay: write a single line with the full test description
408
dbustests[cl][func] = "okay"
411
htmlfile.write('<span class="OKAY">%s</span></pre></body>' % cgi.escape(line))
415
# failed: start writing lines into separate log file
416
dbustests[cl][func] = m.group("type")
418
htmlfile.write('<a href="#dbus-traffic">D-Bus traffic</a> <a href="#stdout">output</a>\n\n')
422
if line == 'D-Bus traffic:\n':
424
htmlfile.write('<h3 id="dbus-traffic">D-Bus traffic:</h3>')
425
elif line == 'server output:\n':
427
htmlfile.write('<h3 id="stdout">server output:</h3>')
429
htmlfile.write('<span class="%s">%s</span>' % (linetype, cgi.escape(line)))
432
htmlfile.write('</pre></body')
438
indents.append(indent)
439
for testclass in dbustests:
440
result.write('%s<%s prefix="">\n' %
443
indents.append(indent)
444
for testfunc in dbustests[testclass]:
445
result.write('%s<%s>%s</%s>\n' %
447
dbustests[testclass][testfunc],
451
result.write('%s</%s>\n' %
457
logname = os.path.basename(log)
458
if logname in ['____compare.log',
459
'syncevo.log', # D-Bus server output
460
'dbus.log', # dbus-monitor output
463
# <path>/Client_Sync_eds_contact_testItems.log
464
# <path>/SyncEvo_CmdlineTest_testConfigure.log
465
# <path>/N7SyncEvo11CmdlineTestE_testConfigure.log - C++ name mangling?
466
m = re.match(r'^(Client_Source_|Client_Sync_|N7SyncEvo\d+|[^_]*_)(.*)_([^_]*)\.log', logname)
467
if not m or logname.endswith('.server.log'):
468
print "skipping", logname
470
# Client_Sync_, Client_Source_, SyncEvo_, ...
472
# eds_contact, CmdlineTest, ...
475
casename = m.group(3)
476
# special case grouping of some tests: include group inside casename instead of
478
# <path>/Client_Source_apple_caldav_LinkedItemsDefault_testLinkedItemsParent
479
m = re.match(r'(.*)_(LinkedItems\w+|Suspend|Resend|Retry)', format)
482
casename = m.group(2) + '::' + casename
485
print "skipping", logname
487
# Another special case: suspend/resend/retry uses an intermediate grouping
488
# which we can ignore because the name is repeated in the test case name.
489
# m = re.match(r'(.*)_(Suspend|Resend|Retry)', format)
491
# format = m.group(1)
492
# # Case name *not* extended, in contrast to the
493
# # LinkedItems case above.
494
# if '.' in casename:
496
# print "skipping", logname
498
print "analyzing log %s: prefix %s, subset %s, testcase %s" % (logname, prefix, format, casename)
499
if(format not in logdic):
501
logdic[format].append((casename, log))
502
logprefix[format]=prefix
503
for format in logdic.keys():
505
indents.append(indent)
506
prefix = logprefix[format]
508
# avoid + sign in element name (not allowed by XML);
509
# code reading XML must replace _- with + and __ with _
510
qformat = qformat.replace("_", "__");
511
qformat = qformat.replace("+", "_-");
512
result.write(indent+'<'+qformat+' prefix="'+prefix+'">\n')
513
for casename, log in logdic[format]:
515
indents.append(indent)
516
# must avoid :: in XML
517
tag = casename.replace('::', '__')
518
result.write(indent+'<'+tag+'>')
519
match=format+'::'+casename
520
matchOk=match+": okay \*\*\*"
521
matchKnownFailure=match+": \*\*\* failure ignored \*\*\*"
522
if not os.system("grep -q '" + matchKnownFailure + "' "+log):
523
result.write('knownfailure')
524
elif not os.system("tail -10 %s | grep -q 'external transport failure (local, status 20043)'" % log):
525
result.write('network')
526
elif os.system("grep -q '" + matchOk + "' "+log):
527
result.write('failed')
530
result.write('</'+tag+'>\n')
533
result.write(indent+'</'+qformat+'>\n')
536
result.write(indent+'</'+server+'>\n')
539
if(sourceServersRun == 0):
540
#all source servers have been processed, end the source tag and
542
result.write(indent+'''</source>\n''')
546
result.write(indent+'</sync>\n')
549
result.write(indent+'''</client-test>\n''')
551
indents = indents[-1]
553
if(__name__ == "__main__"):
554
if (len(sys.argv)!=7):
555
# srcdir and basedir must be usable inside the shell started by shellprefix (typically
557
print "usage: python resultchecker.py resultdir servers resulturi srcdir shellprefix backenddir"
added
removed
test/resultchecker.py
