1
by Matthias Klose
Import upstream version 2.6.1 |
1 |
#!/bin/sh
|
2 |
||
3 |
## Script to build and test the latest python from svn. It basically
|
|
4 |
## does this:
|
|
5 |
## svn up ; ./configure ; make ; make test ; make install ; cd Doc ; make
|
|
6 |
##
|
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
7 |
## Logs are kept and rsync'ed to the webhost. If there are test failure(s),
|
1
by Matthias Klose
Import upstream version 2.6.1 |
8 |
## information about the failure(s) is mailed.
|
9 |
##
|
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
10 |
## The user must be a member of the webmaster group locally and on webhost.
|
11 |
##
|
|
1
by Matthias Klose
Import upstream version 2.6.1 |
12 |
## This script is run on the PSF's machine as user neal via crontab.
|
13 |
##
|
|
14 |
## Yes, this script would probably be easier in python, but then
|
|
15 |
## there's a bootstrap problem. What if Python doesn't build?
|
|
16 |
##
|
|
17 |
## This script should be fairly clean Bourne shell, ie not too many
|
|
18 |
## bash-isms. We should try to keep it portable to other Unixes.
|
|
19 |
## Even though it will probably only run on Linux. I'm sure there are
|
|
20 |
## several GNU-isms currently (date +%s and readlink).
|
|
21 |
##
|
|
22 |
## Perhaps this script should be broken up into 2 (or more) components.
|
|
23 |
## Building doc is orthogonal to the rest of the python build/test.
|
|
24 |
##
|
|
25 |
||
26 |
## FIXME: we should detect test hangs (eg, if they take more than 45 minutes)
|
|
27 |
||
28 |
## FIXME: we should run valgrind
|
|
29 |
## FIXME: we should run code coverage
|
|
30 |
||
31 |
## Utilities invoked in this script include:
|
|
32 |
## basename, date, dirname, expr, grep, readlink, uname
|
|
33 |
## cksum, make, mutt, rsync, svn
|
|
34 |
||
35 |
## remember where did we started from
|
|
36 |
DIR=`dirname $0` |
|
37 |
if [ "$DIR" = "" ]; then |
|
38 |
DIR="." |
|
39 |
fi
|
|
40 |
||
41 |
## make directory absolute
|
|
42 |
DIR=`readlink -f $DIR` |
|
43 |
FULLPATHNAME="$DIR/`basename $0`" |
|
44 |
## we want Misc/..
|
|
45 |
DIR=`dirname $DIR` |
|
46 |
||
47 |
## Configurable options
|
|
48 |
||
49 |
FAILURE_SUBJECT="Python Regression Test Failures" |
|
50 |
#FAILURE_MAILTO="YOUR_ACCOUNT@gmail.com"
|
|
51 |
FAILURE_MAILTO="python-checkins@python.org" |
|
52 |
#FAILURE_CC="optional--uncomment and set to desired address"
|
|
53 |
||
54 |
REMOTE_SYSTEM="neal@dinsdale.python.org" |
|
55 |
REMOTE_DIR="/data/ftp.python.org/pub/www.python.org/doc/current" |
|
56 |
REMOTE_DIR_DIST="/data/ftp.python.org/pub/python/doc/current" |
|
57 |
RESULT_FILE="$DIR/build/index.html" |
|
58 |
INSTALL_DIR="/tmp/python-test-2.6/local" |
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
59 |
RSYNC_OPTS="-C -e ssh -rlogD" |
1
by Matthias Klose
Import upstream version 2.6.1 |
60 |
|
61 |
# Always run the installed version of Python.
|
|
62 |
PYTHON=$INSTALL_DIR/bin/python |
|
63 |
||
64 |
# Python options and regression test program that should always be run.
|
|
65 |
REGRTEST_ARGS="-E -tt $INSTALL_DIR/lib/python2.6/test/regrtest.py" |
|
66 |
||
67 |
REFLOG="build/reflog.txt.out" |
|
68 |
# These tests are not stable and falsely report leaks sometimes.
|
|
69 |
# The entire leak report will be mailed if any test not in this list leaks.
|
|
70 |
# Note: test_XXX (none currently) really leak, but are disabled
|
|
71 |
# so we don't send spam. Any test which really leaks should only
|
|
72 |
# be listed here if there are also test cases under Lib/test/leakers.
|
|
73 |
LEAKY_TESTS="test_(asynchat|cmd_line|docxmlrpc|dumbdbm|file|ftplib|httpservers|imaplib|popen2|socket|smtplib|sys|telnetlib|threadedtempfile|threading|threadsignals|urllib2_localnet|xmlrpc)" |
|
74 |
||
75 |
# Skip these tests altogether when looking for leaks. These tests
|
|
76 |
# do not need to be stored above in LEAKY_TESTS too.
|
|
77 |
# test_compiler almost never finishes with the same number of refs
|
|
78 |
# since it depends on other modules, skip it.
|
|
79 |
# test_logging causes hangs, skip it.
|
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
80 |
# KBK 21Apr09: test_httpservers causes hangs, skip for now.
|
81 |
LEAKY_SKIPS="-x test_compiler test_logging test_httpservers" |
|
1
by Matthias Klose
Import upstream version 2.6.1 |
82 |
|
83 |
# Change this flag to "yes" for old releases to only update/build the docs.
|
|
84 |
BUILD_DISABLED="yes" |
|
85 |
||
86 |
## utility functions
|
|
87 |
current_time() { |
|
88 |
date +%s |
|
89 |
}
|
|
90 |
||
91 |
update_status() { |
|
92 |
now=`current_time` |
|
93 |
time=`expr $now - $3` |
|
94 |
echo "<li><a href=\"$2\">$1</a> <font size=\"-1\">($time seconds)</font></li>" >> $RESULT_FILE |
|
95 |
}
|
|
96 |
||
97 |
place_summary_first() { |
|
98 |
testf=$1 |
|
99 |
sed -n '/^[0-9][0-9]* tests OK\./,$p' < $testf \ |
|
100 |
| egrep -v '\[[0-9]+ refs\]' > $testf.tmp |
|
101 |
echo "" >> $testf.tmp |
|
102 |
cat $testf >> $testf.tmp |
|
103 |
mv $testf.tmp $testf |
|
104 |
}
|
|
105 |
||
106 |
count_failures () { |
|
107 |
testf=$1 |
|
108 |
n=`grep -ic " failed:" $testf` |
|
109 |
if [ $n -eq 1 ] ; then |
|
110 |
n=`grep " failed:" $testf | sed -e 's/ .*//'` |
|
111 |
fi
|
|
112 |
echo $n |
|
113 |
}
|
|
114 |
||
115 |
mail_on_failure() { |
|
116 |
if [ "$NUM_FAILURES" != "0" ]; then |
|
117 |
dest=$FAILURE_MAILTO |
|
118 |
# FAILURE_CC is optional.
|
|
119 |
if [ "$FAILURE_CC" != "" ]; then |
|
120 |
dest="$dest -c $FAILURE_CC" |
|
121 |
fi
|
|
122 |
if [ "x$3" != "x" ] ; then |
|
123 |
(echo "More important issues:" |
|
124 |
echo "----------------------" |
|
125 |
egrep -v "$3" < $2 |
|
126 |
echo "" |
|
127 |
echo "Less important issues:" |
|
128 |
echo "----------------------" |
|
129 |
egrep "$3" < $2) |
|
130 |
else
|
|
131 |
cat $2
|
|
132 |
fi | mutt -s "$FAILURE_SUBJECT $1 ($NUM_FAILURES)" $dest |
|
133 |
fi
|
|
134 |
}
|
|
135 |
||
136 |
## setup
|
|
137 |
cd $DIR |
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
138 |
make clobber > /dev/null 2>&1 |
139 |
cp -p Modules/Setup.dist Modules/Setup |
|
140 |
# But maybe there was no Makefile - we are only building docs. Clear build:
|
|
141 |
rm -rf build/ |
|
1
by Matthias Klose
Import upstream version 2.6.1 |
142 |
mkdir -p build |
143 |
rm -rf $INSTALL_DIR
|
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
144 |
## get the path we are building
|
145 |
repo_path=$(grep "url=" .svn/entries | sed -e s/\\W*url=// -e s/\"//g) |
|
1
by Matthias Klose
Import upstream version 2.6.1 |
146 |
|
147 |
## create results file
|
|
148 |
TITLE="Automated Python Build Results" |
|
149 |
echo "<html>" >> $RESULT_FILE |
|
150 |
echo " <head>" >> $RESULT_FILE |
|
151 |
echo " <title>$TITLE</title>" >> $RESULT_FILE |
|
152 |
echo " <meta http-equiv=\"refresh\" content=\"43200\">" >> $RESULT_FILE |
|
153 |
echo " </head>" >> $RESULT_FILE |
|
154 |
echo "<body>" >> $RESULT_FILE |
|
155 |
echo "<h2>Automated Python Build Results</h2>" >> $RESULT_FILE |
|
156 |
echo "<table>" >> $RESULT_FILE |
|
157 |
echo " <tr>" >> $RESULT_FILE |
|
158 |
echo " <td>Built on:</td><td>`date`</td>" >> $RESULT_FILE |
|
159 |
echo " </tr><tr>" >> $RESULT_FILE |
|
160 |
echo " <td>Hostname:</td><td>`uname -n`</td>" >> $RESULT_FILE |
|
161 |
echo " </tr><tr>" >> $RESULT_FILE |
|
162 |
echo " <td>Platform:</td><td>`uname -srmpo`</td>" >> $RESULT_FILE |
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
163 |
echo " </tr><tr>" >> $RESULT_FILE |
164 |
echo " <td>URL:</td><td>$repo_path</td>" >> $RESULT_FILE |
|
1
by Matthias Klose
Import upstream version 2.6.1 |
165 |
echo " </tr>" >> $RESULT_FILE |
166 |
echo "</table>" >> $RESULT_FILE |
|
167 |
echo "<ul>" >> $RESULT_FILE |
|
168 |
||
169 |
## update, build, and test
|
|
170 |
ORIG_CHECKSUM=`cksum $FULLPATHNAME` |
|
171 |
F=svn-update.out |
|
172 |
start=`current_time` |
|
173 |
svn update >& build/$F |
|
174 |
err=$? |
|
175 |
update_status "Updating" "$F" $start |
|
176 |
if [ $err = 0 -a "$BUILD_DISABLED" != "yes" ]; then |
|
177 |
## FIXME: we should check if this file has changed.
|
|
178 |
## If it has changed, we should re-run the script to pick up changes.
|
|
179 |
if [ "$ORIG_CHECKSUM" != "$ORIG_CHECKSUM" ]; then |
|
180 |
exec $FULLPATHNAME $@ |
|
181 |
fi
|
|
182 |
||
183 |
F=svn-stat.out |
|
184 |
start=`current_time` |
|
185 |
svn stat >& build/$F |
|
186 |
## ignore some of the diffs
|
|
187 |
NUM_DIFFS=`egrep -vc '^. (@test|db_home|Lib/test/(regrtest\.py|db_home))$' build/$F` |
|
188 |
update_status "svn stat ($NUM_DIFFS possibly important diffs)" "$F" $start |
|
189 |
||
190 |
F=configure.out |
|
191 |
start=`current_time` |
|
192 |
./configure --prefix=$INSTALL_DIR --with-pydebug >& build/$F |
|
193 |
err=$? |
|
194 |
update_status "Configuring" "$F" $start |
|
195 |
if [ $err = 0 ]; then |
|
196 |
F=make.out |
|
197 |
start=`current_time` |
|
198 |
make >& build/$F |
|
199 |
err=$? |
|
200 |
warnings=`grep warning build/$F | egrep -vc "te?mpnam(_r|)' is dangerous,"` |
|
201 |
update_status "Building ($warnings warnings)" "$F" $start |
|
202 |
if [ $err = 0 ]; then |
|
203 |
## make install
|
|
204 |
F=make-install.out |
|
205 |
start=`current_time` |
|
206 |
make install >& build/$F |
|
207 |
update_status "Installing" "$F" $start |
|
208 |
||
209 |
if [ ! -x $PYTHON ]; then |
|
210 |
ln -s ${PYTHON}2.* $PYTHON |
|
211 |
fi
|
|
212 |
||
213 |
## make and run basic tests
|
|
214 |
F=make-test.out |
|
215 |
start=`current_time` |
|
216 |
$PYTHON $REGRTEST_ARGS -u urlfetch >& build/$F |
|
217 |
NUM_FAILURES=`count_failures build/$F` |
|
218 |
place_summary_first build/$F
|
|
219 |
update_status "Testing basics ($NUM_FAILURES failures)" "$F" $start |
|
220 |
mail_on_failure "basics" build/$F |
|
221 |
||
222 |
F=make-test-opt.out |
|
223 |
start=`current_time` |
|
224 |
$PYTHON -O $REGRTEST_ARGS -u urlfetch >& build/$F |
|
225 |
NUM_FAILURES=`count_failures build/$F` |
|
226 |
place_summary_first build/$F
|
|
227 |
update_status "Testing opt ($NUM_FAILURES failures)" "$F" $start |
|
228 |
mail_on_failure "opt" build/$F |
|
229 |
||
230 |
## run the tests looking for leaks
|
|
231 |
F=make-test-refleak.out |
|
232 |
start=`current_time` |
|
233 |
## ensure that the reflog exists so the grep doesn't fail
|
|
234 |
touch $REFLOG
|
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
235 |
$PYTHON $REGRTEST_ARGS -R 4:3:$REFLOG -u network $LEAKY_SKIPS >& build/$F |
1
by Matthias Klose
Import upstream version 2.6.1 |
236 |
LEAK_PAT="($LEAKY_TESTS|sum=0)" |
237 |
NUM_FAILURES=`egrep -vc "$LEAK_PAT" $REFLOG` |
|
238 |
place_summary_first build/$F
|
|
239 |
update_status "Testing refleaks ($NUM_FAILURES failures)" "$F" $start |
|
240 |
mail_on_failure "refleak" $REFLOG "$LEAK_PAT" |
|
241 |
||
242 |
## now try to run all the tests
|
|
243 |
F=make-testall.out |
|
244 |
start=`current_time` |
|
245 |
## skip curses when running from cron since there's no terminal
|
|
246 |
## skip sound since it's not setup on the PSF box (/dev/dsp)
|
|
247 |
$PYTHON $REGRTEST_ARGS -uall -x test_curses test_linuxaudiodev test_ossaudiodev >& build/$F |
|
248 |
NUM_FAILURES=`count_failures build/$F` |
|
249 |
place_summary_first build/$F
|
|
250 |
update_status "Testing all except curses and sound ($NUM_FAILURES failures)" "$F" $start |
|
251 |
mail_on_failure "all" build/$F |
|
252 |
fi
|
|
253 |
fi
|
|
254 |
fi
|
|
255 |
||
256 |
||
257 |
## make doc
|
|
258 |
cd $DIR/Doc |
|
259 |
F="make-doc.out" |
|
260 |
start=`current_time` |
|
1.1.7
by Matthias Klose
Import upstream version 2.6.5~rc2 |
261 |
make clean > ../build/$F 2>&1 |
262 |
make checkout html >> ../build/$F 2>&1 |
|
263 |
err=$? |
|
1
by Matthias Klose
Import upstream version 2.6.1 |
264 |
update_status "Making doc" "$F" $start |
265 |
if [ $err != 0 ]; then |
|
266 |
NUM_FAILURES=1 |
|
267 |
mail_on_failure "doc" ../build/$F |
|
268 |
fi
|
|
269 |
||
270 |
F="make-doc-dist.out" |
|
271 |
start=`current_time` |
|
272 |
if [ $conflict_count == 0 ]; then |
|
273 |
make dist >& ../build/$F |
|
274 |
err=$? |
|
275 |
fi
|
|
276 |
update_status "Making downloadable doc" "$F" $start |
|
277 |
if [ $err != 0 ]; then |
|
278 |
NUM_FAILURES=1 |
|
279 |
mail_on_failure "doc dist" ../build/$F |
|
280 |
fi
|
|
281 |
||
282 |
echo "</ul>" >> $RESULT_FILE |
|
283 |
echo "</body>" >> $RESULT_FILE |
|
284 |
echo "</html>" >> $RESULT_FILE |
|
285 |
||
286 |
## copy results
|
|
1.1.8
by Matthias Klose
Import upstream version 2.6.5 |
287 |
## (not used anymore, the daily build is now done directly on the server)
|
288 |
#chgrp -R webmaster build/html
|
|
289 |
#chmod -R g+w build/html
|
|
290 |
#rsync $RSYNC_OPTS build/html/* $REMOTE_SYSTEM:$REMOTE_DIR
|
|
291 |
#rsync $RSYNC_OPTS dist/* $REMOTE_SYSTEM:$REMOTE_DIR_DIST
|
|
292 |
#cd ../build
|
|
293 |
#rsync $RSYNC_OPTS index.html *.out $REMOTE_SYSTEM:$REMOTE_DIR/results/
|