1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
|
#!/usr/bin/env python
import imp, os, sys, shutil, subprocess, tempfile, time, string, glob, pprint
from fabric.api import env, run, local, cd
from fabric.operations import put
# TODO create a vm with a completely clean install (maybe just psql).
# TODO the ubuntu vm is used with the 'openerp' user. A different user
# should be used as 'openerp' is also the user running openerp-server.
# TODO Test the running server after the .deb is installed.
# TODO md5 hashes
# TODO the build format of the .deb packages is 1, instead of 3 originally,
# because the file names are not correct w.r.t. the version.
env.hosts = ['root@openerp.com']
env.show = ['debug']
#
# Configuration. Instead of modifying the following variables, you can simply
# create a local-config.py file.
#
# Bazaar repositories to package.
REPOS = os.path.realpath('repos')
# Where the resulting .tar.gz, .exe, and .deb will be generated.
PACKAGES = os.path.realpath('packages') # TODO actually this variable instead of 'packages'
# Where the log files will be written.
RESULTS = os.path.realpath('results')
WINDOWS_XP_SNAPSHOT = '/home/odoo/vm/winxpsp3/winxpsp3_openerp.vmdk'
WINDOWS_XP_IDENTITY = '-i /home/odoo/vm/winxpsp3/winxpsp3_openerp_rsa'
WINDOWS_XP_POSTGRES_PASSWORD = 'openpgpwd'
UBUNTU_SNAPSHOT = '/home/odoo/vm/ubuntu1004/ubuntu-10.04.2-server-i386.img'
UBUNTU_IP = '127.0.0.1'
UBUNTU_PORT = '2225'
UBUNTU_IDENTITY = '-i /home/odoo/vm/ubuntu1004/ubuntu-10.04.2-server-i386_rsa'
# This is used to derive minor and major versions for the Windows packaging.
# This could be part of the PROJECTS dictionary below.
VERSIONS = {
'6.0': {
'major': '6',
'minor': '0',
},
# '6.1-dev': {
# 'major': '6',
# 'minor': '1-dev',
# }
}
# For each branch we want to package, for each project, we give:
# - the path to the setup.py if any,
# - the directory where all the code is (actually used only in
# the server to know the location of the addons directory),
# - the path to the release.py file,
# - the repository URI.
PROJECTS = {
'6.0': {
'addons': (None, 'openerp', 'release.py', 'http://bazaar.launchpad.net/~openerp/openobject-addons/6.0'),
'server': ('setup.py', 'bin', 'release.py', 'http://bazaar.launchpad.net/~openerp/openobject-server/6.0'),
'client': ('setup.py', 'bin', 'release.py', 'http://bazaar.launchpad.net/~openerp/openobject-client/6.0'),
'web': ('setup.py', 'openobject', 'release.py', 'http://bazaar.launchpad.net/~openerp/openobject-client-web/6.0'),
},
# '6.1-dev': {
# 'addons': (None, 'openerp', 'release.py', 'http://bazaar.launchpad.net/~openerp/openobject-addons/trunk'),
# 'server': ('setup.py', 'openerp', 'release.py', 'http://bazaar.launchpad.net/~openerp/openobject-server/trunk'),
# 'client': ('setup.py', 'bin', 'release.py', 'http://bazaar.launchpad.net/~openerp/openobject-client/trunk'),
# 'client': ('setup.py', 'bin', 'release.py', 'http://bazaar.launchpad.net/~openerp-dev/openobject-client/trunk-gtk-glade-revert'),
# 'web': ('setup.py', 'openobject', 'release.py', 'http://bazaar.launchpad.net/~openerp/openobject-client-web/trunk'),
# },
}
INSTALL_MODULES = ['base']
# Replaces configuration variables from user-defined file.
configurables = [ 'REPOS', 'PACKAGES', 'RESULTS',
'WINDOWS_XP_SNAPSHOT', 'WINDOWS_XP_IDENTITY', 'WINDOWS_XP_POSTGRES_PASSWORD',
'UBUNTU_SNAPSHOT', 'UBUNTU_IDENTITY',
'VERSIONS', 'PROJECTS'
]
if os.path.exists('local-conf.py'):
conf = {}
execfile('local-conf.py', conf)
def use(name):
if name in conf:
globals()[name] = conf.get(name)
for name in configurables:
use(name)
def server_setup_exe_name(version,revno,date):
return "openerp-server-setup-%s-%s-r%s.exe" % (version,date,revno)
def server_setup_exe_path(version,revno,date):
return "packages/%s/openerp-server-setup-%s-%s-r%s.exe" % (version,version,date,revno)
def current_project_bare_names(project, version, date=None):
directory = os.path.abspath(os.path.join(REPOS,version,project))
cmd = ['python', PROJECTS[version][project][0], '--fullname']
name = subprocess.Popen(cmd, cwd=directory, stdout=subprocess.PIPE).communicate()[0].strip()
if date is None:
date = time.strftime("%Y%m%d")
revno = subprocess.Popen(['bzr','revno',os.path.join(REPOS,version,project)], stdout=subprocess.PIPE).communicate()[0].strip()
return (name, "%s-%s-r%s" % (name, date, revno))
def current_project_tars(project, version, date=None):
name, fullname = current_project_bare_names(project, version, date=date)
return ("%s.tar.gz" % name, "%s.tar.gz" % fullname)
def current_project_debs(project, version, date=None):
name, fullname = current_project_bare_names(project, version, date=date)
return ("%s.deb" % name, "%s.deb" % fullname)
def create_directory(dirname):
if not os.path.exists(dirname):
os.mkdir(dirname)
create_directory(REPOS)
create_directory(PACKAGES)
create_directory(RESULTS)
for version in PROJECTS:
create_directory(os.path.join(REPOS,version))
create_directory(os.path.join(PACKAGES,version))
def system(l):
print l
if isinstance(l,list):
rc=os.spawnvp(os.P_WAIT, l[0], l)
elif isinstance(l,str):
tmp=['sh','-c',l]
rc=os.spawnvp(os.P_WAIT, tmp[0], tmp)
return rc
def debian_kvm(work):
l=("kvm -nographic -net nic -net user,hostfwd=tcp:127.0.0.1:%s-:22 -snapshot %s"%(UBUNTU_PORT, UBUNTU_SNAPSHOT)).split(" ")
print l
pid=os.spawnvp(os.P_NOWAIT, l[0], l)
print "sleep 5"
time.sleep(5)
work('openerp@%s' % UBUNTU_IP, UBUNTU_PORT) # TODO config variable for the user
print "Killing the debian kvm with pid",pid
os.kill(pid,15)
time.sleep(5)
def windows_kvm(work,forwards=None,date=None):
f = ''
if forwards is not None and len(forwards):
f = ',' + ','.join(['hostfwd=tcp:127.0.0.1:%s-:%s' % x for x in forwards])
l=("kvm -nographic -net nic -net user,hostfwd=tcp:127.0.0.1:2222-:22%s -snapshot %s" % (f,WINDOWS_XP_SNAPSHOT)).split(" ")
print l
pid=os.spawnvp(os.P_NOWAIT, l[0], l)
print "sleep 60"
time.sleep(60)
work('Stef@127.0.0.1','2222', date=date)
print "Killing the windows kvm with pid",pid
os.kill(pid,15)
time.sleep(5)
def show_configurables():
""" Show the name and value of the configuration variables """
for name in configurables:
print string.ljust(name,20), ":", globals()[name]
def versions():
""" Show the possible versions """
for version in PROJECTS:
print version
def update_repos(tag=None):
""" Make bzr pull on each branch and copy the addons in the server """
#system("[ -d .bzr ] || bzr init-repository .")
for version in PROJECTS:
for project in PROJECTS[version]:
dest = os.path.join(REPOS,version,project)
if not os.path.isdir(dest):
system(['bzr','branch',PROJECTS[version][project][3],dest])
else:
system(['bzr','pull','-d',dest,PROJECTS[version][project][3]])
if tag:
system(['bzr', 'pull', '-d', dest, '-r', tag, '--overwrite'])
if PROJECTS[version][project][0] is not None:
cmd = ['python', PROJECTS[version][project][0], '--version']
v = subprocess.Popen(cmd, cwd=dest, stdout=subprocess.PIPE).communicate()[0].strip()
cmd = "rsync -av --delete"
cmd += " --exclude .bzr/"
cmd += " --exclude .bzrignore"
cmd += " --exclude /__init__.py"
cmd += " --exclude /base"
cmd += " --exclude /base_quality_interrogation.py"
cmd += " %s/%s/addons/" % (REPOS,version)
cmd += " %s/%s/server/%s/addons/" % (REPOS,version, PROJECTS[version]['server'][1])
system(cmd)
def sdist(version, project, date=None):
""" Generate the source tarball for specific version/project"""
if date is None:
date = time.strftime("%Y%m%d")
if PROJECTS[version][project][0]: # if there is a setup.py
name, fullname = current_project_tars(project, version, date=date)
packages_dir = os.path.abspath(os.path.join(PACKAGES,version))
if os.path.exists(os.path.join(packages_dir, fullname)):
print "The file", fullname, "already exists, doing nothing."
return
directory = os.path.abspath(os.path.join(REPOS,version,project))
cmd=['python', os.path.join(directory, PROJECTS[version][project][0]), '--quiet', 'sdist', '-d', packages_dir]
print cmd
subprocess.Popen(cmd, cwd=directory).communicate()
print "Renaming", os.path.join(packages_dir, name), "to", os.path.join(packages_dir, fullname)
os.rename(os.path.join(packages_dir, name), os.path.join(packages_dir, fullname))
def sdist_all(date=None):
""" Generate the source tarballs for all versions/projects """
for version in PROJECTS:
for project in PROJECTS[version]:
sdist(version, project, date=date)
# TODO untested
def tarball_test(version):
""" Install a .tar.gz using easy_install in a Ubuntu virtual machine """
tar_name, tar_fullname = current_project_tars('server', version)
def work(ip, port):
system("scp %s -P %s packages/%s/%s %s:" % (UBUNTU_IDENTITY,port,version,tar_fullname,ip))
system('ssh %s -p %s %s "sudo easy_install %s"' % (UBUNTU_IDENTITY,port,ip,tar_fullname))
system('ssh %s -p %s %s "createdb pack"' % (UBUNTU_IDENTITY,port,ip))
# Create the config file. TODO It seems there is a bug when there is no config file.
system('ssh %s -p %s %s "openerp-server -s --stop-after-init"' % (UBUNTU_IDENTITY,port,ip))
install_modules = ','.join(list(set(INSTALL_MODULES)))
# TODO --logfile doesn't work for 6.0, logs are on the console.
system('ssh %s -p %s %s "openerp-server -d pack -i %s --stop-after-init --log-level=test --logfile=openerp-server.log"' % (UBUNTU_IDENTITY,port,ip,install_modules))
system("scp %s -P %s %s:openerp-server.log %s/%s-easy-install.log"%(UBUNTU_IDENTITY,port,ip,RESULTS,tar_fullname[:-7]))
debian_kvm(work)
def debian_build(version=None, date=None):
""" Generate a .deb for a specific version """
if version is None or version not in PROJECTS:
print "debian_build: version should be one of", PROJECTS.keys()
sys.exit()
for project in ['server', 'client']:
name, fullname = current_project_tars(project, version, date=date)
deb_name, deb_fullname = current_project_debs(project, version, date=date)
if os.path.exists('%s/%s/%s' % (PACKAGES, version, deb_fullname)):
print "The file", deb_fullname, "already exists, doing nothing."
continue
print "Creating .deb for", fullname
if not os.path.exists('packages/%s/%s' % (version, fullname)):
print "packages/%s/%s doesn't exist, exiting," % (version, fullname)
print "you can generate it by running fab sdist:%s,%s" % (version, project)
sys.exit()
# cp tar.gz to debianization, untar it, cd it and run dpkg-buildpackage, retreive resulting .deb.
system('rm -rf debianization')
system('mkdir debianization')
system('cp -a packages/%s/%s debianization/' % (version, fullname))
if project == 'client':
barename, barename_fullname = current_project_bare_names(project, version, date=date)
barename_app, barename_version = barename.rsplit('-', 1)
system('cp -a packages/%s/%s debianization/%s' % (version, fullname, '%s_%s.orig.tar.gz' % (barename_app, barename_version,)))
system('tar xf debianization/%s --directory debianization' % fullname)
subprocess.Popen(['dpkg-buildpackage', '-rfakeroot'], cwd='debianization/%s'%name[:-7]).communicate()
system('mv debianization/openerp-%s_*_all.deb packages/%s/%s' % (project,version,deb_fullname)) # TODO correct version
#system('rm -rf debianization')
def debian_build_all(date=None):
""" Generate a .deb for each version """
for version in PROJECTS:
debian_build(version, date=date)
# TODO test the client .deb
def debian_test(version, date=None):
""" Install a .deb in a Ubuntu virtual machine """
deb_name, deb_fullname = current_project_debs('server', version, date=date)
def work(ip, port):
system("scp %s -P %s packages/%s/%s %s:" % (UBUNTU_IDENTITY,port,version,deb_fullname,ip))
system('ssh %s -p %s %s "sudo dpkg -i %s"' % (UBUNTU_IDENTITY,port,ip,deb_fullname))
system('ssh %s -p %s %s "sudo apt-get --yes -f install"' % (UBUNTU_IDENTITY,port,ip))
system('ssh %s -p %s %s "createdb pack"' % (UBUNTU_IDENTITY,port,ip))
# Create the config file. TODO It seems there is a bug when there is no config file.
system('ssh %s -p %s %s "openerp-server -s --stop-after-init"' % (UBUNTU_IDENTITY,port,ip))
install_modules = ','.join(list(set(INSTALL_MODULES)))
# TODO --logfile doesn't work for 6.0, logs are on the console.
system('ssh %s -p %s %s "openerp-server -d pack -i %s --stop-after-init --log-level=test --logfile=openerp-server.log"' % (UBUNTU_IDENTITY,port,ip,install_modules))
system("scp %s -P %s %s:openerp-server.log %s/%s.log"%(UBUNTU_IDENTITY,port,ip,RESULTS,deb_fullname[:-4]))
debian_kvm(work)
def debian_test_all(date=None):
""" Install the .deb for each version in a Ubuntu virtual machine """
for version in PROJECTS:
debian_test(version, date=date)
# TODO untested
def rpm_build(version):
""" Generate a .rpm for a specific version """
if PROJECTS[version][project][0]: # if there is a setup.py
#name, fullname = current_project_rpms(project, version)
packages_dir = os.path.abspath(os.path.join(PACKAGES,version))
#if os.path.exists(os.path.join(packages_dir, fullname)):
# print "The file", fullname, "already exists, doing nothing."
# return
directory = os.path.abspath(os.path.join(REPOS,version,project))
# TODO pass it some .spec related flags, check the version (there is a test in 6.0's setup.py)
cmd=['python', os.path.join(directory, PROJECTS[version][project][0]), '--quiet', 'bdist_rpm', '-d', packages_dir]
print cmd
subprocess.Popen(cmd, cwd=directory).communicate()
#print "Renaming", os.path.join(packages_dir, name), "to", os.path.join(packages_dir, fullname)
#os.rename(os.path.join(packages_dir, name), os.path.join(packages_dir, fullname))
def windows_build(version=None, date=None):
""" Generate Windows installers using a Windows virtual machine """
if version is None or version not in PROJECTS:
print "windows_build: version should be one of", PROJECTS.keys()
sys.exit()
def work(ip, port, date=None):
# The revno is the one from the server
revno = subprocess.Popen(['bzr','revno',os.path.join(REPOS,version,'server')], stdout=subprocess.PIPE).communicate()[0].strip()
if date is None:
date = time.strftime("%Y%m%d")
system('rsync --timeout=60 -av -e "ssh %s -p %s" --delete --exclude .bzr/ --exclude .bzrignore --exclude /packages/ %s/%s/ %s:openerp-packaging/'%(WINDOWS_XP_IDENTITY,port,REPOS,version,ip))
f = open('windows-installer/Makefile.version','w')
f.write("MAJOR_VERSION=%s\n" % VERSIONS[version]['major'])
f.write("MINOR_VERSION=%s\n" % VERSIONS[version]['minor'])
f.write("REVISION_VERSION=%s\n" % revno)
f.write("BUILD_VERSION=%s\n" % date)
f.close()
system('rsync --timeout=60 -av -e "ssh %s -p %s" --delete windows-installer %s:openerp-packaging/'%(WINDOWS_XP_IDENTITY,port,ip))
system('ssh %s -p %s %s "cd openerp-packaging/windows-installer;time make allinone;"'%(WINDOWS_XP_IDENTITY,port,ip))
system('rsync --timeout=60 -av -e "ssh %s -p %s" %s:openerp-packaging/windows-installer/files/ %s/%s/ '%(WINDOWS_XP_IDENTITY,port,ip,PACKAGES,version))
print "all-in-one copied in %s/%s" % (PACKAGES, version)
windows_kvm(work, date=date)
def windows_build_all(date=None):
""" Generate Windows installers using a Windows virtual machine for each version """
for version in PROJECTS:
windows_build(version, date=date)
def windows_test_base(version, date=None):
""" Install a Windows .exe in a Windows virtual machine """
# The revno is the one from the server
revno = subprocess.Popen(['bzr','revno',os.path.join(REPOS,version,'server')], stdout=subprocess.PIPE).communicate()[0].strip()
if date is None:
date = time.strftime("%Y%m%d")
setuppath = "packages/%s/openerp-server-setup-%s-%s-r%s.exe" % (version, version, date, revno)
windows_test_setup_base(setuppath)
def windows_test_base_all(date=None):
""" Install Windows .exe's in a Windows virtual machine """
for version in PROJECTS:
windows_test_base(version, date=date)
def windows_test_setup_base(setuppath, date=None):
setupdir, setupexe = os.path.split(setuppath)
setup_no_ext, ext = os.path.splitext(setupexe)
# Could use more the fabric stuffs instead.
def work(ip, port, date=None):
# Change postgres config so it accepts our connections.
system("scp %s -P %s pg_hba.conf '%s:/cygdrive/c/Program\ Files/OpenERP\ 6.0/PostgreSQL/data/pg_hba.conf'"%(WINDOWS_XP_IDENTITY,port,ip))
system("scp %s -P %s postgresql.conf '%s:/cygdrive/c/Program\ Files/OpenERP\ 6.0/PostgreSQL/data/postgresql.conf'"%(WINDOWS_XP_IDENTITY,port,ip))
system("""ssh %s -p %s %s 'net stop "PostgreSQL For OpenERP"'"""%(WINDOWS_XP_IDENTITY,port,ip))
system("""ssh %s -p %s %s 'net start "PostgreSQL For OpenERP"'"""%(WINDOWS_XP_IDENTITY,port,ip))
system("scp %s -P %s %s %s:"%(WINDOWS_XP_IDENTITY,port,setuppath,ip))
system('ssh %s -p %s %s "./%s /S /D=\'c:\\cygwin\\home\\Stef\\server\'"'%(WINDOWS_XP_IDENTITY,port,ip,setupexe))
subprocess.call('createdb -e -h 127.0.0.1 -p 5433 -U openpg pack'.split(' '), env={'PGPASSWORD':WINDOWS_XP_POSTGRES_PASSWORD})
system('ssh %s -p %s %s "./server/openerp-server.exe -d pack -i base,report_webkit --stop-after-init --log-level=test"'%(WINDOWS_XP_IDENTITY,port,ip))
system("scp %s -P %s %s:server/openerp-server.log %s/%s.log"%(WINDOWS_XP_IDENTITY,port,ip,RESULTS,setup_no_ext))
windows_kvm(work, [(5433, 5432)], date=date)
def daily():
update_repos()
sdist_all()
debian_build_all()
windows_build_all()
debian_test_all()
windows_test_base_all()
def release(tag='6.0.0', date=None):
sdist_all(date)
debian_build_all(date)
# update_repos(tag=tag)
# sdist_all(date)
# debian_build_all(date)
# windows_build_all(date)
# debian_test_all(date)
# windows_test_base_all(date)
def summary():
packages = {}
for version in PROJECTS:
packages[version] = {}
dest = os.path.join(REPOS,version,'server')
cmd = ['python', PROJECTS[version]['server'][0], '--version']
v = subprocess.Popen(cmd, cwd=dest, stdout=subprocess.PIPE).communicate()[0].strip()
for project in PROJECTS[version]:
packages[version][project] = glob.glob(os.path.join(PACKAGES, version, 'openerp-%s-%s-*.tar.gz' % (project, v)))
def f(t):
deb = t[:-7] + '.deb'
log = string.replace(t[:-7] + '.log', os.path.join(PACKAGES,version), RESULTS)
exe = string.replace(t[:-7] + '.exe', 'openerp-' + project, 'openerp-' + project + '-setup')
if not os.path.exists(log):
log = None
if not os.path.exists(deb):
deb = None
if not os.path.exists(exe):
exe = None
return (t, deb, exe, log) # TODO check if the logs are ok
packages[version][project] = map(f, packages[version][project])
packages[version]['allinone'] = 'TODO'
pprint.pprint(packages)
return packages
|