← Back to branch summary

~ubuntu-branches/ubuntu/jaunty/calibre/jaunty-backports

« back to all changes in this revision

Viewing changes to upload.py

  • Committer: Bazaar Package Importer
  • Author(s): Martin Pitt
  • Date: 2009-01-20 17:14:02 UTC
  • Revision ID: james.westby@ubuntu.com-20090120171402-8y3znf6nokwqe80k
Tags: upstream-0.4.125+dfsg
Import upstream version 0.4.125+dfsg

Show diffs side-by-side

added added

removed removed

Lines of Context:
upload.py
 
1
#!/usr/bin/python
 
2
import sys, os, shutil, time, tempfile, socket, fcntl, struct, cStringIO, pycurl, re
 
3
sys.path.append('src')
 
4
import subprocess
 
5
from subprocess import check_call as _check_call
 
6
from functools import partial
 
7
 
 
8
def get_ip_address(ifname):
 
9
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
 
10
    return socket.inet_ntoa(fcntl.ioctl(
 
11
        s.fileno(),
 
12
        0x8915,  # SIOCGIFADDR
 
13
        struct.pack('256s', ifname[:15])
 
14
    )[20:24])
 
15
 
 
16
try:
 
17
    HOST=get_ip_address('eth0')
 
18
except:
 
19
    HOST=get_ip_address('wlan0')
 
20
PROJECT=os.path.basename(os.getcwd())
 
21
 
 
22
from calibre import __version__, __appname__
 
23
 
 
24
PREFIX = "/var/www/calibre.kovidgoyal.net"
 
25
DOWNLOADS = PREFIX+"/htdocs/downloads"
 
26
DOCS = PREFIX+"/htdocs/apidocs"
 
27
USER_MANUAL = PREFIX+'/htdocs/user_manual'
 
28
HTML2LRF = "src/calibre/ebooks/lrf/html/demo"
 
29
TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
 
30
MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
 
31
BUILD_SCRIPT ='''\
 
32
#!/bin/bash
 
33
export CALIBRE_BUILDBOT=1
 
34
cd ~/build && \
 
35
rsync -avz --exclude src/calibre/plugins --exclude calibre/src/calibre.egg-info --exclude docs --exclude .bzr --exclude .build --exclude build --exclude dist --exclude "*.pyc" --exclude "*.pyo" rsync://%(host)s/work/%(project)s . && \
 
36
cd %(project)s && \
 
37
%%s && \
 
38
rm -rf build/* dist/* && \
 
39
%%s %%s
 
40
'''%dict(host=HOST, project=PROJECT)
 
41
check_call = partial(_check_call, shell=True)
 
42
#h = Host(hostType=VIX_SERVICEPROVIDER_VMWARE_WORKSTATION)
 
43
 
 
44
 
 
45
def tag_release():
 
46
    print 'Tagging release'
 
47
    check_call('bzr tag '+__version__)
 
48
    check_call('bzr commit --unchanged -m "IGN:Tag release"')
 
49
 
 
50
def installer_name(ext):
 
51
    if ext in ('exe', 'dmg'):
 
52
        return 'dist/%s-%s.%s'%(__appname__, __version__, ext)
 
53
    return 'dist/%s-%s-i686.%s'%(__appname__, __version__, ext)
 
54
 
 
55
def start_vm(vm, ssh_host, build_script, sleep=75):
 
56
    vmware = ('vmware', '-q', '-x', '-n', vm)
 
57
    subprocess.Popen(vmware)
 
58
    t = tempfile.NamedTemporaryFile(suffix='.sh')
 
59
    t.write(build_script)
 
60
    t.flush()
 
61
    print 'Waiting for VM to startup'
 
62
    while subprocess.call('ping -q -c1 '+ssh_host, shell=True, stdout=open('/dev/null', 'w')) != 0:
 
63
        time.sleep(5)
 
64
    time.sleep(20)
 
65
    print 'Trying to SSH into VM'
 
66
    subprocess.check_call(('scp', t.name, ssh_host+':build-'+PROJECT))
 
67
    subprocess.check_call('ssh -t %s bash build-%s'%(ssh_host, PROJECT), shell=True)
 
68
 
 
69
def run_windows_install_jammer(installer):
 
70
    ibp = os.path.abspath('installer/windows')
 
71
    sys.path.insert(0, ibp)
 
72
    import build_installer
 
73
    sys.path.remove(ibp)
 
74
    build_installer.run_install_jammer(installer_name=os.path.basename(installer))
 
75
    if not os.path.exists(installer):
 
76
        raise Exception('Failed to run installjammer')
 
77
 
 
78
def build_windows(shutdown=True):
 
79
    installer = installer_name('exe')
 
80
    vm = '/vmware/Windows XP/Windows XP Professional.vmx'
 
81
    start_vm(vm, 'windows', BUILD_SCRIPT%('python setup.py develop', 'python','installer\\\\windows\\\\freeze.py'))
 
82
    if os.path.exists('build/py2exe'):
 
83
        shutil.rmtree('build/py2exe')
 
84
    subprocess.check_call(('scp', '-rp', 'windows:build/%s/build/py2exe'%PROJECT, 'build'))
 
85
    if not os.path.exists('build/py2exe'):
 
86
        raise Exception('Failed to run py2exe')
 
87
    if shutdown:
 
88
        subprocess.Popen(('ssh', 'windows', 'shutdown', '-s', '-t', '0'))
 
89
    run_windows_install_jammer(installer)
 
90
    return os.path.basename(installer)
 
91
 
 
92
def build_osx(shutdown=True):
 
93
    installer = installer_name('dmg')
 
94
    vm = '/vmware/Mac OSX/Mac OSX.vmx'
 
95
    python = '/Library/Frameworks/Python.framework/Versions/Current/bin/python'
 
96
    start_vm(vm, 'osx', (BUILD_SCRIPT%('sudo %s setup.py develop'%python, python, 'installer/osx/freeze.py')).replace('rm ', 'sudo rm '))
 
97
    subprocess.check_call(('scp', 'osx:build/%s/dist/*.dmg'%PROJECT, 'dist'))
 
98
    if not os.path.exists(installer):
 
99
        raise Exception('Failed to build installer '+installer)
 
100
    if shutdown:
 
101
        subprocess.Popen(('ssh', 'osx', 'sudo', '/sbin/shutdown', '-h', 'now'))
 
102
    return os.path.basename(installer)
 
103
 
 
104
 
 
105
def build_linux(*args, **kwargs):
 
106
    installer = installer_name('tar.bz2')
 
107
    exec open('installer/linux/freeze.py')
 
108
    freeze()
 
109
    if not os.path.exists(installer):
 
110
        raise Exception('Failed to build installer '+installer)
 
111
    return os.path.basename(installer)
 
112
 
 
113
def build_installers():
 
114
    return build_linux(), build_windows(), build_osx()
 
115
 
 
116
def upload_demo():
 
117
    check_call('''html2lrf --title='Demonstration of html2lrf' --author='Kovid Goyal' '''
 
118
               '''--header --output=/tmp/html2lrf.lrf %s/demo.html '''
 
119
               '''--serif-family "/usr/share/fonts/corefonts, Times New Roman" '''
 
120
               '''--mono-family  "/usr/share/fonts/corefonts, Andale Mono" '''
 
121
               ''''''%(HTML2LRF,))
 
122
    check_call('cd src/calibre/ebooks/lrf/html/demo/ && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf')
 
123
    check_call('''scp /tmp/html-demo.zip divok:%s/'''%(DOWNLOADS,))
 
124
    check_call('''txt2lrf -t 'Demonstration of txt2lrf' -a 'Kovid Goyal' '''
 
125
               '''--header -o /tmp/txt2lrf.lrf %s/demo.txt'''%(TXT2LRF,) )
 
126
    check_call('cd src/calibre/ebooks/lrf/txt/demo/ && zip -j /tmp/txt-demo.zip * /tmp/txt2lrf.lrf')
 
127
    check_call('''scp /tmp/txt-demo.zip divok:%s/'''%(DOWNLOADS,))
 
128
 
 
129
def curl_list_dir(url=MOBILEREAD, listonly=1):
 
130
    c = pycurl.Curl()
 
131
    c.setopt(pycurl.URL, url)
 
132
    c.setopt(c.FTP_USE_EPSV, 1)
 
133
    c.setopt(c.NETRC, c.NETRC_REQUIRED)
 
134
    c.setopt(c.FTPLISTONLY, listonly)
 
135
    c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
 
136
    b = cStringIO.StringIO()
 
137
    c.setopt(c.WRITEFUNCTION, b.write)
 
138
    c.perform()
 
139
    c.close()
 
140
    return b.getvalue().split() if listonly else b.getvalue().splitlines()
 
141
 
 
142
def curl_delete_file(path, url=MOBILEREAD):
 
143
    c = pycurl.Curl()
 
144
    c.setopt(pycurl.URL, url)
 
145
    c.setopt(c.FTP_USE_EPSV, 1)
 
146
    c.setopt(c.NETRC, c.NETRC_REQUIRED)
 
147
    print 'Deleting file %s on %s'%(path, url)
 
148
    c.setopt(c.QUOTE, ['dele '+ path])
 
149
    c.perform()
 
150
    c.close()
 
151
 
 
152
 
 
153
def curl_upload_file(stream, url):
 
154
    c = pycurl.Curl()
 
155
    c.setopt(pycurl.URL, url)
 
156
    c.setopt(pycurl.UPLOAD, 1)
 
157
    c.setopt(c.NETRC, c.NETRC_REQUIRED)
 
158
    c.setopt(pycurl.READFUNCTION, stream.read)
 
159
    stream.seek(0, 2)
 
160
    c.setopt(pycurl.INFILESIZE_LARGE, stream.tell())
 
161
    stream.seek(0)
 
162
    c.setopt(c.NOPROGRESS, 0)
 
163
    c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
 
164
    print 'Uploading file %s to url %s' % (getattr(stream, 'name', ''), url)
 
165
    try:
 
166
        c.perform()
 
167
        c.close()
 
168
    except:
 
169
        pass
 
170
    files = curl_list_dir(listonly=0)
 
171
    for line in files:
 
172
        line = line.split()
 
173
        if url.endswith(line[-1]):
 
174
            size = long(line[4])
 
175
            stream.seek(0,2)
 
176
            if size != stream.tell():
 
177
                raise RuntimeError('curl failed to upload %s correctly'%getattr(stream, 'name', ''))
 
178
 
 
179
 
 
180
 
 
181
def upload_installer(name):
 
182
    if not os.path.exists(name):
 
183
        return
 
184
    bname = os.path.basename(name)
 
185
    pat = re.compile(bname.replace(__version__, r'\d+\.\d+\.\d+'))
 
186
    for f in curl_list_dir():
 
187
        if pat.search(f):
 
188
            curl_delete_file('/calibre/'+f)
 
189
    curl_upload_file(open(name, 'rb'), MOBILEREAD+os.path.basename(name))
 
190
 
 
191
def upload_installers():
 
192
    for i in ('dmg', 'exe', 'tar.bz2'):
 
193
        upload_installer(installer_name(i))
 
194
 
 
195
    check_call('''ssh divok echo %s \\> %s/latest_version'''%(__version__, DOWNLOADS))
 
196
 
 
197
 
 
198
def upload_docs():
 
199
    check_call('''epydoc --config epydoc.conf''')
 
200
    check_call('''scp -r docs/html divok:%s/'''%(DOCS,))
 
201
    check_call('''epydoc -v --config epydoc-pdf.conf''')
 
202
    check_call('''scp docs/pdf/api.pdf divok:%s/'''%(DOCS,))
 
203
 
 
204
def upload_user_manual():
 
205
    check_call('python setup.py manual')
 
206
    check_call('scp -r src/calibre/manual/.build/html/* divok:%s'%USER_MANUAL)
 
207
    
 
208
def build_src_tarball():
 
209
    check_call('bzr export dist/calibre-%s.tar.gz'%__version__)
 
210
 
 
211
def upload_src_tarball():
 
212
    check_call('ssh divok rm -f %s/calibre-\*.tar.gz'%DOWNLOADS)
 
213
    check_call('scp dist/calibre-*.tar.gz divok:%s/'%DOWNLOADS)
 
214
 
 
215
def stage_one():
 
216
    check_call('sudo rm -rf build src/calibre/plugins/*', shell=True)
 
217
    os.mkdir('build')
 
218
    shutil.rmtree('docs')
 
219
    os.mkdir('docs')
 
220
    check_call('python setup.py build_ext build', shell=True)
 
221
    check_call('sudo python setup.py develop', shell=True)
 
222
    tag_release()
 
223
    upload_demo()
 
224
 
 
225
def stage_two():
 
226
    subprocess.check_call('rm -rf dist/*', shell=True)
 
227
    build_installers()
 
228
 
 
229
def stage_three():
 
230
    print 'Uploading installers...'
 
231
    upload_installers()
 
232
    print 'Uploading documentation...'
 
233
    #upload_docs()
 
234
    upload_user_manual()
 
235
    print 'Uploading to PyPI...'
 
236
    check_call('rm -f dist/*')
 
237
    check_call('python setup.py register')
 
238
    check_call('sudo rm -rf build src/calibre/plugins/*')
 
239
    os.mkdir('build')
 
240
    check_call('python2.5 setup.py build_ext bdist_egg --exclude-source-files upload')
 
241
    check_call('sudo rm -rf build src/calibre/plugins/*')
 
242
    os.mkdir('build')
 
243
    check_call('python setup.py build_ext bdist_egg --exclude-source-files upload')
 
244
    check_call('python setup.py sdist upload')
 
245
    upload_src_tarball()
 
246
    check_call('''rm -rf dist/* build/*''')
 
247
    check_call('''ssh divok bzr update /var/www/calibre.kovidgoyal.net/calibre/''')
 
248
 
 
249
def betas():
 
250
    subprocess.check_call('rm -f dist/*', shell=True)
 
251
    build_installers()
 
252
    check_call('ssh divok rm -f  /var/www/calibre.kovidgoyal.net/htdocs/downloads/betas/*')
 
253
    check_call('scp dist/* divok:/var/www/calibre.kovidgoyal.net/htdocs/downloads/betas/')
 
254
 
 
255
def main(args=sys.argv):
 
256
    print 'Starting stage one...'
 
257
    stage_one()
 
258
    print 'Starting stage two...'
 
259
    stage_two()
 
260
    print 'Starting stage three...'
 
261
    stage_three()
 
262
    print 'Finished'
 
263
    return 0
 
264
 
 
265
 
 
266
if __name__ == '__main__':
 
267
    sys.exit(main())