1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
|
#!/usr/bin/env python
# this is part of langpack-o-matic, by Martin Pitt <martin.pitt@canonical.com>
#
# (C) 2005 Canonical Ltd.
#
# Update all language-pack-%PKGNAME% source packages (they must be in the state of
# the previous upload). If a locale does not yet have base and support
# packages, these are created.
#
# After this script finishes successfully, there will be a file
# 'updated-packages' which contains the source package directory names of all
# packages that were updated.
#
# Usage: import <archive> <distro-release>
#
# (C) 2005 Canonical Ltd.
#
# Author: Martin Pitt <martin.pitt@canonical.com>
import os, os.path, sys, subprocess, filecmp, tempfile, shutil
# change working directory to the directory of this script
os.chdir(os.path.dirname(sys.argv[0]))
# import our own libraries
sys.path.append('lib')
import pkg_classify, localeinfo, macros, makepkg
# global variables
locinfo = localeinfo.SupportedLocales()
macros_map = {} # class -> locale -> LangpackMacros
def get_current_macros(cls, locale):
'''Return a LangpackMacros object for the given class and locale.
The LangpackMacros objects are cached for performace reasons.'''
loc_map = macros_map.setdefault(cls, {})
if not loc_map.has_key(locale):
loc_map[locale] = macros.LangpackMacros(locale, cls, release, version)
return loc_map[locale]
def package_updated(pkg):
'''Check if the given package has already been updated (i. e. it appears in
updated-packages).'''
if not os.path.isfile('updated-packages'):
return False
for p in open('updated-packages'):
if p.strip() == pkg.strip():
return True
return False
def write_po(locale, domain, pkgdir, contents):
'''Write file contents to pkgdir/data/locale/LC_MESSAGES/domain.po.'''
print "Copying %s/%s into package %s" % (locale, domain, pkgdir)
try:
os.makedirs(pkgdir + '/data/' + locale + '/LC_MESSAGES')
except:
pass
print >> open('%s/data/%s/LC_MESSAGES/%s.po' % (pkgdir, locale, domain), 'w'), contents
def read_po(locale, domain, pkgdir):
'''Read file contents from pkgdir/data/locale/domain.po.
Return None if the file does not exist. Strips off surrounding white
space.'''
try:
return open('%s/data/%s/LC_MESSAGES/%s.po' % (pkgdir, locale, domain)).read().strip()
except:
return None
def normalize_po(contents):
'''Return PO contents in a canonical format suitable for comparison.'''
if contents == None:
return None
msgfmt = subprocess.Popen(('/usr/bin/msgfmt', '-o', '-', '-'), 0, None,
subprocess.PIPE, subprocess.PIPE, subprocess.PIPE)
try:
(out, err) = msgfmt.communicate(contents)
except OSError:
print "ERROR: msgfmt failed with OSError, not normalizing"
return contents
if msgfmt.returncode:
print "ERROR: msgfmt failed with code %i, not normalizing" % msgfmt.returncode
return contents
msgunfmt = subprocess.Popen(('/usr/bin/msgunfmt', '-'), 0, None,
subprocess.PIPE, subprocess.PIPE)
if msgunfmt.returncode:
raise Exception, 'msgunfmt returned with exit code ' + str(msgunfmt.returncode)
(out, err) = msgunfmt.communicate(out)
return out
def install_po(locale, domain, contents):
'''Install translation file.
locale: Target locale
domain: Translation domain
contents: The actual translation data (PO file contents)
'''
try:
cls = classifier.classify_domain(domain)
except KeyError:
print >> sys.stderr, "WARNING: unknown translation domain:", domain
return
macr = get_current_macros(cls, locale)
support_pkg = macr.subst_string(basepath+'/sources-support/language-support-%PKGNAME%')
update_pkg = macr.subst_string(basepath+'/sources-update/language-pack-%PKGNAME%')
base_pkg = macr.subst_string(basepath+'/sources-base/language-pack-%PKGNAME%-base')
# create support package; disable per-class support packages
if cls == '' and not os.path.isdir(support_pkg):
# do not build new packages for stable release updates
if release.find('-') > 0:
print >> sys.stderr, "WARNING: not building new package %s for stable release update" % support_pkg
return
makepkg.make_pkg('skel-support', support_pkg, macr)
# if base package does not exist, create it
if not os.path.isdir(base_pkg):
# do not build new packages for stable release updates
if release.find('-') > 0:
print >> sys.stderr, "WARNING: not building new package %s for stable release update" % support_pkg
return
# determine name of tarball with extra files
extra_tar = 'extra-files/%s-%s.tar' % (cls, macr['LCODE'])
if not os.path.isfile(extra_tar):
extra_tar = extra_tar + '.gz'
if not os.path.isfile(extra_tar):
extra_tar = None
# add locales to extra tarball of base package
locale_tar = None
if cls == '' and not release.startswith('breezy'):
print 'Creating locale tarball'
locale_tar = locinfo.create_locale_tar(macr['LCODE'])
if extra_tar != None:
raise Exception, 'Not yet implemented: tarball merging (locale+extra.tar)'
else:
extra_tar = locale_tar
makepkg.make_pkg('skel-base', base_pkg, macr, extra_tar)
if locale_tar != None:
os.unlink(locale_tar)
# sanity check: we just created -base, so the update package should not
# be present
if os.path.isdir(update_pkg):
raise Exception, 'Inconsistency: just created fresh base, but update package already exists'
# Create an empty update package
makepkg.make_pkg('skel-update', update_pkg, macr)
# workaround for Rosetta exported files without translations
ncontents = normalize_po(contents)
if ncontents == None or ncontents.strip() == '':
return
# prefer to change the base package if we already changed it
if package_updated(base_pkg):
write_po(locale, domain, base_pkg, contents)
else:
if ncontents != normalize_po(read_po(locale, domain, base_pkg)) and \
ncontents != normalize_po(read_po(locale, domain, update_pkg)):
if not package_updated(update_pkg):
# if we have an extra tarball, do not install it if the same
# version is already in the base package
# XXX: deactivated for now, it costs lots of time, does not
# respect locales, and is useless ATM
#if extra_tar:
# base_extra_tar = os.path.join(base_pkg, 'data',
# os.path.basename(extra_tar))
# if os.path.isfile(base_extra_tar) and \
# filecmp(extra_tar, base_extra_tar):
# extra_tar = None
makepkg.make_pkg('skel-update', update_pkg, macr)
write_po(locale, domain, update_pkg, contents)
#
# main
#
if len(sys.argv) != 3:
print >> sys.stderr, 'Usage:', sys.argv[0], '<archive> <distro release>'
sys.exit(-1)
archive_fname = sys.argv[1]
release = sys.argv[2]
# verify release and create config file
basepath = '../' + release
if not os.path.isdir(basepath):
raise OSError, 'Invalid distribution release, %s does not exist' % basepath
contentdirbase = tempfile.mkdtemp()
try:
# extract tarball to a temporary dir
result = os.spawnlp(os.P_WAIT, 'tar', 'tar', '-C', contentdirbase, '-xzf',
os.path.abspath(archive_fname))
if result != 0:
raise Exception, "Error executing tar"
toplevel_dirs = os.listdir(contentdirbase)
if len(toplevel_dirs) != 1:
raise Exception, 'Archive does not contain a single top level directory'
content_dir = os.path.join(contentdirbase, toplevel_dirs[0])
# generate time stamp config file
timestamp_file = os.path.join(content_dir, 'timestamp.txt')
if not os.path.exists(timestamp_file):
raise Exception, 'Archive does not contain a timestamp'
version = open(timestamp_file).read().strip()
os.unlink(timestamp_file)
# initialize domain map
map_file = os.path.join(content_dir, 'mapping.txt')
if not os.path.exists(map_file):
raise Exception, 'Archive does not contain a domain map file (mapping.txt)'
map = open(map_file).read()
classifier = pkg_classify.PackageClassificator(release, map)
os.unlink(map_file)
# Process every .po file
lastlocale = ''
valid_locale = {}
for root, dirs, files in os.walk(content_dir):
for f in files:
if f.endswith('.pot'):
continue
file = os.path.join(root, f)
comp = file.split(os.sep)[-3:]
# Verify that we have locale/LC_MESSAGES/domain.po
if len(comp) < 3 or comp[2] == '':
continue
if comp[1] != 'LC_MESSAGES':
raise IOError, 'Invalid file: %s' % file
(domain, ext) = os.path.splitext(comp[2])
if ext != '.po':
raise IOError, 'Unknown file type: %s' % file
# Ignore ISO files
if domain.startswith('iso_'):
continue
# Verify that we have a known locale
locale = comp[0]
if locale != lastlocale:
print '--------------------------------'
print 'Processing locale %s...' % locale
lastlocale = locale
# check language
if not locinfo.known_language((locale.split('_')[0]).split('@')[0]):
print 'Skipping unknown language in ', locale
continue
# check country, if present
noat = locale.split('@')[0]
if noat.find('_') >= 0:
if not locinfo.known_country(noat.split('_')[1]):
print 'Skipping unknown country in', locale
continue
# Everything is fine, install it
install_po(locale, domain, open(file).read())
finally:
shutil.rmtree(contentdirbase, True)
|