1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
|
#!/usr/bin/env python
# this is part of langpack-o-matic, by Martin Pitt <martin.pitt@canonical.com>
#
# (C) 2005, 2011 Canonical Ltd.
#
# Update all language-pack-%PKGNAME% source packages (they must be in the state of
# the previous upload). If a locale does not yet have a -base package, it gets
# created.
#
# After this script finishes successfully, there will be a file
# 'updated-packages' which contains the source package directory names of all
# packages that were updated.
#
# Usage: import <archive> <distro-release> <target dir>
import os, os.path, sys, subprocess, filecmp, tempfile, shutil
import optparse, logging
DEFAULT_MIRROR = 'http://archive.ubuntu.com/ubuntu'
# change working directory to the directory of this script
os.chdir(os.path.dirname(sys.argv[0]))
# import our own libraries
sys.path.append('lib')
import pkg_classify, localeinfo, macros, makepkg, static_translations, xpi
# global variables
locinfo = localeinfo.SupportedLocales()
macros_map = {} # class -> locale -> LangpackMacros
static_tar_dir = None
def parse_argv():
'''Parse command line options.
Return (options, args) pair.
'''
optparser = optparse.OptionParser('%prog <translation tarball> <distro release> <target dir>')
optparser.add_option('--mirror', default=DEFAULT_MIRROR,
metavar='URL', help='Archive mirror URL')
optparser.add_option('-s', '--no-static', action='store_false',
dest='static', default=True,
help='Disable inclusion of static translations (GNOME help)')
optparser.add_option('--no-classes', action='store_false',
dest='classes', default=True,
help='Disable splitting by classes (GNOME/KDE/common)')
optparser.add_option('-v', '--verbose', action='store_true', default=False,
help='Verbose logging')
(opts, args) = optparser.parse_args()
if len(args) != 3:
optparser.error('incorrect number of arguments; use --help for a short help')
if opts.verbose:
logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s')
else:
logging.basicConfig(level=logging.WARNING, format='%(levelname)s: %(message)s')
return opts, args
def get_current_macros(cls, locale, version):
'''Return a LangpackMacros object for the given class and locale.
The LangpackMacros objects are cached for performace reasons.'''
loc_map = macros_map.setdefault(cls, {})
if locale not in loc_map:
# get available packages from cache
release_nopocket = release.split('-')[0]
if release_nopocket not in get_current_macros.release_packages:
get_current_macros.release_packages[release_nopocket] = pkg_classify.available_packages(options.mirror, release_nopocket)
loc_map[locale] = macros.LangpackMacros(locale, cls, release, version,
get_current_macros.release_packages[release_nopocket])
return loc_map[locale]
get_current_macros.release_packages = {}
def package_updated(pkg):
'''Check if the given package has already been updated (i. e. it appears in
updated-packages).'''
if not os.path.isfile('updated-packages'):
return False
for p in open('updated-packages'):
if p.strip() == pkg.strip():
return True
return False
def write_po(locale, domain, pkgdir, contents):
'''Write file contents to pkgdir/data/locale/LC_MESSAGES/domain.po.'''
logging.debug('Copying %s/%s into package %s', locale, domain, pkgdir)
try:
os.makedirs(pkgdir + '/data/' + locale + '/LC_MESSAGES')
except:
pass
dest = '%s/data/%s/LC_MESSAGES/%s.po' % (pkgdir, locale, domain)
if locale.startswith('en_'):
# many languages legitimagely have identical strings, such as fr
# or pt_BR using the same string as English, but pt does not. So only
# used msgequal for English.
msgequal = subprocess.Popen(['bin/msgequal', '-', dest],
stdin=subprocess.PIPE)
msgequal.communicate(contents)
assert msgequal.returncode == 0
else:
f = open(dest, 'w')
f.write(contents)
f.close()
def read_po(locale, domain, pkgdir):
'''Read file contents from pkgdir/data/locale/domain.po.
Return None if the file does not exist. Strips off surrounding white
space.'''
try:
return open('%s/data/%s/LC_MESSAGES/%s.po' % (pkgdir, locale, domain)).read().strip()
except:
return None
def normalize_po(contents):
'''Return PO contents in a canonical format suitable for comparison.'''
if contents == None:
return None
msgfmt = subprocess.Popen(('/usr/bin/msgfmt', '-o', '-', '-'), 0, None,
subprocess.PIPE, subprocess.PIPE, subprocess.PIPE)
try:
(out, err) = msgfmt.communicate(contents)
except OSError as e:
logging.warning('msgfmt failed with OSError: %s, not normalizing', str(e))
return contents
if msgfmt.returncode:
logging.warning('msgfmt failed with code %i, not normalizing', msgfmt.returncode)
return contents
msgunfmt = subprocess.Popen(('/usr/bin/msgunfmt', '-'),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if msgunfmt.returncode:
raise Exception('msgunfmt returned with exit code ' + str(msgunfmt.returncode))
(out, err) = msgunfmt.communicate(out)
# remove X-Launchpad-Export-Date:
export_date = out.find('\n"X-Launchpad-Export-Date: ')
if export_date >= 0:
out = out[:export_date] + out[out.index('\n', export_date+1):]
return out
def install_po(locale, domain, contents, data_version, include_static):
'''Install translation file.
locale: Target locale
domain: Translation domain
data_version: version number of the PO export
contents: The actual translation data (PO file contents)
There is a magic domain None which just causes the class-less base package
to be created, but nothing installed into it. This is important since the
common package must always exist. If e. g. just -gnome exist, its
dependencies are unsatisfyable, and it's missing the extra tarball and
locales. (LP #422760, #335307)
'''
try:
if domain and classifier:
cls = classifier.classify_domain(domain)
else:
cls = ''
except KeyError:
logging.warning('unknown translation domain: %s', domain)
return
macr = get_current_macros(cls, locale, data_version)
update_pkg = macr.subst_string(target_dir+'/sources-update/language-pack-%PKGNAME%')
base_pkg = macr.subst_string(target_dir+'/sources-base/language-pack-%PKGNAME%-base')
# if base package does not exist, create it
if not os.path.isdir(base_pkg):
# determine name of tarball with extra files
extra_tar = 'extra-files/%s-%s.tar' % (cls, macr['PKGCODE'])
if not os.path.isfile(extra_tar):
extra_tar = extra_tar + '.gz'
if not os.path.isfile(extra_tar):
extra_tar = None
# add locales to extra tarball of base package
locale_tar = None
if cls == '':
logging.debug('Creating locale tarball')
locale_tar = locinfo.create_locale_tar(macr['PKGCODE'])
if extra_tar != None:
raise Exception('Not yet implemented: tarball merging (locale+extra.tar)')
else:
extra_tar = locale_tar
makepkg.make_pkg('skel-base', base_pkg, macr, extra_tar)
if locale_tar != None:
os.unlink(locale_tar)
# add static translations
if include_static:
if not os.path.isdir(static_tar_dir):
logging.debug('Downloading and preparing static translations...')
# lazily download static tars, so that we don't have to do it when
# building updates only
os.mkdir(static_tar_dir)
tarballs = static_translations.get_static_translation_tarballs(release)
static_translations.create_static_tarballs(tarballs, static_tar_dir)
static_tar = os.path.join(static_tar_dir, macr['PKGNAME'] + '.tar')
if os.path.exists(static_tar):
logging.debug('Adding static tarball %s', static_tar)
shutil.move(static_tar, os.path.join(base_pkg, 'data', 'static.tar'))
# add XPIs and Firefox search plugins
if cls == '':
xpi.install_xpis(release_version, locale, base_pkg)
# sanity check: we just created -base, so the update package should not
# be present
if os.path.isdir(update_pkg):
raise Exception('Inconsistency: just created fresh base, but update package already exists')
# determine %BASEVERDEP% macro (needs to be postponed until here, since we
# know where the -base package is, and which version it has, and it is
# not created yet)
if 'BASEVERDEP' not in macr:
macr['BASEVERDEP'] = ' (>= %s)' % makepkg.get_pkg_version(base_pkg)
# Create an empty update package
makepkg.make_pkg('skel-update', update_pkg, macr)
else:
# determine %BASEVERDEP% macro (needs to be postponed until here, since we
# know where the -base package is, and which version it has, and is not
# updated yet)
if 'BASEVERDEP' not in macr:
macr['BASEVERDEP'] = ' (>= %s)' % makepkg.get_pkg_version(base_pkg)
# workaround for Rosetta exported files without translations
ncontents = normalize_po(contents)
if ncontents == None or ncontents.strip() == '':
return
if domain is None:
return
# ensure that we always have the common package
if macr['CLASS']:
common_base_pkg = macr.subst_string(target_dir+'/sources-base/language-pack-%PKGCODE%-base')
if not os.path.isdir(common_base_pkg):
logging.debug('Creating common package for %s', macr['PKGCODE'])
install_po(locale, None, None, data_version, options.static)
# prefer to change the base package if we already changed it
if package_updated(base_pkg):
write_po(locale, domain, base_pkg, contents)
else:
if ncontents != normalize_po(read_po(locale, domain, base_pkg)) and \
ncontents != normalize_po(read_po(locale, domain, update_pkg)):
if not package_updated(update_pkg):
# if we have an extra tarball, do not install it if the same
# version is already in the base package
# XXX: deactivated for now, it costs lots of time, does not
# respect locales, and is useless ATM
#if extra_tar:
# base_extra_tar = os.path.join(base_pkg, 'data',
# os.path.basename(extra_tar))
# if os.path.isfile(base_extra_tar) and \
# filecmp(extra_tar, base_extra_tar):
# extra_tar = None
makepkg.make_pkg('skel-update', update_pkg, macr)
write_po(locale, domain, update_pkg, contents)
#
# main
#
options, args = parse_argv()
(archive_fname, release, target_dir) = args
if not os.path.isdir(target_dir):
sys.stderr.write('Target directory does not exist\n')
sys.exit(1)
release_version = get_current_macros('', 'en_GB.UTF-8', 'invalid').subst_string('%RELEASEVERSION%')
assert release_version, 'no release version for ' + release
# unpack translation tarball
contentdirbase = tempfile.mkdtemp()
try:
# extract tarball to a temporary dir
if archive_fname[-4:] == '.tar' :
result = os.spawnlp(os.P_WAIT, 'tar', 'tar', '-C', contentdirbase, '-xf',
os.path.abspath(archive_fname))
elif archive_fname[-7:] == '.tar.gz' :
result = os.spawnlp(os.P_WAIT, 'tar', 'tar', '-C', contentdirbase, '-xzf' ,
os.path.abspath(archive_fname))
elif archive_fname[-8:] == '.tar.bz2' :
result = os.spawnlp(os.P_WAIT, 'tar', 'tar', '-C', contentdirbase, '--bzip2 -xf',
os.path.abspath(archive_fname))
elif archive_fname[-9:] == '.tar.lzma' :
result = os.spawnlp(os.P_WAIT, 'tar', 'tar', '-C', contentdirbase, '--lzma -xf',
os.path.abspath(archive_fname))
else:
sys.stderr.write('Unknown tar format')
result = 1
if result != 0:
sys.stderr.write('Error executing tar, aborting')
sys.exit(1)
toplevel_dirs = os.listdir(contentdirbase)
if len(toplevel_dirs) != 1:
raise Exception('Archive does not contain a single top level directory')
content_dir = os.path.join(contentdirbase, toplevel_dirs[0])
static_tar_dir = os.path.join(contentdirbase, 'static-tars')
# generate time stamp config file
timestamp_file = os.path.join(content_dir, 'timestamp.txt')
if not os.path.exists(timestamp_file):
raise Exception('Archive does not contain a timestamp')
data_version = open(timestamp_file).read().strip()
os.unlink(timestamp_file)
# initialize domain map
map_file = os.path.join(content_dir, 'mapping.txt')
if not os.path.exists(map_file):
raise Exception('Archive does not contain a domain map file (mapping.txt)')
if options.classes:
classifier = pkg_classify.PackageClassificator(release, map_file,
options.mirror)
else:
classifier = None
os.unlink(map_file)
# Process every .po file
lastlocale = ''
valid_locale = {}
for root, dirs, files in os.walk(content_dir):
for f in files:
if f.endswith('.pot'):
continue
file = os.path.join(root, f)
comp = file.split(os.sep)[-3:]
# xpi PO files need special treatment; po2xpi is currently broken,
# so ignore these for now
if comp[0] == 'xpi':
continue
# Verify that we have locale/LC_MESSAGES/domain.po
if len(comp) < 3 or comp[2] == '':
continue
if comp[1] != 'LC_MESSAGES':
raise IOError('Invalid file: %s' % file)
(domain, ext) = os.path.splitext(comp[2])
if ext != '.po':
raise IOError('Unknown file type: %s' % file)
# Ignore ISO files
if domain.startswith('iso_'):
continue
# Verify that we have a known locale
locale = comp[0]
if locale != lastlocale:
logging.debug('--------------------------------')
logging.debug('Processing locale %s...', locale)
lastlocale = locale
# check language
lang = (locale.split('_')[0]).split('@')[0]
if not (locinfo.known_language(lang) and locinfo.language_locales(lang)):
logging.warning('Skipping unknown language: %s', locale)
continue
# check country, if present
noat = locale.split('@')[0]
if noat.find('_') >= 0:
(lang, country) = noat.split('_')
#XXX: hack: ignore invalid/obsolete per-country locales from LP
if lang not in ('zh', 'en') and noat != 'pt_BR':
logging.warning('Skipping obsolete locale: %s', locale)
continue
if not locinfo.known_country(country):
logging.warning('Skipping unknown country: %s', locale)
continue
if noat == 'zh':
# split into zh_* now
logging.warning('Skipping obsolete locale: %s', locale)
continue
# Everything is fine, install it
install_po(locale, domain, open(file).read(), data_version, options.static)
finally:
shutil.rmtree(contentdirbase, True)
|