1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
|
"""PO files contain the translated text in some language.
Launchpad exports are read in to the data structure that was created by reading
GRDs and their associated XTB files, and then we read Launchpad's exported PO
files to set canonical translations.
We deal with GRIT GRD/XTB data internally, and at the PO/POT edge, we translate
from and to what gettext and LP expect.
On import, we also map wonky GRD language codes to true modern languge to find
the files that LP uses. Also on import, we map gettext '%{FOO}' markers back
to GRD '<ph name="FOO" />' markers before store them.
Later, we write results to be imported back into Launchpad, with translation
hints that include 1) the original template text, 2) GRD description, and 3)
translations from XTB that are not the same as LP that LP is overriding, 4)
filename of the XTB source, 5) a formatted resource ID from the XTB entry.
On export, we also change wonky GRD language codes to true modern languge
codes.
white-space
# translator-comments
#. extracted-comments
#: reference...
#, flag...
#| msgid previous-untranslated-string
msgid untranslated-string
msgstr translated-string
"""
import logging
import re
import os
from hashlib import md5
from .gettext import gettext_file_sorting_function, grd_msg_write_in_po
from .utils import evaluate_grd_condition
# gen$ wget -nv -O - 'https://translations.launchpad.net/chromium-browser/translations' |grep 'href="/chromium-browser/translations/+lang/' |cut -d/ -f5 |cut -d\" -f1
known_lp_languages = set("af,am,ar,hy,ast,eu,be,bn,bs,pt_BR,br,bg,my,ca,ca@valencia,zh_HK,zh_CN,zh_TW,kw,hr,cs,da,nl,en_AU,en_GB,eo,et,fo,fil,fi,fr,fy,gd,gl,ka,de,el,gu,he,hi,hu,is,id,ia,ga,it,ja,kbd,kn,kk,km,rw,ky,ko,ku,la,lv,lt,jbo,mk,ms,ml,mr,mhr,nb,nn,or,om,os,fa,pl,pt,pa,ro,ru,sr,shn,szl,si,sk,sl,es,sw,sv,ta,ta_LK,te,th,tr,uk,ur,ug,uz,vi,cy,es_419".split(","))
translator_logger = logging.getLogger("to launchpad")
def decode_lpisms_and_store(txln_info, lp_lang, parse_state, filename):
r"""A translation entry is coming in from LP, so we convert it to a format
that the GRD expects. Reverse the ${} to <ph name="" /> conversion that
the exporter does.
>>> decode_lpisms_and_store({"4038523199754403233":{}}, "en-AU", {"msgid":["id"], "msgstr":["\\\\xb1 Count"], "h_key":["4038523199754403233"]}, "f")
{'4038523199754403233': {'langs': {'en-AU': [('\\xb1 Count', 'PO', 'f')]}}}
>>> decode_lpisms_and_store({"4038523199754403233":{}}, "he", {"msgid":["id"], "msgstr":["s\nx"], "h_key":["4038523199754403233"]}, "f")
{'4038523199754403233': {'langs': {'iw': [('s\nx', 'PO', 'f')]}}}
"""
if "msgid" not in parse_state or parse_state["msgid"] == [""]:
#logging.warn("Ignoring state with no msgid. Probably a header.")
return txln_info
msgstr = "".join(parse_state["msgstr"])
msgid = "".join(parse_state["msgid"])
if msgstr == "":
# We have no translation for this key. That's okay!
return txln_info
assert not msgstr.endswith("\n")
if "<ph" in msgstr:
translator_logger.error("%s msgid %r msgstr %r PO has GRD placeholders. That can't be right.", lp_lang, msgid, msgstr)
return txln_info
assert "<ph" not in msgstr, msgstr
msgstr = re.sub(r"%\{(\w+)\}", r"""<ph name="\1" />""", msgstr)
charmap = { "x":"x", "n":"\n", "\\":"\\", '"':'"', }
# RE removes one layer of backslash escaping. This matches one backslash and one nonnewline char.
try:
msgstr = re.sub(r"\\(.)", lambda m: (charmap[m.group(1)]), msgstr)
except KeyError as exc:
translator_logger.error("%s msgid %r msgstr %r has a backslash expression %s that we don't understand. Discarding.", lp_lang, msgid, msgstr, exc)
return txln_info
template_placeholders = sorted(re.findall(r"%\{(\w+)\}", msgid))
translation_placeholders = sorted(re.findall(r"%\{(\w+)\}", msgid))
if template_placeholders != translation_placeholders:
template_placeholders = set(template_placeholders)
translation_placeholders = set(translation_placeholders)
too_many = translation_placeholders - template_placeholders
fewer = template_placeholders - translation_placeholders
if too_many:
translator_logger.error("%s msgid %r msgstr %r has unknown placeholders %r. Discarding.", lp_lang, msgid, msgstr, too_many)
return txln_info
elif fewer:
translator_logger.warn("%s msgid %r msgstr %r is missing placeholders. That's weird. Keeping anyway.", lp_lang, msgid, msgstr)
else:
translator_logger.warn("%s msgid %r msgstr %r is has placeholders used a different number of times. That's weird. Keeping anyway.", lp_lang, msgid, msgstr)
computed_grdsid, grdsid_transformed_string = po_keystring_to_grdsid(msgid)
if "h_key" not in parse_state:
logging.warn("PO comment %r has no asserted SID", parse_state)
grdsid = computed_grdsid
else:
grdsid = parse_state["h_key"][0]
if computed_grdsid != grdsid:
logging.error("Computed grdsid %r does not match asserted one %r for %r (as %r)", computed_grdsid, grdsid, msgid, grdsid_transformed_string)
return txln_info
if computed_grdsid not in txln_info:
logging.info("grdsid %r from PO %r is correct, but is not in any GRD", computed_grdsid, filename)
txln_info[computed_grdsid] = dict()
grd_lang = find_grd_lang(lp_lang, txln_info, grdsid)
if grdsid not in txln_info:
logging.debug("Huh! Adding a new %s/%s translation for %r, which is not already known.", lp_lang, grd_lang, msgstr)
if "langs" not in txln_info[grdsid]:
txln_info[grdsid]["langs"] = dict()
if grd_lang in txln_info[grdsid]["langs"]:
other = txln_info[grdsid]["langs"][grd_lang][0]
if msgstr != other[0]:
pass
#logging.debug("Ooo! We have a correction lp %r != %s %r", msgstr, other[1], other[0])
else:
pass
# Normal case. No logging.
#logging.debug("Oh! We already have exactly %r from %s %s.", msgstr, other[1], other[2:])
else:
# Normal case. No logging.
txln_info[grdsid]["langs"][grd_lang] = list()
txln_info[grdsid]["langs"][grd_lang].append((msgstr, "PO", filename))
return txln_info
def find_grd_lang(lp_lang, txln_info, grdsid):
r"""With a segment of translation info, map it back to a fuzzy selection of
GRD translations. Use translations we already have for this string first,
then use a GRD lookup, then fall back to the same as LP. This is used for
importing translations, using data from LP, which we consider foreign and
of needing mapping to our canonical GRD form.
>>> find_grd_lang("he", {"x": {"langs": { "iw":"", "en":"" }}}, "x")
'iw'
>>> find_grd_lang("nb", {"x": {"langs": { "es":"", "en":"" }}}, "x")
'no'
>>> find_grd_lang("zh_TW", {"x": {"langs": { "es":"", "en":"" }}}, "x")
'zh-TW'
"""
preference_list = {'nb':'no', 'pt-PT':'pt', 'he':'iw'} # map good to maybe-in-use bad
lp_lang = lp_lang.replace("_", "-")
translation_array = txln_info[grdsid]
if lp_lang in translation_array:
return lp_lang # Definitely use the same thing.
maybe = preference_list.get(lp_lang)
if maybe:
return maybe
return lp_lang
def po_keystring_to_grdsid(keystring):
r"""Generate int fingerprint from of the leading 64 bits of the md5.
>>> po_keystring_to_grdsid("The Chromium Authors")
('985602178874221306', ...)
>>> po_keystring_to_grdsid('Relaunch Chrome')
('8914504000324227558', ...)
>>> po_keystring_to_grdsid(' Relaunch Chrome ')
('8914504000324227558', ...)
>>> po_keystring_to_grdsid("Del")
('6135826906199951471', ...)
>>> po_keystring_to_grdsid('Try it out - type \\"orchids\\" and press Enter.')
('1464570622807304272', ...)
>>> po_keystring_to_grdsid('''%{BEGIN_BOLD}Warning:%{END_BOLD} Chromium cannot prevent extensions from recording your browsing history. To disable this extension in incognito mode, unselect this option.''')
('4207043877577553402', ...)
>>> po_keystring_to_grdsid('%{PRODUCT_NAME} Preferences')
('3664704721673470303', ...)
>>> po_keystring_to_grdsid("The page at %{SITE}")
('1741763547273950878', ...)
>>> po_keystring_to_grdsid('f&oo "%{IDS_XX}" bar')
('8733787356824206756', ...)
>>> po_keystring_to_grdsid('CAPS LOCK is on\\\\nPress both Shift keys to cancel')
(..., 'CAPS LOCK is on\\nPress both Shift keys to cancel')
>>> po_keystring_to_grdsid('Your preferences file is corrupt or invalid.\\\\n\\\\nChromium is unable to recover your settings.')
('2632329825580776666', ...)
>#>> po_keystring_to_grdsid("If you use a proxy server, check your proxy settings or contact your\\n network administrator to make sure the proxy server is working. If you\\n don't believe you should be using a proxy server, adjust your proxy\\n settings:\\n %{PLATFORM_TEXT}")
('1681824736556685', ...)
"""
charmap = { "n":"\n", "\\":"\\", '"':'"', "x":"\\x", }
# See svn log http://src.chromium.org/svn/trunk/src/tools/grit/grit/extern/FP.py
# RE removes one layer of backslash escaping. This matches one backslash and one nonnewline char.
keystring = re.sub(r"\\(.)", lambda m: (charmap[m.group(1)]), keystring)
keystring = re.sub(r"(?!<\\)%\{(\w+)\}", r"\1", keystring, flags=re.DOTALL).strip()
#keystring = keystring.replace("©", chr(169))
hex128 = md5(keystring.encode("utf-8")).hexdigest()
hex64 = int(hex128[:16], 16)
return str(hex64 & 0x7FFFFFFFFFFFFFFF), keystring
def load_po(txln_info, lp_lang, filename):
"""The controller for reading a PO file in. LP/GRIT translation happens in
the functions this calls."""
with open(filename, encoding="utf-8") as f:
parse_po(txln_info, lp_lang, f, filename)
def parse_header(line):
c1 = line[0]
try:
c2 = line[1]
except IndexError:
return None, None
assert c1 == "#"
if c2 == " ":
pass
#logging.warn("Unexpected translator comment: %s", line)
elif c2 == ".":
remainder = line[3:].strip()
if remainder.startswith("IDS_"):
assert " " not in remainder
return "h_stringkey", remainder
elif remainder.startswith("- "):
pass
elif remainder.startswith("note: "):
pass
elif remainder.startswith("in: "):
pass
elif remainder.startswith("UNUSED: "):
pass
elif remainder.startswith("originally "):
pass
else:
logging.warn("Unexpected extractor comment: %s", line)
elif c2 == ":":
l = line[3:].strip()
if l.startswith("id: "):
items = l.split()
return "h_key", items[1]
elif l.startswith("intermediary:"):
pass
else:
logging.warn("Unexpected reference comment: %s", line)
else:
logging.warn("Unexpected unknown comment: %s", line)
return None, None
def parse_po(txln_info, lp_lang, file_iterable, filename, store=decode_lpisms_and_store):
r"""The controller for reading a PO file in. LP/GRIT translation happens in
the functions this calls.
>#>> r = parse_po({"5076340679995252485": {"langs": {"iw":[("&הדבק", "GRD", "ui_strings_iw.xtb")]}}}, "he", ['#. IDS_CONTENT_CONTEXT_BACK', '#. - description: The name of the Back command in the content area context menu', '''#. - condition: not pp_ifdef('use_titlecase')''', '#. IDS_CONTENT_CONTEXT_BACK', '#. - description: In Title Case: The name of the Back command in the content area context menu', '''#. - condition: pp_ifdef('use_titlecase')''', '#. IDS_HISTORY_BACK_LINUX', '#. - description: The Linux menu item for back in the history menu.', '#. - condition: is_posix and not is_macosx', '#: id: 3943582379552582368 (used in the following branches: trunk, dev, beta, stable)', 'msgid "&Back"', 'msgstr "&אחורה"', '', '#. IDS_FILE_BROWSER_PASTE_BUTTON_LABEL', '#. - description: Button Label.', '#. IDS_KEYBOARD_OVERLAY_PASTE', '#. - description: The text in the keyboard overlay to explain the shortcut.', '''#. - condition: pp_ifdef('chromeos')''', '#. IDS_PASTE_MAC', '#. - description: The Mac menu item for paste in the edit menu.', '#. - condition: is_macosx', '#: id: 3943857333388298514 (used in the following branches: trunk, dev, beta, stable)', 'msgid "Paste"', 'msgstr "הדבק"', '', '#. IDS_CLOUD_PRINT_SETUP_ANYWHERE_HEADER', '#. - description: Intro line about printing from anywhere.', '#: id: 3944384147860595744 (used in the following branches: trunk, dev, beta, stable)', 'msgid "Print from anywhere"', 'msgstr "הדפס מכל מקום"'], "he.po"); sorted(r.keys()), sorted(r["5076340679995252485"]["langs"].keys()), r["5076340679995252485"]["langs"]["iw"]
['3943582379552582368', '3943857333388298514', '3944384147860595744', '5076340679995252485'], ['iw'], [('&הדבק', 'GRD', 'ui_strings_iw.xtb'), ('הדבק", 'PO', 'he.po')]
>#>> parse_po({"1105130903706696483":{}}, "en_GB", ['#', 'msgid ""', 'msgstr ""', '"Project-Id-Version: chromium-browser"', '', '#. IDS_SESSION_CRASHED_VIEW_MESSAGE', '''#. - description: Message shown when the last session didn't exit cleanly.''', '#: id: 1105130903706696483 (used in the following branches: trunk, dev, beta, stable)', 'msgid ""', '''"Chromium didn't shut down correctly. To reopen the pages you had open, click "''', '"Restore."', 'msgstr ""', '''"Chromium didn't shut down correctly. To reopen the pages you had open, click "''', '"Restore."'], "f")["1105130903706696483"]["langs"]["en_GB"]
[("Chromium didn't shut down correctly. To reopen the pages you had open, click Restore.", 'PO', 'f')]
"""
state = dict(parsing="msgid")
for line in file_iterable:
line = line.rstrip("\n")
if not line:
continue
elif line.startswith("#"):
if state["parsing"] != "header":
# This line starts a header. That means the previous entry is
# finished its body, and we have to process it first, then
# reset the state, then parse this line for its header info.
if "msgstr" in state:
store(txln_info, lp_lang, state, filename)
state = dict(parsing="header")
key, value = parse_header(line)
if key:
if key not in state:
state[key] = list()
state[key].append(value)
elif line.startswith('msg') or line[0] == '"':
if line.startswith('msg'):
noun, line = line.split(" ", 1)
state["parsing"] = noun
state[noun] = list()
assert line.endswith('"')
assert line.startswith('"')
try:
state[noun].append(line[1:-1])
except KeyError:
print(filename, state, line)
raise
else:
logging.error("WTF %r", line)
# final one
store(txln_info, lp_lang, state, filename)
return txln_info
def save_all_po(txln_info, container_directory, template_name):
"""The edge of canonicalGRD-to-launchpad translation."""
file_map = dict()
file_written_grdsid = dict()
def po_file(lang):
"""Returns a file object and a mutable set. Gettext cannot have
duplicate templates, and hash key collision is one way we keep
duplicate items from occuring, but we also remap from some form to
another here, and that opens a gateway for duplicates to slip in. If
GRD (wrongly!) has translations for "pt" and "pt_PT", then the language
hash didn't make them unique, so we also keep track of what gresids
we've written BY FILE, not just GRD language code."""
lang = lang.replace("-", "_")
lang = { "no":"nb", "iw":"he", "pt_PT":"pt" }.get(lang, lang)
if lang not in file_map:
if lang not in known_lp_languages:
logging.error("Language %r is not in the list of known good languages.", lang)
try:
os.makedirs(os.path.join(container_directory, template_name), 0o755)
except:
pass
file_map[lang] = open(os.path.join(container_directory, template_name, lang + ".po"), "w")
file_map[lang].write("""# Chromium Translations for lang {0!r}.\n# Copyright 2013 Canonical\n# This file is distributed inder the same license as the chromium-browser package.\n# Contact Chad MILLER <chad.miller@canonical.com>\n\nmsgid ""\nmsgstr ""\n"Project-Id-Version: chromium-browser.head\\n"\n"Report-Msgid-Bugs-To: https://bugs.launchpad.net/ubuntu/+source/chromium-"\n"browser/+filebug\\n"\n"Language-Team: Launchpad {0} Translators <lp-l10n-{0}@lists.launchpad.net>\\n"\n"MIME-Version: 1.0\\n"\n"Content-Type: text/plain; charset=UTF-8\\n"\n"Content-Transfer-Encoding: 8bit\\n"\n\n""".format(lang.replace("_", "-").lower()))
if lang not in file_written_grdsid:
file_written_grdsid[lang] = set()
return file_map[lang], file_written_grdsid[lang]
try:
for grdsid, template_info in sorted(txln_info.items(), key=gettext_file_sorting_function):
if "refs" in template_info:
# For each of these, create a translation stanza in every language it lists.
if "langs" in template_info:
for lang, translations in template_info["langs"].items():
# Each of these writes a single stanza.
out, grdsids_written = po_file(lang)
if grdsid in grdsids_written:
logging.info("Skipping writing %r again for lang %s, another language in %s must have mapped onto this one already.", lang, grdsid, list(template_info["langs"].keys()))
continue
grdsids_written.add(grdsid)
out.write("\n")
condition_evaluation = False
for ref in template_info["refs"]:
# If ANY template references to this language survive, we recommend translating. No condition means True.
if not ref["conditions"]:
condition_evaluation = True
else:
# For this language, we can have many conditions, and all have to pass for this language to pass.
this_cond = all(evaluate_grd_condition(x, lang) for x in set(ref["conditions"]))
condition_evaluation = condition_evaluation or this_cond
if not condition_evaluation:
break
if not condition_evaluation:
out.write("#. UNUSED: {0} == False\n".format(" OR ".join(sorted(set(("("+ " and ".join(c for c in ref["conditions"] if c != "()") + ")") for ref in template_info["refs"])))))
for note in sorted(set(r["note"].strip() for r in template_info["refs"]))[:5]:
if not note:
continue
out.write("#. note: {0!r}\n".format(note))
# Notes to nonprogrammer translators about tricky things they might mess up.
if "\\x" in template_info["text"]:
out.write("#. note: Take care to keep backslash-x-letter-letter together.\n")
for example in sorted(r.get("example", "") for r in template_info["refs"]):
if example:
out.write("#. ex: {0}\n".format(example))
for ref in sorted(template_info["refs"], key=lambda r: r["grdfile"])[:2]:
out.write("#. in: {grdfile}, as {id}\n".format(**ref))
out.write("#: id: {0}; intermediary: cmlpgrid v1\n".format(grdsid))
po_translations = [x for x in translations if x[1] == "PO"]
if po_translations:
canonical_translation = po_translations[0]
else:
canonical_translation = translations[-1]
for translation in translations:
if translation[1] != "PO":
out.write("#. originally {0!r}\n".format(translation[0], translation[2]))
grd_msg_write_in_po("msgid", template_info["text"], out)
grd_msg_write_in_po("msgstr", canonical_translation[0], out)
out.write("\n")
finally:
for f in file_map.values():
f.close()
|