1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
|
"""Tests for the Tools/i18n/msgfmt.py tool."""
import json
import sys
import unittest
from gettext import GNUTranslations
from pathlib import Path
from test.support.os_helper import temp_cwd
from test.support.script_helper import assert_python_failure, assert_python_ok
from test.test_tools import imports_under_tool, skip_if_missing, toolsdir
skip_if_missing('i18n')
data_dir = (Path(__file__).parent / 'msgfmt_data').resolve()
script_dir = Path(toolsdir) / 'i18n'
msgfmt_py = script_dir / 'msgfmt.py'
with imports_under_tool("i18n"):
import msgfmt
def compile_messages(po_file, mo_file):
assert_python_ok(msgfmt_py, '-o', mo_file, po_file)
class CompilationTest(unittest.TestCase):
def test_compilation(self):
self.maxDiff = None
with temp_cwd():
for po_file in data_dir.glob('*.po'):
with self.subTest(po_file=po_file):
mo_file = po_file.with_suffix('.mo')
with open(mo_file, 'rb') as f:
expected = GNUTranslations(f)
tmp_mo_file = mo_file.name
compile_messages(po_file, tmp_mo_file)
with open(tmp_mo_file, 'rb') as f:
actual = GNUTranslations(f)
self.assertDictEqual(actual._catalog, expected._catalog)
def test_translations(self):
with open(data_dir / 'general.mo', 'rb') as f:
t = GNUTranslations(f)
self.assertEqual(t.gettext('foo'), 'foo')
self.assertEqual(t.gettext('bar'), 'baz')
self.assertEqual(t.pgettext('abc', 'foo'), 'bar')
self.assertEqual(t.pgettext('xyz', 'foo'), 'bar')
self.assertEqual(t.gettext('Multilinestring'), 'Multilinetranslation')
self.assertEqual(t.gettext('"escapes"'), '"translated"')
self.assertEqual(t.gettext('\n newlines \n'), '\n translated \n')
self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 1),
'One email sent.')
self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 2),
'%d emails sent.')
self.assertEqual(t.npgettext('abc', 'One email sent.',
'%d emails sent.', 1),
'One email sent.')
self.assertEqual(t.npgettext('abc', 'One email sent.',
'%d emails sent.', 2),
'%d emails sent.')
def test_invalid_msgid_plural(self):
with temp_cwd():
Path('invalid.po').write_text('''\
msgid_plural "plural"
msgstr[0] "singular"
''')
res = assert_python_failure(msgfmt_py, 'invalid.po')
err = res.err.decode('utf-8')
self.assertIn('msgid_plural not preceded by msgid', err)
def test_plural_without_msgid_plural(self):
with temp_cwd():
Path('invalid.po').write_text('''\
msgid "foo"
msgstr[0] "bar"
''')
res = assert_python_failure(msgfmt_py, 'invalid.po')
err = res.err.decode('utf-8')
self.assertIn('plural without msgid_plural', err)
def test_indexed_msgstr_without_msgid_plural(self):
with temp_cwd():
Path('invalid.po').write_text('''\
msgid "foo"
msgid_plural "foos"
msgstr "bar"
''')
res = assert_python_failure(msgfmt_py, 'invalid.po')
err = res.err.decode('utf-8')
self.assertIn('indexed msgstr required for plural', err)
def test_generic_syntax_error(self):
with temp_cwd():
Path('invalid.po').write_text('''\
"foo"
''')
res = assert_python_failure(msgfmt_py, 'invalid.po')
err = res.err.decode('utf-8')
self.assertIn('Syntax error', err)
class POParserTest(unittest.TestCase):
@classmethod
def tearDownClass(cls):
# msgfmt uses a global variable to store messages,
# clear it after the tests.
msgfmt.MESSAGES.clear()
def test_strings(self):
# Test that the PO parser correctly handles and unescape
# strings in the PO file.
# The PO file format allows for a variety of escape sequences,
# octal and hex escapes.
valid_strings = (
# empty strings
('""', ''),
('"" "" ""', ''),
# allowed escape sequences
(r'"\\"', '\\'),
(r'"\""', '"'),
(r'"\t"', '\t'),
(r'"\n"', '\n'),
(r'"\r"', '\r'),
(r'"\f"', '\f'),
(r'"\a"', '\a'),
(r'"\b"', '\b'),
(r'"\v"', '\v'),
# non-empty strings
('"foo"', 'foo'),
('"foo" "bar"', 'foobar'),
('"foo""bar"', 'foobar'),
('"" "foo" ""', 'foo'),
# newlines and tabs
(r'"foo\nbar"', 'foo\nbar'),
(r'"foo\n" "bar"', 'foo\nbar'),
(r'"foo\tbar"', 'foo\tbar'),
(r'"foo\t" "bar"', 'foo\tbar'),
# escaped quotes
(r'"foo\"bar"', 'foo"bar'),
(r'"foo\"" "bar"', 'foo"bar'),
(r'"foo\\" "bar"', 'foo\\bar'),
# octal escapes
(r'"\120\171\164\150\157\156"', 'Python'),
(r'"\120\171\164" "\150\157\156"', 'Python'),
(r'"\"\120\171\164" "\150\157\156\""', '"Python"'),
# hex escapes
(r'"\x50\x79\x74\x68\x6f\x6e"', 'Python'),
(r'"\x50\x79\x74" "\x68\x6f\x6e"', 'Python'),
(r'"\"\x50\x79\x74" "\x68\x6f\x6e\""', '"Python"'),
)
with temp_cwd():
for po_string, expected in valid_strings:
with self.subTest(po_string=po_string):
# Construct a PO file with a single entry,
# compile it, read it into a catalog and
# check the result.
po = f'msgid {po_string}\nmsgstr "translation"'
Path('messages.po').write_text(po)
# Reset the global MESSAGES dictionary
msgfmt.MESSAGES.clear()
msgfmt.make('messages.po', 'messages.mo')
with open('messages.mo', 'rb') as f:
actual = GNUTranslations(f)
self.assertDictEqual(actual._catalog, {expected: 'translation'})
invalid_strings = (
# "''", # invalid but currently accepted
'"',
'"""',
'"" "',
'foo',
'"" "foo',
'"foo" foo',
'42',
'"" 42 ""',
# disallowed escape sequences
# r'"\'"', # invalid but currently accepted
# r'"\e"', # invalid but currently accepted
# r'"\8"', # invalid but currently accepted
# r'"\9"', # invalid but currently accepted
r'"\x"',
r'"\u1234"',
r'"\N{ROMAN NUMERAL NINE}"'
)
with temp_cwd():
for invalid_string in invalid_strings:
with self.subTest(string=invalid_string):
po = f'msgid {invalid_string}\nmsgstr "translation"'
Path('messages.po').write_text(po)
# Reset the global MESSAGES dictionary
msgfmt.MESSAGES.clear()
with self.assertRaises(Exception):
msgfmt.make('messages.po', 'messages.mo')
class CLITest(unittest.TestCase):
def test_help(self):
for option in ('--help', '-h'):
res = assert_python_ok(msgfmt_py, option)
err = res.err.decode('utf-8')
self.assertIn('Generate binary message catalog from textual translation description.', err)
def test_version(self):
for option in ('--version', '-V'):
res = assert_python_ok(msgfmt_py, option)
out = res.out.decode('utf-8').strip()
self.assertEqual('msgfmt.py 1.2', out)
def test_invalid_option(self):
res = assert_python_failure(msgfmt_py, '--invalid-option')
err = res.err.decode('utf-8')
self.assertIn('Generate binary message catalog from textual translation description.', err)
self.assertIn('option --invalid-option not recognized', err)
def test_no_input_file(self):
res = assert_python_ok(msgfmt_py)
err = res.err.decode('utf-8').replace('\r\n', '\n')
self.assertIn('No input file given\n'
"Try `msgfmt --help' for more information.", err)
def test_nonexistent_file(self):
assert_python_failure(msgfmt_py, 'nonexistent.po')
def update_catalog_snapshots():
for po_file in data_dir.glob('*.po'):
mo_file = po_file.with_suffix('.mo')
compile_messages(po_file, mo_file)
# Create a human-readable JSON file which is
# easier to review than the binary .mo file.
with open(mo_file, 'rb') as f:
translations = GNUTranslations(f)
catalog_file = po_file.with_suffix('.json')
with open(catalog_file, 'w') as f:
data = translations._catalog.items()
data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0]))
json.dump(data, f, indent=4)
f.write('\n')
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
update_catalog_snapshots()
sys.exit(0)
unittest.main()
|
