File: update-gtranslit.py

package info (click to toggle)
glib2.0 2.58.3-2+deb10u2
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, buster-backports, sid
  • size: 48,956 kB
  • sloc: ansic: 452,656; xml: 16,781; python: 6,149; makefile: 3,776; sh: 1,499; perl: 1,140; cpp: 9
file content (410 lines) | stat: -rwxr-xr-x 13,476 bytes parent folder | download | duplicates (4)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
#!/usr/bin/env python3

# Run this script like so:
#
#  ./update-gtranslit.py /path/to/glibc/localedata/locales > gtranslit-data.h

import sys, os

localedir = sys.argv[1]

# returns true if the name looks like a POSIX locale name
def looks_like_locale(name):
    name, _, variant = name.partition('@')

    if '_' not in name:
        return False

    lang, _, land = name.partition('_')

    return len(lang) == 2 or len(lang) == 3 and len(land) == 2

# handles <U1234> style escapes
def unescape(string):
    chunks = []

    n = len(string)
    i = 0

    while i < n:
        start_escape = string.find('<', i)

        if start_escape == -1:
            chunks.append(string[i:])
            break

        assert string[start_escape:start_escape + 2] == '<U'
        start_escape += 2

        end_escape = string.find('>', start_escape)
        assert end_escape != -1

        chunks.append(chr(int(string[start_escape:end_escape], 16)))
        i = end_escape + 1

    return ''.join(chunks)

# Checks if a string is ascii
def is_ascii(string):
    return all(ord(c) < 0x80 for c in string)

# A Mapping is a map from non-ascii strings to ascii strings.
#
# It corresponds to a sequence of one or more mapping lines:
#
#   <U00C4> "<U0041><U0308>";"<U0041><U0045>"
#
# in a file.
class Mapping:
    def __init__(self):
        self.serialised = None
        self.mapping = {}

    # Scans a string like
    #
    #   <U00C4> "<U0041><U0308>";"<U0041><U0045>" % LATIN CAPITAL LETTER A WITH DIAERESIS.
    #
    # and adds the first all-ascii choice (or IGNORE) to the mapping
    # dictionary, with the origin string as the key.  In the case of
    # IGNORE, stores the empty string.
    def consider_mapping_line(self, line):
        key, value, rest = (line + ' % comment').split(maxsplit=2)

        key = unescape(key)

        for alternative in value.split(';'):
            if alternative[0] == '"' and alternative[-1] == '"':
                unescaped = unescape(alternative[1:-1])
                if is_ascii(unescaped):
                    self.mapping[key] = unescaped
                    break

            elif alternative[0] == '<' and alternative[-1] == '>':
                unescaped = unescape(alternative)
                if is_ascii(unescaped):
                    self.mapping[key] = unescaped
                    break

            elif alternative == 'IGNORE':
                self.mapping[key] = ''
                break

    # Performs a normal dictionary merge, but ensures that there are no
    # conflicting entries between the original dictionary and the requested
    # changes
    def merge_mapping(self, changes):
        for key in changes.mapping:
            if key in self.mapping:
                assert self.mapping[key] == changes.mapping[key]

        self.mapping.update(changes.mapping)

    # Can't get much flatter...
    def get_flattened(self):
        return [self]

    def serialise(self, serialiser):
        if self.serialised is None:
            self.serialised = serialiser.add_mapping(self.mapping)

        return self.serialised

# A Chain is a sequence of mappings and chains.
#
# A chain contains another chain whenever "copy" or "include" is
# encountered in a source file.
#
# A chain contains a mapping whenever a sequence of mapping lines:
#
#   <U00C4> "<U0041><U0308>";"<U0041><U0045>"
#
# is encountered in a file.
#
# The order of lookup is reverse: later entries override earlier ones.
class Chain:
    def __init__(self, name):
        self.serialised = None
        self.name = name
        self.chain = []
        self.links = 0

        self.read_from_file(os.path.join(localedir, name))

    def read_from_file(self, filename):
        current_mapping = None
        in_lc_ctype = False
        in_translit = False

        fp = open(filename, encoding='ascii', errors='surrogateescape')

        for line in fp:
            line = line.strip()

            if in_lc_ctype:
                if line == 'END LC_CTYPE':
                    break

                if line.startswith('copy') or line.startswith('include'):
                    if current_mapping:
                        self.chain.append(current_mapping)

                    copyname = unescape(line.split('"', 3)[1])
                    copyfile = get_chain(copyname)
                    self.chain.append(copyfile)
                    copyfile.links += 1

                    current_mapping = None

                elif line == 'translit_start':
                    in_translit = True

                elif line == 'translit_end':
                    in_translit = False

                elif in_translit and line.startswith('<U'):
                    if not current_mapping:
                        current_mapping = Mapping()

                    current_mapping.consider_mapping_line(line)

                elif line == '' or line.startswith('%'):
                    pass

                elif 'default_missing <U003F>':
                    pass

                elif in_translit:
                    print('unknown line:', line)
                    assert False

            elif line == 'LC_CTYPE':
                in_lc_ctype = True

        if current_mapping:
            self.chain.append(current_mapping)

    # If there is only one link to this chain, we may as well just
    # return the contents of the chain so that they can be merged into
    # our sole parent directly.  Otherwise, return ourselves.
    def get_flattened(self):
        if self.links == 1:
            return sum((item.get_flattened() for item in self.chain), [])
        else:
            return [self]

    def serialise(self, serialiser):
        if self.serialised is None:
            # Before we serialise, see if we can optimise a bit
            self.chain = sum((item.get_flattened() for item in self.chain), [])

            i = 0
            while i < len(self.chain) - 1:
                if isinstance(self.chain[i], Mapping) and isinstance(self.chain[i + 1], Mapping):
                    # We have two mappings in a row.  Try to merge them.
                    self.chain[i].merge_mapping(self.chain[i + 1])
                    del self.chain[i + 1]
                else:
                    i += 1

            # If all that is left is one item, just serialise that directly
            if len(self.chain) == 1:
                self.serialised = self.chain[0].serialise(serialiser)
            else:
                ids = [item.serialise(serialiser) for item in self.chain]
                self.serialised = serialiser.add_chain(ids)

        return self.serialised

# Chain cache -- allows sharing of common chains
chains = {}
def get_chain(name):
    if not name in chains:
        chains[name] = Chain(name)

    return chains[name]


# Remove the country name from a locale, preserving variant
# eg: 'sr_RS@latin' -> 'sr@latin'
def remove_country(string):
    base, at, variant = string.partition('@')
    lang, _, land = base.partition('_')
    return lang + at + variant

def encode_range(start, end):
    assert start <= end
    length = end - start

    assert start < 0x1000
    assert length < 0x8

    result = 0x8000 + (length << 12) + start

    assert result < 0x10000

    return result

def c_pair_array(array):
    return '{ ' + ', '.join ('{ %u, %u }' % pair for pair in array) + ' };'

class Serialiser:
    def __init__(self):
        self.mappings = []
        self.chains = []
        self.locales = {}

    def add_mapping(self, mapping):
        if mapping in self.mappings:
            mapping_id = self.mappings.index(mapping)
        else:
            mapping_id = len(self.mappings)
            self.mappings.append(mapping)

        assert mapping_id < 128
        return mapping_id

    def add_chain(self, chain):
        if chain in self.chains:
            chain_id = self.chains.index(chain)
        else:
            chain_id = len(self.chains)
            self.chains.append(chain)

        assert chain_id < 128
        return 128 + chain_id

    def add_locale(self, name, item_id):
        self.locales[name] = item_id

    def add_default(self, item_id):
        self.default = item_id

    def optimise_locales(self):
        # Check if all regions of a language/variant agree
        languages = list(set(remove_country(locale) for locale in self.locales))

        for language in languages:
            locales = [locale for locale in self.locales if remove_country(locale) == language]

            item_id = self.locales[locales[0]]
            if all(self.locales[locale] == item_id for locale in locales):
                self.locales[language] = item_id
                for locale in locales:
                    del self.locales[locale]

        # Check if a variant is the same as the non-variant form
        # eg: 'de@euro' and 'de'
        for variant in list(locale for locale in self.locales if '@' in locale):
            base, _, _ = variant.partition('@')
            if base in self.locales and self.locales[base] == self.locales[variant]:
                del self.locales[variant]

        # Eliminate any entries that are just the same as the C locale
        for locale in list(self.locales):
            if self.locales[locale] == self.default:
                del self.locales[locale]

    def to_c(self):
        src_table = ''
        ascii_table = ''
        mappings_table = []
        mapping_ranges = []
        chains_table = []
        chain_starts = []
        locale_names = ''
        locale_index = []
        max_lookup = 0
        max_localename = 0

        for mapping in self.mappings:
            mapping_ranges.append ((len(mappings_table), len(mapping)))

            for key in sorted(mapping):
                if len(key) == 1 and ord(key[0]) < 0x8000:
                    src_range = ord(key[0])
                else:
                    existing = src_table.find(key)
                    if existing == -1:
                        start = len(src_table)
                        assert all(ord(c) <= 0x10ffff for c in key)
                        src_table += key
                        src_range = encode_range(start, len(src_table))
                        max_lookup = max(max_lookup, len(key))
                    else:
                        src_range = encode_range(existing, existing + len(key))

                value = mapping[key]
                if len(value) == 1 and ord(value[0]) < 0x80:
                    ascii_range = ord(value[0])
                else:
                    existing = ascii_table.find(value)
                    if existing == -1:
                        start = len(ascii_table)
                        assert all(ord(c) < 0x80 for c in value)
                        ascii_table += value
                        ascii_range = encode_range(start, len(ascii_table))
                    else:
                        ascii_range = encode_range(existing, existing + len(value))

                mappings_table.append ((src_range, ascii_range))

            mapping_end = len(mappings_table)

        for chain in self.chains:
            chain_starts.append(len(chains_table))

            for item_id in reversed(chain):
                assert item_id < 0xff
                chains_table.append(item_id)
            chains_table.append(0xff)

        for locale in sorted(self.locales):
            max_localename = max(max_localename, len(locale))
            name_offset = len(locale_names)
            assert all(ord(c) <= 0x7f for c in locale)
            locale_names += (locale + '\0')

            item_id = self.locales[locale]

            assert name_offset < 256
            assert item_id < 256
            locale_index.append((name_offset, item_id))

        print('/* Generated by update-gtranslit.py */')
        print('#define MAX_KEY_SIZE', max_lookup)
        print('#define MAX_LOCALE_NAME', max_localename)
        print('static const gunichar src_table[] = {', ', '.join(str(ord(c)) for c in src_table), '};')
        # cannot do this in plain ascii because of trigraphs... :(
        print('static const gchar ascii_table[] = {', ', '.join(str(ord(c)) for c in ascii_table), '};')
        print('static const struct mapping_entry mappings_table[] =', c_pair_array (mappings_table))
        print('static const struct mapping_range mapping_ranges[] =', c_pair_array (mapping_ranges))
        print('static const guint8 chains_table[] = {', ', '.join(str(i) for i in chains_table), '};')
        print('static const guint8 chain_starts[] = {', ', '.join(str(i) for i in chain_starts), '};')
        print('static const gchar locale_names[] = "' + locale_names.replace('\0', '\\0') + '";')
        print('static const struct locale_entry locale_index[] = ', c_pair_array (locale_index))
        print('static const guint8 default_item_id = %u;' % (self.default,))

    def dump(self):
        print(self.mappings)
        print(self.chains)
        print(self.locales)

locales = []
for name in os.listdir(localedir):
    if looks_like_locale(name):
        chain = get_chain(name)
        locales.append (chain)
        chain.links += 1

serialiser = Serialiser()

for locale in locales:
    serialiser.add_locale(locale.name, locale.serialise(serialiser))

i18n = get_chain('i18n').serialise(serialiser)
combining = get_chain('translit_combining').serialise(serialiser)
serialiser.add_default(serialiser.add_chain([i18n, combining]))

serialiser.optimise_locales()

serialiser.to_c()