File: GenerateCharacterCategory.py

package info (click to toggle)
codequery 1.0.1%2Bdfsg-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 17,860 kB
  • sloc: cpp: 151,420; xml: 16,576; python: 5,602; ansic: 5,487; makefile: 559; perl: 496; ruby: 209; sql: 194; sh: 106; php: 53; vhdl: 51; erlang: 47; objc: 22; lisp: 18; cobol: 18; modula3: 17; asm: 14; fortran: 12; ml: 11; tcl: 6
file content (53 lines) | stat: -rw-r--r-- 1,976 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
#!/usr/bin/env python3
# Script to generate scintilla/src/CharacterCategoryMap.cxx and lexilla/lexlib/CharacterCategory.cxx
# from Python's Unicode data
# Should be run rarely when a Python with a new version of Unicode data is available.
# Requires Python 3.3 or later
# Should not be run with old versions of Python.

import pathlib, platform, sys, unicodedata

from FileGenerator import Regenerate

def findCategories(filename):
    with filename.open(encoding="UTF-8") as infile:
        lines = [x.strip() for x in infile.readlines() if "\tcc" in x]
    values = "".join(lines).replace(" ","").split(",")
    print("Categrories:", values)
    return [v[2:] for v in values]

def updateCharacterCategory(filename):
    values = ["// Created with Python %s,  Unicode %s" % (
        platform.python_version(), unicodedata.unidata_version)]

    startRange = 0
    category = unicodedata.category(chr(startRange))
    table = []
    for ch in range(sys.maxunicode):
        uch = chr(ch)
        current = unicodedata.category(uch)
        if current != category:
            value = startRange * 32 + categories.index(category)
            table.append(value)
            category = current
            startRange = ch
    value = startRange * 32 + categories.index(category)
    table.append(value)

    # the sentinel value is used to simplify CharacterCategoryMap::Optimize()
    category = 'Cn'
    value = (sys.maxunicode + 1)*32 + categories.index(category)
    table.append(value)

    values.extend(["%d," % value for value in table])

    Regenerate(filename, "//", values)


scintillaDirectory = pathlib.Path(__file__).resolve().parent.parent

categories = findCategories(scintillaDirectory / "src" / "CharacterCategoryMap.h")

updateCharacterCategory(scintillaDirectory / "src" / "CharacterCategoryMap.cxx")

updateCharacterCategory(scintillaDirectory.parent / "lexilla" / "lexlib" / "CharacterCategory.cxx")