1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
# utils/update_confusables.py - Utility to update definitions of unicode
# confusables
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
import os.path
import re
import sys
def _usage(program_name):
return 'usage: {}'.format(
program_name)
def _help(program_name):
return '{}\n\n'.format(_usage(program_name)) + \
'This script generates include/swift/Parse/Confusables.def from ' \
'utils/UnicodeData/confusables.txt.\n' \
'The latest version of the data file can be found at ' \
'ftp://ftp.unicode.org/Public/security/latest/confusables.txt.'
def main(args=sys.argv):
program_name = os.path.basename(args.pop(0))
if len(args) == 1 and args[0] in ['-h', '--help']:
print(_help(program_name))
return 0
charactersToCheck = [
u"(", u")", u"{",
u"}", u"[", u"]",
u".", u",", u":",
u";", u"=", u"@",
u"#", u"&", u"/",
u"|", u"\\", u"-",
u"*", u"+", u">",
u"<", u"!", u"?"
]
modifiedHex = [
hex(ord(char))[2:].zfill(4).upper() for char in charactersToCheck
]
basepath = os.path.dirname(__file__)
confusablesFilePath = os.path.abspath(
os.path.join(basepath, "UnicodeData/confusables.txt")
)
pairs = []
regex = r"(.+)\W+;\W+(.+)\W+;\W+MA*.[#*]*.[(].*[)](.+)\W→(.+)\W#.*"
with open(confusablesFilePath, 'r') as f:
pattern = re.compile(regex)
for line in f:
match = pattern.match(line)
if match is not None:
confusedString = match.group(1).replace(" ", "")
normalString = match.group(2).replace(" ", "")
confusedName = match.group(3).strip().title()
normalName = match.group(4).strip().replace("-", " ").title()
for hexValue in modifiedHex:
if hexValue == normalString:
confused = hex(int(confusedString, 16))
normal = hex(int(normalString, 16))
pairs.append((confused, confusedName,
normal, normalName))
defFilePath = os.path.abspath(
os.path.join(basepath, "..", "include/swift/Parse/Confusables.def")
)
with open(defFilePath, 'w') as f:
f.write("//===--- Confusables.def - Confusable unicode characters")
f.write(" ------------------===//")
header = '''
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
////////////////////////////////////////////////////////////////////////////////
// WARNING: This file is manually generated from
// utils/UnicodeData/confusables.txt and should not be directly modified.
// Run utils/generate_confusables.py to regenerate this file.
////////////////////////////////////////////////////////////////////////////////
'''
f.write(header)
f.write("// CONFUSABLE(CONFUSABLE_POINT, CONFUSABLE_NAME, " +
"BASE_POINT, BASE_NAME)\n\n")
for (confused, confusedName, expected, expectedName) in pairs:
# Ad-hoc substitutions for clarity
mappings = {"Solidus": "Forward Slash",
"Reverse Solidus": "Back Slash"}
newExpectedName = expectedName
if expectedName in mappings:
newExpectedName = mappings[expectedName]
f.write("CONFUSABLE(" + confused + ", " + '"' +
confusedName + '"' + ", " + expected + ", " +
'"' + newExpectedName + '"' + ")\n")
f.write("\n#undef CONFUSABLE\n")
if __name__ == '__main__':
main()
|
