1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
#!/usr/bin/env python3
# Copyright (c) 2024 T. Zachary Laine
#
# Distributed under the Boost Software License, Version 1.0. (See accompanying
# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
# Get the latest version of the data file at:
# https://www.unicode.org/Public/UCD/latest/ucd/CaseFolding.txt
import itertools
f = open('CaseFolding.txt')
lines = f.readlines()
lines = filter(
lambda x: False if (x.startswith('#') or x == '\n' or ' T;' in x or ' S;' in x) else True,
lines)
lines = list(map(lambda x: x.split(';')[:-1], lines))
f_line_indices = list(filter(lambda i: ' F;' in lines[i], range(len(lines))))
# Remove all C; lines that come before a F; line that applies to the same code
# point.
for f_idx in reversed(f_line_indices):
prev = f_idx - 1
if 0 <= prev:
if lines[f_idx][0] == lines[prev][0]:
lines = lines[:prev] + lines[prev + 1:]
max_mapping_len = 1
for line in lines:
mappings = line[2].strip().split(' ')
max_mapping_len = max(len(mappings), max_mapping_len)
def print_cps():
print('// Initial code points from CaseFolding.txt')
print('char32_t const cps[] = {')
str_ = ''
idx = 0
max_per_line = 8
for cp in map(lambda x: '0x' + x[0], lines):
str_ += cp + ', '
idx += 1
if (idx % max_per_line) == 0:
print(' ' + str_)
str_ = ''
if idx % max_per_line:
print(' ' + str_)
print('};\n')
print(f'[[maybe_unused]] char32_t const max_test_cp = 0x{lines[-1][0]} + 100;\n')
array_t = f'std::array<uint32_t, {max_mapping_len} + 1>'
def print_test(line):
num_mapping_cps = len(line[2].strip().split(' '))
mapping = ', '.join(map(lambda x: '0x' + x, line[2].strip().split(' ')))
trailing = ', 0' * (max_mapping_len + 1 - num_mapping_cps)
zeros = ', 0' * (max_mapping_len)
print(f''' {{
{array_t} const expected = {{ {mapping}{trailing} }};
{array_t} result = {{ 0{zeros} }};
boost::parser::detail::case_fold(0x{line[0]}, result.begin());
BOOST_TEST(result == expected);
}}''')
def print_tests():
idx = 0
max_per_TEST = 50
print(f'// hits_{int(idx / max_per_TEST)})\n{{')
for line in lines:
idx += 1
if (idx % max_per_TEST) == 0:
print(f'''}}
// test_{int(idx / max_per_TEST)})
{{''')
print_test(line)
print('}\n')
print('''// Copyright (c) 2024 T. Zachary Laine
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
// Warning: This header is auto-generated (see misc/generate_case_fold_tests.py).
#include <boost/parser/parser.hpp>
#include <boost/core/lightweight_test.hpp>
int main()
{
''')
print_cps()
print_tests()
print(f'''// misses
{{
char32_t next_cp = 0;
for (char32_t const * it = cps; it != std::end(cps); ++it) {{
for (char32_t cp = next_cp; cp < *it; ++cp) {{
{array_t} const expected = {{ cp, 0 }};
{array_t} result = {{ 0 }};
auto const first = result.data();
auto const last = boost::parser::detail::case_fold(cp, first);
BOOST_TEST(std::equal(first, last, expected.begin()));
BOOST_TEST(result == expected);
}}
next_cp = *it + 1;
}}
}}
return boost::report_errors();
}}''')
|
