1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
#!perl
use strict;
use warnings;
use lib 't';
use Test::More tests => 1604;
use Util qw[pack_utf8];
BEGIN {
use_ok('Unicode::UTF8', qw[ decode_utf8 ]);
}
my @tests = (
[ "\x80", "\x{FFFD}" x 1 ],
[ "\x80\x80", "\x{FFFD}" x 2 ],
[ "\x80\x80\x80", "\x{FFFD}" x 3 ],
[ "\xC0\x80", "\x{FFFD}" x 2 ],
[ "\xC1\x80", "\x{FFFD}" x 2 ],
[ "\xC2", "\x{FFFD}" x 1 ],
[ "\xE0\x80\x80", "\x{FFFD}" x 3 ],
[ "\xE0\xA0", "\x{FFFD}" x 1 ],
[ "\xE0\x9F\x80", "\x{FFFD}" x 3 ],
[ "\xED\xA0\x80", "\x{FFFD}" x 3 ],
[ "\xED\x80", "\x{FFFD}" x 1 ],
[ "\xED\xBF\x80", "\x{FFFD}" x 3 ],
[ "\xF0\x80\x80\x80", "\x{FFFD}" x 4 ],
[ "\xF0\x90\x80", "\x{FFFD}" x 1 ],
[ "\xF0\x8F\x80\x80", "\x{FFFD}" x 4 ],
[ "\xF4\x80\x80", "\x{FFFD}" x 1 ],
[ "\xF4\x90\x80\x80", "\x{FFFD}" x 4 ],
[ "\xF5\x80\x80", "\x{FFFD}" x 3 ],
[ "\xF5\x80\x80\x80", "\x{FFFD}" x 4 ],
[ "\xF6\x80\x80", "\x{FFFD}" x 3 ],
[ "\xF7\x80\x80", "\x{FFFD}" x 3 ],
[ "\xF8\x80\x80\x80", "\x{FFFD}" x 4 ],
[ "\xF9\x80", "\x{FFFD}" x 2 ],
[ "\xFA\x80", "\x{FFFD}" x 2 ],
[ "\xFB\x80", "\x{FFFD}" x 2 ],
[ "\xFC\x80", "\x{FFFD}" x 2 ],
[ "\xFD\x80", "\x{FFFD}" x 2 ],
[ "\xFE\x80", "\x{FFFD}" x 2 ],
[ "\xFF\x80", "\x{FFFD}" x 2 ],
[ "\xC2\x20\x80", "\x{FFFD}\x20\x{FFFD}" ],
[ "\xDF\x20\x80", "\x{FFFD}\x20\x{FFFD}"],
[ "\xE0\xA0\x20", "\x{FFFD}\x20" ],
[ "\xEF\x80\x20", "\x{FFFD}\x20" ],
[ "\xF0\x90\x20\x80", "\x{FFFD}\x20\x{FFFD}" ],
[ "\xF4\x80\x20\x80", "\x{FFFD}\x20\x{FFFD}" ],
);
# \xE0 [\xA0-\xBF]
for my $o (0xA0..0xBF) {
push @tests, [ pack('C2', 0xE0, $o), "\x{FFFD}" ];
}
for my $o (0x00..0x9F, 0xC0..0xFF) {
push @tests, [ pack('C2', 0xE0, $o),
pack('U2', 0xFFFD, $o < 0x80 ? $o : 0xFFFD) ];
}
# \xED [\x80-\x9F]
for my $o (0x80..0x9F) {
push @tests, [ pack('C2', 0xED, $o), "\x{FFFD}" ];
}
for my $o (0x00..0x7F, 0xA0..0xFF) {
push @tests, [ pack('C2', 0xED, $o),
pack('U2', 0xFFFD, $o < 0x80 ? $o : 0xFFFD) ];
}
# \xF0 [\x90-\xBF]
for my $o (0x90..0xBF) {
push @tests, [ pack('C2', 0xF0, $o), "\x{FFFD}" ];
}
for my $o (0x00..0x8F, 0xC0..0xFF) {
push @tests, [ pack('C2', 0xF0, $o),
pack('U2', 0xFFFD, $o < 0x80 ? $o : 0xFFFD) ];
}
# \xF4 [\x80-\x8F]
for my $o (0x80..0x8F) {
push @tests, [ pack('C2', 0xF4, $o), "\x{FFFD}" ];
}
for my $o (0x00..0x7F, 0x90..0xFF) {
push @tests, [ pack('C2', 0xF4, $o),
pack('U2', 0xFFFD, $o < 0x80 ? $o : 0xFFFD) ];
}
for (my $i = 0x80; $i < 0x10FFFF; $i += 0x1000) {
my $octets = pack_utf8($i);
push @tests, [ substr($octets, 0, -1), "\x{FFFD}" ];
push @tests, [ substr($octets, 1), "\x{FFFD}" x (length($octets) - 1) ];
}
foreach my $test (@tests) {
my ($octets, $exp) = @$test;
my $got = do {
no warnings 'utf8';
decode_utf8($octets);
};
my $name = sprintf 'decode_utf8(<%s>) eq <%s>',
join(' ', map { sprintf '%.2X', ord } split //, $octets),
join(' ', map { sprintf '%.4X', ord } split //, $exp);
is($got, $exp, $name);
}
|
