1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
|
package mahonia
// Generic converters for multibyte character sets.
// An mbcsTrie contains the data to convert from the character set to Unicode.
// If a character would be encoded as "\x01\x02\x03", its unicode value would be found at t.children[1].children[2].children[3].rune
// children either is nil or has 256 elements.
type mbcsTrie struct {
// For leaf nodes, the Unicode character that is represented.
char rune
// For non-leaf nodes, the trie to decode the remainder of the character.
children []mbcsTrie
}
// A MBCSTable holds the data to convert to and from Unicode.
type MBCSTable struct {
toUnicode mbcsTrie
fromUnicode map[rune]string
}
// AddCharacter adds a character to the table. rune is its Unicode code point,
// and bytes contains the bytes used to encode it in the character set.
func (table *MBCSTable) AddCharacter(c rune, bytes string) {
if table.fromUnicode == nil {
table.fromUnicode = make(map[rune]string)
}
table.fromUnicode[c] = bytes
trie := &table.toUnicode
for i := 0; i < len(bytes); i++ {
if trie.children == nil {
trie.children = make([]mbcsTrie, 256)
}
b := bytes[i]
trie = &trie.children[b]
}
trie.char = c
}
func (table *MBCSTable) Decoder() Decoder {
return func(p []byte) (c rune, size int, status Status) {
if len(p) == 0 {
status = NO_ROOM
return
}
if p[0] == 0 {
return 0, 1, SUCCESS
}
trie := &table.toUnicode
for trie.char == 0 {
if trie.children == nil {
return 0xfffd, 1, INVALID_CHAR
}
if len(p) < size+1 {
return 0, 0, NO_ROOM
}
trie = &trie.children[p[size]]
size++
}
c = trie.char
status = SUCCESS
return
}
}
func (table *MBCSTable) Encoder() Encoder {
return func(p []byte, c rune) (size int, status Status) {
bytes := table.fromUnicode[c]
if bytes == "" {
if len(p) > 0 {
p[0] = '?'
return 1, INVALID_CHAR
} else {
return 0, NO_ROOM
}
}
if len(p) < len(bytes) {
return 0, NO_ROOM
}
return copy(p, bytes), SUCCESS
}
}
|
