1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
package textseg
import (
"fmt"
"reflect"
"strings"
"testing"
"unicode/utf8"
)
func TestScanGraphemeClusters(t *testing.T) {
tests := unicodeGraphemeTests
for i, test := range tests {
t.Run(fmt.Sprintf("%03d-%x", i, test.input), func(t *testing.T) {
got, err := AllTokens(test.input, ScanGraphemeClusters)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !reflect.DeepEqual(got, test.output) {
// Also get the rune values resulting from decoding utf8,
// since they are generally easier to look up to figure out
// what's failing.
runes := make([]string, 0, len(test.input))
seqs := make([][]byte, 0, len(test.input))
categories := make([]string, 0, len(test.input))
buf := test.input
for len(buf) > 0 {
r, size := utf8.DecodeRune(buf)
runes = append(runes, fmt.Sprintf("0x%04x", r))
seqs = append(seqs, buf[:size])
categories = append(categories, _GraphemeRuneType(r).String())
buf = buf[size:]
}
t.Errorf(
"wrong result\ninput: %s\nutf8s: %s\nrunes: %s\ncats: %s\ngot: %s\nwant: %s",
formatBytes(test.input),
formatByteRanges(seqs),
strings.Join(runes, " "),
strings.Join(categories, " "),
formatByteRanges(got),
formatByteRanges(test.output),
)
}
})
}
}
func formatBytes(buf []byte) string {
strs := make([]string, len(buf))
for i, b := range buf {
strs[i] = fmt.Sprintf("0x%02x", b)
}
return strings.Join(strs, " ")
}
func formatByteRanges(bufs [][]byte) string {
strs := make([]string, len(bufs))
for i, b := range bufs {
strs[i] = formatBytes(b)
}
return strings.Join(strs, " | ")
}
|
