1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
|
package mnemonicode
import (
"unicode"
"unicode/utf8"
)
// modified version of bufio.ScanWords from bufio/scan.go
// scanWords is a split function for a Scanner that returns
// each non-letter separated word of text, with surrounding
// non-leters deleted. It will never return an empty string.
// The definition of letter is set by unicode.IsLetter.
func scanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
// Skip leading non-letters.
start := 0
for width := 0; start < len(data); start += width {
var r rune
r, width = utf8.DecodeRune(data[start:])
if unicode.IsLetter(r) {
break
}
}
if atEOF && len(data) == 0 {
return 0, nil, nil
}
// Scan until non-letter, marking end of word.
for width, i := 0, start; i < len(data); i += width {
var r rune
r, width = utf8.DecodeRune(data[i:])
if !unicode.IsLetter(r) {
return i + width, data[start:i], nil
}
}
// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
if atEOF && len(data) > start {
return len(data), data[start:], nil
}
// Request more data.
return 0, nil, nil
}
|
