File: tokenize_fuzz.go

package info (click to toggle)
golang-github-jdkato-prose 1.2.1-1
  • links: PTS, VCS
  • area: main
  • in suites: bookworm, forky, sid, trixie
  • size: 12,904 kB
  • sloc: python: 115; makefile: 52; sh: 41
file content (21 lines) | stat: -rw-r--r-- 295 bytes parent folder | download | duplicates (2) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
 
// +build gofuzz

package tokenize

func Fuzz(data []byte) int {
	s := string(data)

	t1 := NewWordBoundaryTokenizer()
	t1.Tokenize(s)

	t2 := NewWordPunctTokenizer()
	t2.Tokenize(s)

	t3 := NewTreebankWordTokenizer()
	t3.Tokenize(s)

	t4 := NewBlanklineTokenizer()
	t4.Tokenize(s)

	return 0
}