File: treebank_test.go

package info (click to toggle)
golang-github-jdkato-prose 1.1.0%2Bgit20171031.e27abfd-2
  • links: PTS, VCS
  • area: main
  • in suites: buster
  • size: 12,848 kB
  • sloc: python: 115; makefile: 55; sh: 21
file content (24 lines) | stat: -rw-r--r-- 482 bytes parent folder | download | duplicates (3) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
package tokenize

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestTreebankWordTokenizer(t *testing.T) {
	input, output := getWordData("treebank_words.json")
	word := NewTreebankWordTokenizer()
	for i, s := range input {
		assert.Equal(t, output[i], word.Tokenize(s))
	}
}

func BenchmarkTreebankWordTokenizer(b *testing.B) {
	word := NewTreebankWordTokenizer()
	for n := 0; n < b.N; n++ {
		for _, s := range getWordBenchData() {
			word.Tokenize(s)
		}
	}
}