File: export_test.go

package info (click to toggle)
golang-github-blevesearch-segment 0.0~git20160915.0.762005e-7
  • links: PTS, VCS
  • area: main
  • in suites: bookworm, forky, sid, trixie
  • size: 4,288 kB
  • sloc: ruby: 202; makefile: 2
file content (20 lines) | stat: -rw-r--r-- 415 bytes parent folder | download | duplicates (3) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package segment

// Exported for testing only.
import (
	"unicode/utf8"
)

func (s *Segmenter) MaxTokenSize(n int) {
	if n < utf8.UTFMax || n > 1e9 {
		panic("bad max token size")
	}
	if n < len(s.buf) {
		s.buf = make([]byte, n)
	}
	s.maxTokenSize = n
}