File: types.ts

package info (click to toggle)
tokenizers 0.20.3%2Bdfsg-1
  • links: PTS, VCS
  • area: main
  • in suites: experimental
  • size: 5,480 kB
  • sloc: python: 4,499; javascript: 419; makefile: 124
file content (7 lines) | stat: -rw-r--r-- 441 bytes parent folder | download 
1
2
3
4
5
6
7
 
export type TextInputSequence = string
export type PreTokenizedInputSequence = string[]
export type InputSequence = TextInputSequence | PreTokenizedInputSequence

export type TextEncodeInput = TextInputSequence | [TextInputSequence, TextInputSequence]
export type PreTokenizedEncodeInput = PreTokenizedInputSequence | [PreTokenizedInputSequence, PreTokenizedInputSequence]
export type EncodeInput = TextEncodeInput | PreTokenizedEncodeInput