File: 2022.go

package info (click to toggle)
golang-github-gogits-chardet 0.0~git20150115.0.2404f77+dfsg-4
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid
  • size: 172 kB
  • sloc: makefile: 3
file content (102 lines) | stat: -rw-r--r-- 2,277 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
package chardet

import (
	"bytes"
)

type recognizer2022 struct {
	charset string
	escapes [][]byte
}

func (r *recognizer2022) Match(input *recognizerInput) (output recognizerOutput) {
	return recognizerOutput{
		Charset:    r.charset,
		Confidence: r.matchConfidence(input.input),
	}
}

func (r *recognizer2022) matchConfidence(input []byte) int {
	var hits, misses, shifts int
input:
	for i := 0; i < len(input); i++ {
		c := input[i]
		if c == 0x1B {
			for _, esc := range r.escapes {
				if bytes.HasPrefix(input[i+1:], esc) {
					hits++
					i += len(esc)
					continue input
				}
			}
			misses++
		} else if c == 0x0E || c == 0x0F {
			shifts++
		}
	}
	if hits == 0 {
		return 0
	}
	quality := (100*hits - 100*misses) / (hits + misses)
	if hits+shifts < 5 {
		quality -= (5 - (hits + shifts)) * 10
	}
	if quality < 0 {
		quality = 0
	}
	return quality
}

var escapeSequences_2022JP = [][]byte{
	{0x24, 0x28, 0x43}, // KS X 1001:1992
	{0x24, 0x28, 0x44}, // JIS X 212-1990
	{0x24, 0x40},       // JIS C 6226-1978
	{0x24, 0x41},       // GB 2312-80
	{0x24, 0x42},       // JIS X 208-1983
	{0x26, 0x40},       // JIS X 208 1990, 1997
	{0x28, 0x42},       // ASCII
	{0x28, 0x48},       // JIS-Roman
	{0x28, 0x49},       // Half-width katakana
	{0x28, 0x4a},       // JIS-Roman
	{0x2e, 0x41},       // ISO 8859-1
	{0x2e, 0x46},       // ISO 8859-7
}

var escapeSequences_2022KR = [][]byte{
	{0x24, 0x29, 0x43},
}

var escapeSequences_2022CN = [][]byte{
	{0x24, 0x29, 0x41}, // GB 2312-80
	{0x24, 0x29, 0x47}, // CNS 11643-1992 Plane 1
	{0x24, 0x2A, 0x48}, // CNS 11643-1992 Plane 2
	{0x24, 0x29, 0x45}, // ISO-IR-165
	{0x24, 0x2B, 0x49}, // CNS 11643-1992 Plane 3
	{0x24, 0x2B, 0x4A}, // CNS 11643-1992 Plane 4
	{0x24, 0x2B, 0x4B}, // CNS 11643-1992 Plane 5
	{0x24, 0x2B, 0x4C}, // CNS 11643-1992 Plane 6
	{0x24, 0x2B, 0x4D}, // CNS 11643-1992 Plane 7
	{0x4e},             // SS2
	{0x4f},             // SS3
}

func newRecognizer_2022JP() *recognizer2022 {
	return &recognizer2022{
		"ISO-2022-JP",
		escapeSequences_2022JP,
	}
}

func newRecognizer_2022KR() *recognizer2022 {
	return &recognizer2022{
		"ISO-2022-KR",
		escapeSequences_2022KR,
	}
}

func newRecognizer_2022CN() *recognizer2022 {
	return &recognizer2022{
		"ISO-2022-CN",
		escapeSequences_2022CN,
	}
}