File: charset.go

package info (click to toggle)
golang-github-advancedlogic-goose 0.0~git20200830.1225d53%2Bds-2
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, bullseye-backports
  • size: 472 kB
  • sloc: makefile: 128; sh: 11
file content (92 lines) | stat: -rw-r--r-- 2,714 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
package goose

import (
	"log"
	"strings"
	"unicode/utf8"

	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
)

// NormaliseCharset Overrides/fixes charset names to something we can parse.
// Fixes common mispellings and uses a canonical name for equivalent encodings.
// @see https://encoding.spec.whatwg.org#names-and-labels
func NormaliseCharset(characterSet string) string {
	characterSet = strings.ToUpper(characterSet)
	switch characterSet {
	case "UTF8", "UT-8", "UTR-8", "UFT-8", "UTF8-WITHOUT-BOM", "UTF8_GENERAL_CI":
		return "UTF-8"
	// override Japanese
	// CP943: IBM OS/2 Japanese, superset of Cp932 and Shift-JIS
	case "CP943", "CP943C", "SIFT_JIS", "SHIFT-JIS":
		return "SHIFT_JIS"
	// override Korean
	case "EUC-KR", "MS949", "KSC5601", "WINDOWS-949", "KS_C_5601-1987", "KSC_5601":
		return "UHC"
	// override Thai
	//case "TIS-620", "WINDOWS-874":
	//	return "ISO-8859-11"
	// override latin-2
	case "LATIN2_HUNGARIAN_CI", "LATIN2":
		return "LATIN-2"
	// override cyrillic
	case "WIN1251", "WIN-1251", "WINDOWS-1251":
		return "CP1251"
	// override Hebrew
	case "WINDOWS-1255":
		return "ISO-8859-8"
	// override Turkish
	//case "WINDOWS-1254":
	//	return "ISO-8859-9"
	// override the parsing of ISO-8859-1 to behave as Windows-1252 (CP1252):
	// in ISO-8859-1, everything from 128-255 in the ASCII table are ctrl characters,
	// whilst in CP1252 they're symbols
	// override Baltic
	case "WINDOWS-1257":
		return "ISO-8859-13"
	case "ANSI", "LATIN-1", "ISO", "RFC", "MACINTOSH", "8859-1", "8859-15", "ISO8859-1", "ISO8859-15", "ISO-8559-1", "ISO-8859-1", "ISO-8859-15":
		return "CP1252"
	}
	return characterSet
}

// UTF8encode converts a string from the source character set to UTF-8, skipping invalid byte sequences
// @see http://stackoverflow.com/questions/32512500/ignore-illegal-bytes-when-decoding-text-with-go
func UTF8encode(raw string, sourceCharset string) string {
	enc, name := charset.Lookup(sourceCharset)
	if nil == enc {
		log.Println("Cannot convert from", sourceCharset, ":", name)
		return raw
	}

	dst := make([]byte, len(raw))
	d := enc.NewDecoder()

	var (
		in  int
		out int
	)
	for in < len(raw) {
		// Do the transformation
		ndst, nsrc, err := d.Transform(dst[out:], []byte(raw[in:]), true)
		in += nsrc
		out += ndst
		if err == nil {
			// Completed transformation
			break
		}
		if err == transform.ErrShortDst {
			// Our output buffer is too small, so we need to grow it
			t := make([]byte, (cap(dst)+1)*2)
			copy(t, dst)
			dst = t
			continue
		}
		// We're here because of at least one illegal character. Skip over the current rune
		// and try again.
		_, width := utf8.DecodeRuneInString(raw[in:])
		in += width
	}
	return string(dst)
}