1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
/************************************************************************
**
** Copyright (C) 2016-2020 Kevin B. Hendricks Stratford, Ontario, Canada
** Copyright (C) 2013 John Schember <john@nachtimwald.com>
** Copyright (C) 2009-2011 Strahinja Markovic <strahinja.markovic@gmail.com>
**
** This file is part of PageEdit.
**
** PageEdit is free software: you can redistribute it and/or modify
** it under the terms of the GNU General Public License as published by
** the Free Software Foundation, either version 3 of the License, or
** (at your option) any later version.
**
** PageEdit is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU General Public License for more details.
**
** You should have received a copy of the GNU General Public License
** along with PageEdit. If not, see <http://www.gnu.org/licenses/>.
**
*************************************************************************/
#include <string>
#include <QFile>
#include <QString>
#include <QTextCodec>
#include <QRegularExpression>
#include "Utility.h"
#include "pageedit_exception.h"
#include "HTMLEncodingResolver.h"
const QString ENCODING_ATTRIBUTE = "encoding\\s*=\\s*(?:\"|')([^\"']+)(?:\"|')";
const QString CHARSET_ATTRIBUTE = "charset\\s*=\\s*(?:\"|')([^\"']+)(?:\"|')";
const QString STANDALONE_ATTRIBUTE = "standalone\\s*=\\s*(?:\"|')([^\"']+)(?:\"|')";
const QString VERSION_ATTRIBUTE = "<\\?xml[^>]*version\\s*=\\s*(?:\"|')([^\"']+)(?:\"|')[^>]*>";
// Accepts a full path to an HTML file.
// Reads the file, detects the encoding
// and returns the text converted to Unicode.
QString HTMLEncodingResolver::ReadHTMLFile(const QString &fullfilepath)
{
QFile file(fullfilepath);
// Check if we can open the file
if (!file.open(QFile::ReadOnly)) {
std::string msg = file.fileName().toStdString() + ": " + file.errorString().toStdString();
throw (CannotOpenFile(msg));
}
QByteArray data = file.readAll();
return Utility::ConvertLineEndings(GetCodecForHTML(data)->toUnicode(data));
}
// Accepts an HTML stream and tries to determine its encoding;
// if no encoding is detected, the default codec for this locale is returned.
// We use this function because Qt's QTextCodec::codecForHtml() function
// leaves a *lot* to be desired.
const QTextCodec *HTMLEncodingResolver::GetCodecForHTML(const QByteArray &raw_text)
{
unsigned char c1;
unsigned char c2;
unsigned char c3;
unsigned char c4;
QString text;
QTextCodec *codec;
if (raw_text.count() < 4) {
return QTextCodec::codecForName("UTF-8");
}
// Check the BOM if present.
c1 = raw_text.at(0);
c2 = raw_text.at(1);
c3 = raw_text.at(2);
c4 = raw_text.at(3);
if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
return QTextCodec::codecForName("UTF-8");
} else if (c1 == 0xFF && c2 == 0xFE && c3 == 0 && c4 == 0) {
return QTextCodec::codecForName("UTF-32LE");
} else if (c1 == 0 && c2 == 0 && c3 == 0xFE && c4 == 0xFF) {
return QTextCodec::codecForName("UTF-32BE");
} else if (c1 == 0xFE && c2 == 0xFF) {
return QTextCodec::codecForName("UTF-16BE");
} else if (c1 == 0xFF && c2 == 0xFE) {
return QTextCodec::codecForName("UTF-16LE");
}
// Alternating char followed by 0 is typical of utf 16 le without BOM.
if (c1 != 0 && c2 == 0 && c3 != 0 && c4 == 0) {
return QTextCodec::codecForName("UTF-16LE");
}
// Try to find an ecoding specified in the file itself.
text = Utility::Substring(0, 1024, raw_text);
// Check if the xml encoding attribute is set.
QRegularExpression enc_re(ENCODING_ATTRIBUTE);
QRegularExpressionMatch enc_mo = enc_re.match(text);
if (enc_mo.hasMatch()) {
codec = QTextCodec::codecForName(enc_mo.captured(1).toLatin1().toUpper());
if (codec) {
return codec;
}
}
// Check if the charset is set in the head.
QRegularExpression char_re(CHARSET_ATTRIBUTE);
QRegularExpressionMatch char_mo = char_re.match(text);
if (char_mo.hasMatch()) {
codec = QTextCodec::codecForName(char_mo.captured(1).toLatin1().toUpper());
if (codec) {
return codec;
}
}
// See if all characters within this document are utf-8.
if (IsValidUtf8(raw_text)) {
return QTextCodec::codecForName("UTF-8");
}
// Finally, let Qt guess and if it doesn't know it will return the codec
// for the current locale.
text = raw_text;
return QTextCodec::codecForHtml(raw_text, QTextCodec::codecForLocale());
}
// This function goes through the entire byte array
// and tries to see whether this is a valid UTF-8 sequence.
// If it's valid, this is probably a UTF-8 string.
bool HTMLEncodingResolver::IsValidUtf8(const QByteArray &string)
{
// This is an implementation of the Perl code written here:
// http://www.w3.org/International/questions/qa-forms-utf-8
//
// Basically, UTF-8 has a very specific byte-pattern. This function
// checks if the sent byte-sequence conforms to this pattern.
// If it does, chances are *very* high that this is UTF-8.
//
// This function is written to be fast, not pretty.
if (string.isNull()) {
return false;
}
int index = 0;
while (index < string.size()) {
QByteArray dword = string.mid(index, 4);
if (dword.size() < 4) {
dword = dword.leftJustified(4, '\0');
}
const unsigned char *bytes = (const unsigned char *) dword.constData();
// ASCII
if (bytes[0] == 0x09 ||
bytes[0] == 0x0A ||
bytes[0] == 0x0D ||
(0x20 <= bytes[0] && bytes[0] <= 0x7E)
) {
index += 1;
}
// non-overlong 2-byte
else if ((0xC2 <= bytes[0] && bytes[0] <= 0xDF) &&
(0x80 <= bytes[1] && bytes[1] <= 0xBF)
) {
index += 2;
} else if ((bytes[0] == 0xE0 && // excluding overlongs
(0xA0 <= bytes[1] && bytes[1] <= 0xBF) &&
(0x80 <= bytes[2] && bytes[2] <= 0xBF)) ||
(((0xE1 <= bytes[0] && bytes[0] <= 0xEC) || // straight 3-byte
bytes[0] == 0xEE ||
bytes[0] == 0xEF) &&
(0x80 <= bytes[1] && bytes[1] <= 0xBF) &&
(0x80 <= bytes[2] && bytes[2] <= 0xBF)) ||
(bytes[0] == 0xED && // excluding surrogates
(0x80 <= bytes[1] && bytes[1] <= 0x9F) &&
(0x80 <= bytes[2] && bytes[2] <= 0xBF))
) {
index += 3;
} else if ((bytes[0] == 0xF0 && // planes 1-3
(0x90 <= bytes[1] && bytes[1] <= 0xBF) &&
(0x80 <= bytes[2] && bytes[2] <= 0xBF) &&
(0x80 <= bytes[3] && bytes[3] <= 0xBF)) ||
((0xF1 <= bytes[0] && bytes[0] <= 0xF3) && // planes 4-15
(0x80 <= bytes[1] && bytes[1] <= 0xBF) &&
(0x80 <= bytes[2] && bytes[2] <= 0xBF) &&
(0x80 <= bytes[3] && bytes[3] <= 0xBF)) ||
(bytes[0] == 0xF4 && // plane 16
(0x80 <= bytes[1] && bytes[1] <= 0x8F) &&
(0x80 <= bytes[2] && bytes[2] <= 0xBF) &&
(0x80 <= bytes[3] && bytes[3] <= 0xBF))
) {
index += 4;
} else {
return false;
}
}
return true;
}
|
