1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "core/dom/DocumentStatisticsCollector.h"
#include "core/dom/Document.h"
#include "core/frame/FrameView.h"
#include "core/html/HTMLHeadElement.h"
#include "core/html/HTMLLinkElement.h"
#include "core/testing/DummyPageHolder.h"
#include "public/platform/WebDistillability.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "wtf/text/StringBuilder.h"
#include <memory>
namespace blink {
// Saturate the length of a paragraph to save time.
const unsigned kTextContentLengthSaturation = 1000;
// Filter out short P elements. The threshold is set to around 2 English
// sentences.
const unsigned kParagraphLengthThreshold = 140;
class DocumentStatisticsCollectorTest : public ::testing::Test {
protected:
void SetUp() override;
void TearDown() override { ThreadState::current()->collectAllGarbage(); }
Document& document() const { return m_dummyPageHolder->document(); }
void setHtmlInnerHTML(const String&);
private:
std::unique_ptr<DummyPageHolder> m_dummyPageHolder;
};
void DocumentStatisticsCollectorTest::SetUp() {
m_dummyPageHolder = DummyPageHolder::create(IntSize(800, 600));
}
void DocumentStatisticsCollectorTest::setHtmlInnerHTML(
const String& htmlContent) {
document().documentElement()->setInnerHTML((htmlContent));
}
// This test checks open graph articles can be recognized.
TEST_F(DocumentStatisticsCollectorTest, HasOpenGraphArticle) {
setHtmlInnerHTML(
"<head>"
// Note the case-insensitive matching of the word "article".
" <meta property='og:type' content='arTiclE' />"
"</head>");
WebDistillabilityFeatures features =
DocumentStatisticsCollector::collectStatistics(document());
EXPECT_TRUE(features.openGraph);
}
// This test checks non-existence of open graph articles can be recognized.
TEST_F(DocumentStatisticsCollectorTest, NoOpenGraphArticle) {
setHtmlInnerHTML(
"<head>"
" <meta property='og:type' content='movie' />"
"</head>");
WebDistillabilityFeatures features =
DocumentStatisticsCollector::collectStatistics(document());
EXPECT_FALSE(features.openGraph);
}
// This test checks element counts are correct.
TEST_F(DocumentStatisticsCollectorTest, CountElements) {
setHtmlInnerHTML(
"<form>"
" <input type='text'>"
" <input type='password'>"
"</form>"
"<pre></pre>"
"<p><a> </a></p>"
"<ul><li><p><a> </a></p></li></ul>");
WebDistillabilityFeatures features =
DocumentStatisticsCollector::collectStatistics(document());
EXPECT_FALSE(features.openGraph);
EXPECT_EQ(10u, features.elementCount);
EXPECT_EQ(2u, features.anchorCount);
EXPECT_EQ(1u, features.formCount);
EXPECT_EQ(1u, features.textInputCount);
EXPECT_EQ(1u, features.passwordInputCount);
EXPECT_EQ(2u, features.pCount);
EXPECT_EQ(1u, features.preCount);
}
// This test checks score calculations are correct.
TEST_F(DocumentStatisticsCollectorTest, CountScore) {
setHtmlInnerHTML(
"<p class='menu' id='article'>1</p>" // textContentLength = 1
"<ul><li><p>12</p></li></ul>" // textContentLength = 2, skipped because
// under li
"<p class='menu'>123</p>" // textContentLength = 3, skipped because
// unlikelyCandidates
"<p>"
"12345678901234567890123456789012345678901234567890"
"12345678901234567890123456789012345678901234567890"
"12345678901234567890123456789012345678901234"
"</p>" // textContentLength = 144
"<p style='display:none'>12345</p>" // textContentLength = 5, skipped
// because invisible
"<div style='display:none'><p>123456</p></div>" // textContentLength = 6,
// skipped because
// invisible
"<div style='visibility:hidden'><p>1234567</p></div>" // textContentLength
// = 7, skipped
// because
// invisible
"<p style='opacity:0'>12345678</p>" // textContentLength = 8, skipped
// because invisible
"<p><a href='#'>1234 </a>6 <b> 9</b></p>" // textContentLength = 9
"<ul><li></li><p>123456789012</p></ul>" // textContentLength = 12
);
WebDistillabilityFeatures features =
DocumentStatisticsCollector::collectStatistics(document());
EXPECT_DOUBLE_EQ(features.mozScore, sqrt(144 - kParagraphLengthThreshold));
EXPECT_DOUBLE_EQ(features.mozScoreAllSqrt,
1 + sqrt(144) + sqrt(9) + sqrt(12));
EXPECT_DOUBLE_EQ(features.mozScoreAllLinear, 1 + 144 + 9 + 12);
}
// This test checks saturation of score calculations is correct.
TEST_F(DocumentStatisticsCollectorTest, CountScoreSaturation) {
StringBuilder html;
for (int i = 0; i < 10; i++) {
html.append("<p>");
for (int j = 0; j < 1000; j++) {
html.append("0123456789");
}
html.append("</p>");
}
setHtmlInnerHTML(html.toString());
WebDistillabilityFeatures features =
DocumentStatisticsCollector::collectStatistics(document());
double error = 1e-5;
EXPECT_NEAR(features.mozScore, 6 * sqrt(kTextContentLengthSaturation -
kParagraphLengthThreshold),
error);
EXPECT_NEAR(features.mozScoreAllSqrt, 6 * sqrt(kTextContentLengthSaturation),
error);
EXPECT_NEAR(features.mozScoreAllLinear, 6 * kTextContentLengthSaturation,
error);
}
} // namespace blink
|
