File: scraper_test.go

package info (click to toggle)
miniflux 2.2.6-2
  • links: PTS, VCS
  • area: main
  • in suites: forky
  • size: 5,640 kB
  • sloc: xml: 4,843; javascript: 1,326; sh: 290; makefile: 179
file content (139 lines) | stat: -rw-r--r-- 4,409 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package scraper // import "miniflux.app/v2/internal/reader/scraper"

import (
	"bytes"
	"os"
	"strings"
	"testing"
)

func TestGetPredefinedRules(t *testing.T) {
	if getPredefinedScraperRules("http://www.phoronix.com/") == "" {
		t.Error("Unable to find rule for phoronix.com")
	}

	if getPredefinedScraperRules("https://www.linux.com/") == "" {
		t.Error("Unable to find rule for linux.com")
	}

	if getPredefinedScraperRules("https://linux.com/") == "" {
		t.Error("Unable to find rule for linux.com")
	}

	if getPredefinedScraperRules("https://example.org/") != "" {
		t.Error("A rule not defined should not return anything")
	}
}

func TestWhitelistedContentTypes(t *testing.T) {
	scenarios := map[string]bool{
		"text/html":                            true,
		"TeXt/hTmL":                            true,
		"application/xhtml+xml":                true,
		"text/html; charset=utf-8":             true,
		"application/xhtml+xml; charset=utf-8": true,
		"text/css":                             false,
		"application/javascript":               false,
		"image/png":                            false,
		"application/pdf":                      false,
	}

	for inputValue, expectedResult := range scenarios {
		actualResult := isAllowedContentType(inputValue)
		if actualResult != expectedResult {
			t.Errorf(`Unexpected result for content type whitelist, got "%v" instead of "%v"`, actualResult, expectedResult)
		}
	}
}

func TestSelectorRules(t *testing.T) {
	var ruleTestCases = map[string]string{
		"img.html":    "article > img",
		"iframe.html": "article > iframe",
		"p.html":      "article > p",
	}

	for filename, rule := range ruleTestCases {
		html, err := os.ReadFile("testdata/" + filename)
		if err != nil {
			t.Fatalf(`Unable to read file %q: %v`, filename, err)
		}

		_, actualResult, err := findContentUsingCustomRules(bytes.NewReader(html), rule)
		if err != nil {
			t.Fatalf(`Scraping error for %q - %q: %v`, filename, rule, err)
		}

		expectedResult, err := os.ReadFile("testdata/" + filename + "-result")
		if err != nil {
			t.Fatalf(`Unable to read file %q: %v`, filename, err)
		}

		if actualResult != strings.TrimSpace(string(expectedResult)) {
			t.Errorf(`Unexpected result for %q, got %q instead of %q`, rule, actualResult, expectedResult)
		}
	}
}

func TestParseBaseURLWithCustomRules(t *testing.T) {
	html := `<html><head><base href="https://example.com/"></head><body><img src="image.jpg"></body></html>`
	baseURL, _, err := findContentUsingCustomRules(strings.NewReader(html), "img")
	if err != nil {
		t.Fatalf(`Scraping error: %v`, err)
	}

	if baseURL != "https://example.com/" {
		t.Errorf(`Unexpected base URL, got %q instead of "https://example.com/"`, baseURL)
	}
}

func TestParseMultipleBaseURLWithCustomRules(t *testing.T) {
	html := `<html><head><base href="https://example.com/"><base href="https://example.org/"/></head><body><img src="image.jpg"></body></html>`
	baseURL, _, err := findContentUsingCustomRules(strings.NewReader(html), "img")
	if err != nil {
		t.Fatalf(`Scraping error: %v`, err)
	}

	if baseURL != "https://example.com/" {
		t.Errorf(`Unexpected base URL, got %q instead of "https://example.com/"`, baseURL)
	}
}

func TestParseRelativeBaseURLWithCustomRules(t *testing.T) {
	html := `<html><head><base href="/test"></head><body><img src="image.jpg"></body></html>`
	baseURL, _, err := findContentUsingCustomRules(strings.NewReader(html), "img")
	if err != nil {
		t.Fatalf(`Scraping error: %v`, err)
	}

	if baseURL != "" {
		t.Errorf(`Unexpected base URL, got %q`, baseURL)
	}
}

func TestParseEmptyBaseURLWithCustomRules(t *testing.T) {
	html := `<html><head><base href=" "></head><body><img src="image.jpg"></body></html>`
	baseURL, _, err := findContentUsingCustomRules(strings.NewReader(html), "img")
	if err != nil {
		t.Fatalf(`Scraping error: %v`, err)
	}

	if baseURL != "" {
		t.Errorf(`Unexpected base URL, got %q instead of ""`, baseURL)
	}
}

func TestParseMissingBaseURLWithCustomRules(t *testing.T) {
	html := `<html><head></head><body><img src="image.jpg"></body></html>`
	baseURL, _, err := findContentUsingCustomRules(strings.NewReader(html), "img")
	if err != nil {
		t.Fatalf(`Scraping error: %v`, err)
	}

	if baseURL != "" {
		t.Errorf(`Unexpected base URL, got %q instead of ""`, baseURL)
	}
}