1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
package gofeed
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"github.com/mmcdole/gofeed/atom"
"github.com/mmcdole/gofeed/json"
"github.com/mmcdole/gofeed/rss"
)
// ErrFeedTypeNotDetected is returned when the detection system can not figure
// out the Feed format
var ErrFeedTypeNotDetected = errors.New("Failed to detect feed type")
// HTTPError represents an HTTP error returned by a server.
type HTTPError struct {
StatusCode int
Status string
}
func (err HTTPError) Error() string {
return fmt.Sprintf("http error: %s", err.Status)
}
// Parser is a universal feed parser that detects
// a given feed type, parsers it, and translates it
// to the universal feed type.
type Parser struct {
AtomTranslator Translator
RSSTranslator Translator
JSONTranslator Translator
UserAgent string
Client *http.Client
rp *rss.Parser
ap *atom.Parser
jp *json.Parser
}
// NewParser creates a universal feed parser.
func NewParser() *Parser {
fp := Parser{
rp: &rss.Parser{},
ap: &atom.Parser{},
jp: &json.Parser{},
UserAgent: "Gofeed/1.0",
}
return &fp
}
// Parse parses a RSS or Atom or JSON feed into
// the universal gofeed.Feed. It takes an
// io.Reader which should return the xml/json content.
func (f *Parser) Parse(feed io.Reader) (*Feed, error) {
// Wrap the feed io.Reader in a io.TeeReader
// so we can capture all the bytes read by the
// DetectFeedType function and construct a new
// reader with those bytes intact for when we
// attempt to parse the feeds.
var buf bytes.Buffer
tee := io.TeeReader(feed, &buf)
feedType := DetectFeedType(tee)
// Glue the read bytes from the detect function
// back into a new reader
r := io.MultiReader(&buf, feed)
switch feedType {
case FeedTypeAtom:
return f.parseAtomFeed(r)
case FeedTypeRSS:
return f.parseRSSFeed(r)
case FeedTypeJSON:
return f.parseJSONFeed(r)
}
return nil, ErrFeedTypeNotDetected
}
// ParseURL fetches the contents of a given url and
// attempts to parse the response into the universal feed type.
func (f *Parser) ParseURL(feedURL string) (feed *Feed, err error) {
return f.ParseURLWithContext(feedURL, context.Background())
}
// ParseURLWithContext fetches contents of a given url and
// attempts to parse the response into the universal feed type.
// Request could be canceled or timeout via given context
func (f *Parser) ParseURLWithContext(feedURL string, ctx context.Context) (feed *Feed, err error) {
client := f.httpClient()
req, err := http.NewRequest("GET", feedURL, nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
req.Header.Set("User-Agent", f.UserAgent)
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if resp != nil {
defer func() {
ce := resp.Body.Close()
if ce != nil {
err = ce
}
}()
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, HTTPError{
StatusCode: resp.StatusCode,
Status: resp.Status,
}
}
return f.Parse(resp.Body)
}
// ParseString parses a feed XML string and into the
// universal feed type.
func (f *Parser) ParseString(feed string) (*Feed, error) {
return f.Parse(strings.NewReader(feed))
}
func (f *Parser) parseAtomFeed(feed io.Reader) (*Feed, error) {
af, err := f.ap.Parse(feed)
if err != nil {
return nil, err
}
return f.atomTrans().Translate(af)
}
func (f *Parser) parseRSSFeed(feed io.Reader) (*Feed, error) {
rf, err := f.rp.Parse(feed)
if err != nil {
return nil, err
}
return f.rssTrans().Translate(rf)
}
func (f *Parser) parseJSONFeed(feed io.Reader) (*Feed, error) {
jf, err := f.jp.Parse(feed)
if err != nil {
return nil, err
}
return f.jsonTrans().Translate(jf)
}
func (f *Parser) atomTrans() Translator {
if f.AtomTranslator != nil {
return f.AtomTranslator
}
f.AtomTranslator = &DefaultAtomTranslator{}
return f.AtomTranslator
}
func (f *Parser) rssTrans() Translator {
if f.RSSTranslator != nil {
return f.RSSTranslator
}
f.RSSTranslator = &DefaultRSSTranslator{}
return f.RSSTranslator
}
func (f *Parser) jsonTrans() Translator {
if f.JSONTranslator != nil {
return f.JSONTranslator
}
f.JSONTranslator = &DefaultJSONTranslator{}
return f.JSONTranslator
}
func (f *Parser) httpClient() *http.Client {
if f.Client != nil {
return f.Client
}
f.Client = &http.Client{}
return f.Client
}
|
