File: response_handler.go

package info (click to toggle)
miniflux 2.2.16-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 6,188 kB
  • sloc: xml: 4,853; javascript: 1,158; sh: 257; makefile: 161
file content (253 lines) | stat: -rw-r--r-- 8,579 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package fetcher // import "miniflux.app/v2/internal/reader/fetcher"

import (
	"crypto/x509"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"miniflux.app/v2/internal/locale"
)

type ResponseHandler struct {
	httpResponse *http.Response
	clientErr    error
}

func NewResponseHandler(httpResponse *http.Response, clientErr error) *ResponseHandler {
	return &ResponseHandler{httpResponse: httpResponse, clientErr: clientErr}
}

func (r *ResponseHandler) EffectiveURL() string {
	return r.httpResponse.Request.URL.String()
}

func (r *ResponseHandler) ContentType() string {
	return r.httpResponse.Header.Get("Content-Type")
}

func (r *ResponseHandler) LastModified() string {
	// Ignore caching headers for feeds that do not want any cache.
	if r.httpResponse.Header.Get("Expires") == "0" {
		return ""
	}
	return r.httpResponse.Header.Get("Last-Modified")
}

func (r *ResponseHandler) ETag() string {
	// Ignore caching headers for feeds that do not want any cache.
	if r.httpResponse.Header.Get("Expires") == "0" {
		return ""
	}
	return r.httpResponse.Header.Get("ETag")
}

func (r *ResponseHandler) Expires() time.Duration {
	expiresHeaderValue := r.httpResponse.Header.Get("Expires")
	if expiresHeaderValue != "" {
		t, err := time.Parse(time.RFC1123, expiresHeaderValue)
		if err == nil {
			// This rounds up to the next minute by rounding down and just adding a minute.
			return time.Until(t).Truncate(time.Minute) + time.Minute
		}
	}
	return 0
}

func (r *ResponseHandler) CacheControlMaxAge() time.Duration {
	cacheControlHeaderValue := r.httpResponse.Header.Get("Cache-Control")
	if cacheControlHeaderValue != "" {
		for _, directive := range strings.Split(cacheControlHeaderValue, ",") {
			directive = strings.TrimSpace(directive)
			if strings.HasPrefix(directive, "max-age=") {
				maxAge, err := strconv.Atoi(strings.TrimPrefix(directive, "max-age="))
				if err == nil {
					return time.Duration(maxAge) * time.Second
				}
			}
		}
	}
	return 0
}

func (r *ResponseHandler) ParseRetryDelay() time.Duration {
	retryAfterHeaderValue := r.httpResponse.Header.Get("Retry-After")
	if retryAfterHeaderValue != "" {
		// First, try to parse as an integer (number of seconds)
		if seconds, err := strconv.Atoi(retryAfterHeaderValue); err == nil {
			return time.Duration(seconds) * time.Second
		}

		// If not an integer, try to parse as an HTTP-date
		if t, err := time.Parse(time.RFC1123, retryAfterHeaderValue); err == nil {
			return time.Until(t).Truncate(time.Second)
		}
	}
	return 0
}

func (r *ResponseHandler) IsRateLimited() bool {
	return r.httpResponse != nil && r.httpResponse.StatusCode == http.StatusTooManyRequests
}

func (r *ResponseHandler) IsModified(lastEtagValue, lastModifiedValue string) bool {
	if r.httpResponse.StatusCode == http.StatusNotModified {
		return false
	}

	if r.ETag() != "" {
		return r.ETag() != lastEtagValue
	}

	if r.LastModified() != "" {
		return r.LastModified() != lastModifiedValue
	}

	return true
}

func (r *ResponseHandler) IsRedirect() bool {
	return r.httpResponse != nil &&
		(r.httpResponse.StatusCode == http.StatusMovedPermanently ||
			r.httpResponse.StatusCode == http.StatusFound ||
			r.httpResponse.StatusCode == http.StatusSeeOther ||
			r.httpResponse.StatusCode == http.StatusTemporaryRedirect ||
			r.httpResponse.StatusCode == http.StatusPermanentRedirect)
}

func (r *ResponseHandler) Close() {
	if r.httpResponse != nil && r.httpResponse.Body != nil && r.clientErr == nil {
		r.httpResponse.Body.Close()
	}
}

func (r *ResponseHandler) getReader(maxBodySize int64) io.ReadCloser {
	contentEncoding := strings.ToLower(r.httpResponse.Header.Get("Content-Encoding"))
	slog.Debug("Request response",
		slog.String("effective_url", r.EffectiveURL()),
		slog.String("content_length", r.httpResponse.Header.Get("Content-Length")),
		slog.String("content_encoding", contentEncoding),
		slog.String("content_type", r.httpResponse.Header.Get("Content-Type")),
	)

	reader := r.httpResponse.Body
	switch contentEncoding {
	case "br":
		reader = NewBrotliReadCloser(r.httpResponse.Body)
	case "gzip":
		reader = NewGzipReadCloser(r.httpResponse.Body)
	}
	return http.MaxBytesReader(nil, reader, maxBodySize)
}

func (r *ResponseHandler) Body(maxBodySize int64) io.ReadCloser {
	return r.getReader(maxBodySize)
}

func (r *ResponseHandler) ReadBody(maxBodySize int64) ([]byte, *locale.LocalizedErrorWrapper) {
	limitedReader := r.getReader(maxBodySize)

	buffer, err := io.ReadAll(limitedReader)
	if err != nil && err != io.EOF {
		if err, ok := err.(*http.MaxBytesError); ok {
			return nil, locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: response body too large: %d bytes", err.Limit), "error.http_response_too_large")
		}

		return nil, locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: unable to read response body: %w", err), "error.http_body_read", err)
	}

	if len(buffer) == 0 {
		return nil, locale.NewLocalizedErrorWrapper(errors.New("fetcher: empty response body"), "error.http_empty_response_body")
	}

	return buffer, nil
}

func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper {
	if r.clientErr != nil {
		switch {
		case isSSLError(r.clientErr):
			return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.tls_error", r.clientErr)
		case isNetworkError(r.clientErr):
			return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_operation", r.clientErr)
		case os.IsTimeout(r.clientErr):
			return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_timeout", r.clientErr)
		case errors.Is(r.clientErr, io.EOF):
			return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_empty_response")
		default:
			return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_client_error", r.clientErr)
		}
	}

	switch r.httpResponse.StatusCode {
	case http.StatusUnauthorized:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: access unauthorized (401 status code)"), "error.http_not_authorized")
	case http.StatusForbidden:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: access forbidden (403 status code)"), "error.http_forbidden")
	case http.StatusTooManyRequests:
		return locale.NewLocalizedErrorWrapper(errors.New("fetcher: too many requests (429 status code)"), "error.http_too_many_requests")
	case http.StatusNotFound, http.StatusGone:
		return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: resource not found (%d status code)", r.httpResponse.StatusCode), "error.http_resource_not_found")
	case http.StatusInternalServerError:
		return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: remote server error (%d status code)", r.httpResponse.StatusCode), "error.http_internal_server_error")
	case http.StatusBadGateway:
		return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: bad gateway (%d status code)", r.httpResponse.StatusCode), "error.http_bad_gateway")
	case http.StatusServiceUnavailable:
		return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: service unavailable (%d status code)", r.httpResponse.StatusCode), "error.http_service_unavailable")
	case http.StatusGatewayTimeout:
		return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: gateway timeout (%d status code)", r.httpResponse.StatusCode), "error.http_gateway_timeout")
	}

	if r.httpResponse.StatusCode >= 400 {
		return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: unexpected status code (%d status code)", r.httpResponse.StatusCode), "error.http_unexpected_status_code", r.httpResponse.StatusCode)
	}

	if r.httpResponse.StatusCode != 304 {
		// Content-Length = -1 when no Content-Length header is sent.
		if r.httpResponse.ContentLength == 0 {
			return locale.NewLocalizedErrorWrapper(errors.New("fetcher: empty response body"), "error.http_empty_response_body")
		}
	}

	return nil
}

func isNetworkError(err error) bool {
	if _, ok := err.(*url.Error); ok {
		return true
	}
	if err == io.EOF {
		return true
	}
	var opErr *net.OpError
	if ok := errors.As(err, &opErr); ok {
		return true
	}
	return false
}

func isSSLError(err error) bool {
	var certErr x509.UnknownAuthorityError
	if errors.As(err, &certErr) {
		return true
	}

	var hostErr x509.HostnameError
	if errors.As(err, &hostErr) {
		return true
	}

	var algErr x509.InsecureAlgorithmError
	return errors.As(err, &algErr)
}