1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package processor // import "miniflux.app/v2/internal/reader/processor"
import (
"encoding/json"
"fmt"
"log/slog"
"net/url"
"strings"
"time"
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxyrotator"
"miniflux.app/v2/internal/reader/fetcher"
)
func isYouTubeVideoURL(websiteURL string) bool {
return strings.Contains(websiteURL, "youtube.com/watch?v=")
}
func getVideoIDFromYouTubeURL(websiteURL string) string {
parsedWebsiteURL, err := url.Parse(websiteURL)
if err != nil {
return ""
}
return parsedWebsiteURL.Query().Get("v")
}
func shouldFetchYouTubeWatchTimeForSingleEntry(entry *model.Entry) bool {
return config.Opts.FetchYouTubeWatchTime() && config.Opts.YouTubeAPIKey() == "" && isYouTubeVideoURL(entry.URL)
}
func shouldFetchYouTubeWatchTimeInBulk() bool {
return config.Opts.FetchYouTubeWatchTime() && config.Opts.YouTubeAPIKey() != ""
}
func fetchYouTubeWatchTimeForSingleEntry(websiteURL string) (int, error) {
return fetchWatchTime(websiteURL, `meta[itemprop="duration"]`, true)
}
func fetchYouTubeWatchTimeInBulk(entries []*model.Entry) {
videosEntriesMapping := make(map[string]*model.Entry, len(entries))
videoIDs := make([]string, 0, len(entries))
for _, entry := range entries {
if !isYouTubeVideoURL(entry.URL) {
continue
}
youtubeVideoID := getVideoIDFromYouTubeURL(entry.URL)
if youtubeVideoID == "" {
continue
}
videosEntriesMapping[youtubeVideoID] = entry
videoIDs = append(videoIDs, youtubeVideoID)
}
if len(videoIDs) == 0 {
return
}
watchTimeMap, err := fetchYouTubeWatchTimeFromApiInBulk(videoIDs)
if err != nil {
slog.Warn("Unable to fetch YouTube watch time in bulk", slog.Any("error", err))
return
}
for videoID, watchTime := range watchTimeMap {
if entry, ok := videosEntriesMapping[videoID]; ok {
entry.ReadingTime = int(watchTime.Minutes())
}
}
}
func fetchYouTubeWatchTimeFromApiInBulk(videoIDs []string) (map[string]time.Duration, error) {
slog.Debug("Fetching YouTube watch time in bulk", slog.Any("video_ids", videoIDs))
apiQuery := url.Values{}
apiQuery.Set("id", strings.Join(videoIDs, ","))
apiQuery.Set("key", config.Opts.YouTubeAPIKey())
apiQuery.Set("part", "contentDetails")
apiURL := url.URL{
Scheme: "https",
Host: "www.googleapis.com",
Path: "youtube/v3/videos",
RawQuery: apiQuery.Encode(),
}
requestBuilder := fetcher.NewRequestBuilder()
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxyRotator(proxyrotator.ProxyRotatorInstance)
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String()))
defer responseHandler.Close()
if localizedError := responseHandler.LocalizedError(); localizedError != nil {
slog.Warn("Unable to fetch contentDetails from YouTube API", slog.Any("error", localizedError.Error()))
return nil, localizedError.Error()
}
videos := struct {
Items []struct {
ID string `json:"id"`
ContentDetails struct {
Duration string `json:"duration"`
} `json:"contentDetails"`
} `json:"items"`
}{}
if err := json.NewDecoder(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())).Decode(&videos); err != nil {
return nil, fmt.Errorf("youtube: unable to decode JSON: %v", err)
}
watchTimeMap := make(map[string]time.Duration, len(videos.Items))
for _, video := range videos.Items {
duration, err := parseISO8601Duration(video.ContentDetails.Duration)
if err != nil {
slog.Warn("Unable to parse ISO8601 duration", slog.Any("error", err))
continue
}
watchTimeMap[video.ID] = duration
}
return watchTimeMap, nil
}
|
