1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
// Copyright (c) 2014-2019 Ludovic Fauvet
// Licensed under the MIT license
package scan
import (
"bufio"
"context"
"errors"
"fmt"
"net"
"net/http"
"strconv"
"time"
. "github.com/etix/mirrorbits/config"
"github.com/etix/mirrorbits/core"
"github.com/etix/mirrorbits/database"
"github.com/etix/mirrorbits/mirrors"
"github.com/etix/mirrorbits/utils"
)
var (
userAgent = "Mirrorbits/" + core.VERSION + " TRACE"
clientTimeout = time.Duration(20 * time.Second)
clientDeadline = time.Duration(40 * time.Second)
// ErrNoTrace is returned when no trace file is found
ErrNoTrace = errors.New("No trace file")
)
// Trace is the internal trace handler
type Trace struct {
redis *database.Redis
transport http.Transport
httpClient http.Client
stop <-chan struct{}
}
// NewTraceHandler returns a new instance of the trace file handler.
// Trace files are used to compute the time offset between a mirror
// and the local repository.
func NewTraceHandler(redis *database.Redis, stop <-chan struct{}) *Trace {
t := &Trace{
redis: redis,
stop: stop,
}
t.transport = http.Transport{
DisableKeepAlives: true,
MaxIdleConnsPerHost: 0,
Dial: func(network, addr string) (net.Conn, error) {
deadline := time.Now().Add(clientDeadline)
c, err := net.DialTimeout(network, addr, clientTimeout)
if err != nil {
return nil, err
}
c.SetDeadline(deadline)
return c, nil
},
}
t.httpClient = http.Client{
Transport: &t.transport,
}
return t
}
// GetLastUpdate connects in HTTP to the mirror to get the latest
// trace file and computes the offset of the mirror.
func (t *Trace) GetLastUpdate(mirror mirrors.Mirror) error {
traceFile := GetConfig().TraceFileLocation
if len(traceFile) == 0 {
return ErrNoTrace
}
log.Debugf("Getting latest trace file for %s...", mirror.Name)
// Prepare the mirror URL
var mirrorURL string
if utils.HasAnyPrefix(mirror.HttpURL, "http://", "https://") {
mirrorURL = mirror.HttpURL
} else if mirror.HttpsUp == true {
mirrorURL = "https://" + mirror.HttpURL
} else {
mirrorURL = "http://" + mirror.HttpURL
}
// Prepare the HTTP request
req, err := http.NewRequest("GET", utils.ConcatURL(mirrorURL, traceFile), nil)
req.Header.Set("User-Agent", userAgent)
req.Close = true
// Prepare contexts
ctx, cancel := context.WithTimeout(req.Context(), clientDeadline)
ctx = context.WithValue(ctx, core.ContextMirrorID, mirror.ID)
req = req.WithContext(ctx)
defer cancel()
go func() {
select {
case <-t.stop:
cancel()
case <-ctx.Done():
}
}()
resp, err := t.httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
scanner := bufio.NewScanner(bufio.NewReader(resp.Body))
scanner.Split(bufio.ScanWords)
scanner.Scan()
if err := scanner.Err(); err != nil {
return err
}
timestamp, err := strconv.ParseInt(scanner.Text(), 10, 64)
if err != nil {
return err
}
conn := t.redis.Get()
defer conn.Close()
_, err = conn.Do("HSET", fmt.Sprintf("MIRROR_%d", mirror.ID), "lastModTime", timestamp)
if err != nil {
return err
}
// Publish an update on redis
database.Publish(conn, database.MIRROR_UPDATE, strconv.Itoa(mirror.ID))
log.Debugf("[%s] trace last sync: %s", mirror.Name, time.Unix(timestamp, 0))
return nil
}
|
