File: selection.go

package info (click to toggle)
mirrorbits 0.6.1-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 984 kB
  • sloc: sh: 675; makefile: 93
file content (340 lines) | stat: -rw-r--r-- 9,933 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
// Copyright (c) 2014-2019 Ludovic Fauvet
// Licensed under the MIT license

package http

import (
	"errors"
	"fmt"
	"math"
	"math/rand"
	"sort"
	"strings"
	"time"

	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/mirrors"
	"github.com/etix/mirrorbits/network"
	"github.com/etix/mirrorbits/utils"
)

var (
	ErrInvalidFileInfo = errors.New("Invalid file info (modtime is zero)")
)

type mirrorSelection interface {
	// Selection must return an ordered list of selected mirror,
	// a list of rejected mirrors and and an error code.
	Selection(*Context, *mirrors.Cache, *filesystem.FileInfo, network.GeoIPRecord) (mirrors.Mirrors, mirrors.Mirrors, error)
}

// DefaultEngine is the default algorithm used for mirror selection
type DefaultEngine struct{}

// Selection returns an ordered list of selected mirror, a list of rejected mirrors and and an error code
func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *filesystem.FileInfo, clientInfo network.GeoIPRecord) (mlist mirrors.Mirrors, excluded mirrors.Mirrors, err error) {
	// Bail out early if we don't have valid file details
	if fileInfo.ModTime.IsZero() {
		err = ErrInvalidFileInfo
		return
	}

	// Prepare and return the list of all potential mirrors
	mlist, err = cache.GetMirrors(fileInfo.Path, clientInfo)
	if err != nil {
		return
	}

	// Filter the list of mirrors
	mlist, excluded, closestMirror, farthestMirror := Filter(mlist, ctx.SecureOption(), fileInfo, clientInfo)

	if !clientInfo.IsValid() {
		// Shuffle the list
		//XXX Should we use the fallbacks instead?
		for i := range mlist {
			j := rand.Intn(i + 1)
			mlist[i], mlist[j] = mlist[j], mlist[i]
		}

		// Shortcut
		if !ctx.IsMirrorlist() {
			// Reduce the number of mirrors to process
			mlist = mlist[:utils.Min(5, len(mlist))]
		}
		return
	}

	// We're not interested in divisions by zero
	if closestMirror == 0 {
		closestMirror = math.SmallestNonzeroFloat32
	}

	/* Weight distribution for random selection [Probabilistic weight] */

	// Compute score for each mirror and return the mirrors eligible for weight distribution.
	// This includes:
	// - mirrors found in a 1.5x (configurable) range from the closest mirror
	// - mirrors targeting the given country (as primary or secondary)
	// - mirrors being in the same AS number
	totalScore := 0
	baseScore := int(farthestMirror)
	weights := map[int]int{}
	for i := 0; i < len(mlist); i++ {
		m := &mlist[i]

		m.ComputedScore = baseScore - int(m.Distance) + 1

		if m.Distance <= closestMirror*GetConfig().WeightDistributionRange {
			score := (float32(baseScore) - m.Distance)
			if !network.IsPrimaryCountry(clientInfo, m.CountryFields) {
				score /= 2
			}
			m.ComputedScore += int(score)
		} else if network.IsPrimaryCountry(clientInfo, m.CountryFields) {
			m.ComputedScore += int(float32(baseScore) - (m.Distance * 5))
		} else if network.IsAdditionalCountry(clientInfo, m.CountryFields) {
			m.ComputedScore += int(float32(baseScore) - closestMirror)
		}

		if m.Asnum == clientInfo.ASNum {
			m.ComputedScore += baseScore / 2
		}

		floatingScore := float64(m.ComputedScore) + (float64(m.ComputedScore) * (float64(m.Score) / 100)) + 0.5

		// The minimum allowed score is 1
		m.ComputedScore = int(math.Max(floatingScore, 1))

		if m.ComputedScore > baseScore {
			// The weight must always be > 0 to not break the randomization below
			totalScore += m.ComputedScore - baseScore
			weights[m.ID] = m.ComputedScore - baseScore
		}
	}

	// Get the final number of mirrors selected for weight distribution
	selected := len(weights)

	// Sort mirrors by computed score
	sort.Sort(mirrors.ByComputedScore{Mirrors: mlist})

	if selected > 1 {

		if ctx.IsMirrorlist() {
			// Don't reorder the results, just set the percentage
			for i := 0; i < selected; i++ {
				id := mlist[i].ID
				for j := 0; j < len(mlist); j++ {
					if mlist[j].ID == id {
						mlist[j].Weight = float32(float64(weights[id]) * 100 / float64(totalScore))
						break
					}
				}
			}
		} else {
			// Randomize the order of the selected mirrors considering their weights
			weightedMirrors := make([]mirrors.Mirror, selected)
			rest := totalScore
			for i := 0; i < selected; i++ {
				var id int
				rv := rand.Int31n(int32(rest))
				s := 0
				for k, v := range weights {
					s += v
					if int32(s) > rv {
						id = k
						break
					}
				}
				for _, m := range mlist {
					if m.ID == id {
						m.Weight = float32(float64(weights[id]) * 100 / float64(totalScore))
						weightedMirrors[i] = m
						break
					}
				}
				rest -= weights[id]
				delete(weights, id)
			}

			// Replace the head of the list by its reordered counterpart
			mlist = append(weightedMirrors, mlist[selected:]...)

			// Reduce the number of mirrors to return
			v := math.Min(math.Min(5, float64(selected)), float64(len(mlist)))
			mlist = mlist[:int(v)]
		}
	} else if selected == 1 && len(mlist) > 0 {
		mlist[0].Weight = 100
	}
	return
}

// Filter mirror list, return the list of mirrors candidates for redirection,
// and the list of mirrors that were excluded. Also return the distance of the
// closest and farthest mirrors.
func Filter(mlist mirrors.Mirrors, secureOption SecureOption, fileInfo *filesystem.FileInfo, clientInfo network.GeoIPRecord) (accepted mirrors.Mirrors, excluded mirrors.Mirrors, closestMirror float32, farthestMirror float32) {
	// Check if this file is allowed to be outdated
	checkSize := true
	maxOutdated := time.Duration(0)
	config := GetConfig().AllowOutdatedFiles
	for _, c := range config {
		if strings.HasPrefix(fileInfo.Path, c.Prefix) {
			checkSize = false
			maxOutdated = time.Duration(c.Minutes) * time.Minute
			break
		}
	}

	accepted = make([]mirrors.Mirror, 0, len(mlist))
	excluded = make([]mirrors.Mirror, 0, len(mlist))

	for _, m := range mlist {
		// Is it enabled?
		if !m.Enabled {
			m.ExcludeReason = "Disabled"
			goto discard
		}

		// Is the procol requested supported by the mirror?
		// Is the mirror up for this protocol?
		switch secureOption {
		case WITHTLS:
			// HTTPS explicitly requested
			m.AbsoluteURL = ensureAbsolute(m.HttpURL, "https")
			httpsSupported := !strings.HasPrefix(m.HttpURL, "http://")
			if !httpsSupported {
				m.ExcludeReason = "Not HTTPS"
			} else if !m.HttpsUp {
				m.ExcludeReason = either(m.HttpsDownReason, "Down")
			} else {
				break
			}
			goto discard
		case WITHOUTTLS:
			// HTTP explicitly requested
			m.AbsoluteURL = ensureAbsolute(m.HttpURL, "http")
			httpSupported := !strings.HasPrefix(m.HttpURL, "https://")
			if !httpSupported {
				m.ExcludeReason = "Not HTTP"
			} else if !m.HttpUp {
				m.ExcludeReason = either(m.HttpDownReason, "Down")
			} else {
				break
			}
			goto discard
		default:
			// Any protocol will do - favor HTTPS if avail
			var httpReason, httpsReason string

			m.AbsoluteURL = ensureAbsolute(m.HttpURL, "https")
			httpsSupported := !strings.HasPrefix(m.HttpURL, "http://")
			if !httpsSupported {
				httpsReason = "Not HTTPS"
			} else if !m.HttpsUp {
				httpsReason = either(m.HttpsDownReason, "Down")
			} else {
				break
			}

			m.AbsoluteURL = ensureAbsolute(m.HttpURL, "http")
			httpSupported := !strings.HasPrefix(m.HttpURL, "https://")
			if !httpSupported {
				httpReason = "Not HTTP"
			} else if !m.HttpUp {
				httpReason = either(m.HttpDownReason, "Down")
			} else {
				break
			}

			if httpReason == httpsReason {
				m.ExcludeReason = httpReason
			} else {
				m.ExcludeReason = httpReason + " / " + httpsReason
			}
			goto discard
		}

		// Is it the same size / modtime as source?
		if m.FileInfo != nil {
			if checkSize && m.FileInfo.Size != fileInfo.Size {
				m.ExcludeReason = "File size mismatch"
				goto discard
			}
			if !m.FileInfo.ModTime.IsZero() {
				mModTime := m.FileInfo.ModTime
				if GetConfig().FixTimezoneOffsets {
					offset := time.Duration(m.TZOffset) * time.Millisecond
					mModTime = mModTime.Add(offset)
				}
				precision := m.LastSuccessfulSyncPrecision.Duration()
				mModTime = mModTime.Truncate(precision)
				lModTime := fileInfo.ModTime.Truncate(precision)
				delta := lModTime.Sub(mModTime)
				if delta < 0 || delta > maxOutdated {
					m.ExcludeReason = fmt.Sprintf("Mod time mismatch (diff: %s)", delta)
					goto discard
				}
			}
		}
		// Is it configured to serve its continent only?
		if m.ContinentOnly {
			if !clientInfo.IsValid() || clientInfo.ContinentCode != m.ContinentCode {
				m.ExcludeReason = "Continent only"
				goto discard
			}
		}
		// Is it configured to serve its country only?
		if m.CountryOnly {
			if !clientInfo.IsValid() || !utils.IsInSlice(clientInfo.CountryCode, m.CountryFields) {
				m.ExcludeReason = "Country only"
				goto discard
			}
		}
		// Is it in the same AS number?
		if m.ASOnly {
			if !clientInfo.IsValid() || clientInfo.ASNum != m.Asnum {
				m.ExcludeReason = "AS only"
				goto discard
			}
		}
		// Is the user's country code allowed on this mirror?
		if clientInfo.IsValid() && utils.IsInSlice(clientInfo.CountryCode, m.ExcludedCountryFields) {
			m.ExcludeReason = "User's country restriction"
			goto discard
		}
		// Keep track of the closest and farthest mirrors
		if len(accepted) == 0 {
			closestMirror = m.Distance
		} else if m.Distance < closestMirror {
			closestMirror = m.Distance
		}
		if m.Distance > farthestMirror {
			farthestMirror = m.Distance
		}
		accepted = append(accepted, m)
		continue
	discard:
		excluded = append(excluded, m)
	}

	return
}

// ensureAbsolute returns the url 'as is' if it's absolute (ie. it starts with
// a scheme), otherwise it prepends '<scheme>://' and returns the result.
func ensureAbsolute(url string, scheme string) string {
	if utils.HasAnyPrefix(url, "http://", "https://") {
		return url
	}
	return scheme + "://" + url
}

// either returns s if it's not empty, d otherwise
func either(s string, d string) string {
	if s != "" {
		return s
	}
	return d
}