1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
|
// Copyright (c) 2014-2019 Ludovic Fauvet
// Licensed under the MIT license
package http
import (
"errors"
"fmt"
"math"
"math/rand"
"sort"
"strings"
"time"
. "github.com/etix/mirrorbits/config"
"github.com/etix/mirrorbits/filesystem"
"github.com/etix/mirrorbits/mirrors"
"github.com/etix/mirrorbits/network"
"github.com/etix/mirrorbits/utils"
)
var (
ErrInvalidFileInfo = errors.New("Invalid file info (modtime is zero)")
)
type mirrorSelection interface {
// Selection must return an ordered list of selected mirror,
// a list of rejected mirrors and and an error code.
Selection(*Context, *mirrors.Cache, *filesystem.FileInfo, network.GeoIPRecord) (mirrors.Mirrors, mirrors.Mirrors, error)
}
// DefaultEngine is the default algorithm used for mirror selection
type DefaultEngine struct{}
// Selection returns an ordered list of selected mirror, a list of rejected mirrors and and an error code
func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *filesystem.FileInfo, clientInfo network.GeoIPRecord) (mlist mirrors.Mirrors, excluded mirrors.Mirrors, err error) {
// Bail out early if we don't have valid file details
if fileInfo.ModTime.IsZero() {
err = ErrInvalidFileInfo
return
}
// Prepare and return the list of all potential mirrors
mlist, err = cache.GetMirrors(fileInfo.Path, clientInfo)
if err != nil {
return
}
// Filter the list of mirrors
mlist, excluded, closestMirror, farthestMirror := Filter(mlist, ctx.SecureOption(), fileInfo, clientInfo)
if !clientInfo.IsValid() {
// Shuffle the list
//XXX Should we use the fallbacks instead?
for i := range mlist {
j := rand.Intn(i + 1)
mlist[i], mlist[j] = mlist[j], mlist[i]
}
// Shortcut
if !ctx.IsMirrorlist() {
// Reduce the number of mirrors to process
mlist = mlist[:utils.Min(5, len(mlist))]
}
return
}
// We're not interested in divisions by zero
if closestMirror == 0 {
closestMirror = math.SmallestNonzeroFloat32
}
/* Weight distribution for random selection [Probabilistic weight] */
// Compute score for each mirror and return the mirrors eligible for weight distribution.
// This includes:
// - mirrors found in a 1.5x (configurable) range from the closest mirror
// - mirrors targeting the given country (as primary or secondary)
// - mirrors being in the same AS number
totalScore := 0
baseScore := int(farthestMirror)
weights := map[int]int{}
for i := 0; i < len(mlist); i++ {
m := &mlist[i]
m.ComputedScore = baseScore - int(m.Distance) + 1
if m.Distance <= closestMirror*GetConfig().WeightDistributionRange {
score := (float32(baseScore) - m.Distance)
if !network.IsPrimaryCountry(clientInfo, m.CountryFields) {
score /= 2
}
m.ComputedScore += int(score)
} else if network.IsPrimaryCountry(clientInfo, m.CountryFields) {
m.ComputedScore += int(float32(baseScore) - (m.Distance * 5))
} else if network.IsAdditionalCountry(clientInfo, m.CountryFields) {
m.ComputedScore += int(float32(baseScore) - closestMirror)
}
if m.Asnum == clientInfo.ASNum {
m.ComputedScore += baseScore / 2
}
floatingScore := float64(m.ComputedScore) + (float64(m.ComputedScore) * (float64(m.Score) / 100)) + 0.5
// The minimum allowed score is 1
m.ComputedScore = int(math.Max(floatingScore, 1))
if m.ComputedScore > baseScore {
// The weight must always be > 0 to not break the randomization below
totalScore += m.ComputedScore - baseScore
weights[m.ID] = m.ComputedScore - baseScore
}
}
// Get the final number of mirrors selected for weight distribution
selected := len(weights)
// Sort mirrors by computed score
sort.Sort(mirrors.ByComputedScore{Mirrors: mlist})
if selected > 1 {
if ctx.IsMirrorlist() {
// Don't reorder the results, just set the percentage
for i := 0; i < selected; i++ {
id := mlist[i].ID
for j := 0; j < len(mlist); j++ {
if mlist[j].ID == id {
mlist[j].Weight = float32(float64(weights[id]) * 100 / float64(totalScore))
break
}
}
}
} else {
// Randomize the order of the selected mirrors considering their weights
weightedMirrors := make([]mirrors.Mirror, selected)
rest := totalScore
for i := 0; i < selected; i++ {
var id int
rv := rand.Int31n(int32(rest))
s := 0
for k, v := range weights {
s += v
if int32(s) > rv {
id = k
break
}
}
for _, m := range mlist {
if m.ID == id {
m.Weight = float32(float64(weights[id]) * 100 / float64(totalScore))
weightedMirrors[i] = m
break
}
}
rest -= weights[id]
delete(weights, id)
}
// Replace the head of the list by its reordered counterpart
mlist = append(weightedMirrors, mlist[selected:]...)
// Reduce the number of mirrors to return
v := math.Min(math.Min(5, float64(selected)), float64(len(mlist)))
mlist = mlist[:int(v)]
}
} else if selected == 1 && len(mlist) > 0 {
mlist[0].Weight = 100
}
return
}
// Filter mirror list, return the list of mirrors candidates for redirection,
// and the list of mirrors that were excluded. Also return the distance of the
// closest and farthest mirrors.
func Filter(mlist mirrors.Mirrors, secureOption SecureOption, fileInfo *filesystem.FileInfo, clientInfo network.GeoIPRecord) (accepted mirrors.Mirrors, excluded mirrors.Mirrors, closestMirror float32, farthestMirror float32) {
// Check if this file is allowed to be outdated
checkSize := true
maxOutdated := time.Duration(0)
config := GetConfig().AllowOutdatedFiles
for _, c := range config {
if strings.HasPrefix(fileInfo.Path, c.Prefix) {
checkSize = false
maxOutdated = time.Duration(c.Minutes) * time.Minute
break
}
}
accepted = make([]mirrors.Mirror, 0, len(mlist))
excluded = make([]mirrors.Mirror, 0, len(mlist))
for _, m := range mlist {
// Is it enabled?
if !m.Enabled {
m.ExcludeReason = "Disabled"
goto discard
}
// Is the procol requested supported by the mirror?
// Is the mirror up for this protocol?
switch secureOption {
case WITHTLS:
// HTTPS explicitly requested
m.AbsoluteURL = ensureAbsolute(m.HttpURL, "https")
httpsSupported := !strings.HasPrefix(m.HttpURL, "http://")
if !httpsSupported {
m.ExcludeReason = "Not HTTPS"
} else if !m.HttpsUp {
m.ExcludeReason = either(m.HttpsDownReason, "Down")
} else {
break
}
goto discard
case WITHOUTTLS:
// HTTP explicitly requested
m.AbsoluteURL = ensureAbsolute(m.HttpURL, "http")
httpSupported := !strings.HasPrefix(m.HttpURL, "https://")
if !httpSupported {
m.ExcludeReason = "Not HTTP"
} else if !m.HttpUp {
m.ExcludeReason = either(m.HttpDownReason, "Down")
} else {
break
}
goto discard
default:
// Any protocol will do - favor HTTPS if avail
var httpReason, httpsReason string
m.AbsoluteURL = ensureAbsolute(m.HttpURL, "https")
httpsSupported := !strings.HasPrefix(m.HttpURL, "http://")
if !httpsSupported {
httpsReason = "Not HTTPS"
} else if !m.HttpsUp {
httpsReason = either(m.HttpsDownReason, "Down")
} else {
break
}
m.AbsoluteURL = ensureAbsolute(m.HttpURL, "http")
httpSupported := !strings.HasPrefix(m.HttpURL, "https://")
if !httpSupported {
httpReason = "Not HTTP"
} else if !m.HttpUp {
httpReason = either(m.HttpDownReason, "Down")
} else {
break
}
if httpReason == httpsReason {
m.ExcludeReason = httpReason
} else {
m.ExcludeReason = httpReason + " / " + httpsReason
}
goto discard
}
// Is it the same size / modtime as source?
if m.FileInfo != nil {
if checkSize && m.FileInfo.Size != fileInfo.Size {
m.ExcludeReason = "File size mismatch"
goto discard
}
if !m.FileInfo.ModTime.IsZero() {
mModTime := m.FileInfo.ModTime
if GetConfig().FixTimezoneOffsets {
offset := time.Duration(m.TZOffset) * time.Millisecond
mModTime = mModTime.Add(offset)
}
precision := m.LastSuccessfulSyncPrecision.Duration()
mModTime = mModTime.Truncate(precision)
lModTime := fileInfo.ModTime.Truncate(precision)
delta := lModTime.Sub(mModTime)
if delta < 0 || delta > maxOutdated {
m.ExcludeReason = fmt.Sprintf("Mod time mismatch (diff: %s)", delta)
goto discard
}
}
}
// Is it configured to serve its continent only?
if m.ContinentOnly {
if !clientInfo.IsValid() || clientInfo.ContinentCode != m.ContinentCode {
m.ExcludeReason = "Continent only"
goto discard
}
}
// Is it configured to serve its country only?
if m.CountryOnly {
if !clientInfo.IsValid() || !utils.IsInSlice(clientInfo.CountryCode, m.CountryFields) {
m.ExcludeReason = "Country only"
goto discard
}
}
// Is it in the same AS number?
if m.ASOnly {
if !clientInfo.IsValid() || clientInfo.ASNum != m.Asnum {
m.ExcludeReason = "AS only"
goto discard
}
}
// Is the user's country code allowed on this mirror?
if clientInfo.IsValid() && utils.IsInSlice(clientInfo.CountryCode, m.ExcludedCountryFields) {
m.ExcludeReason = "User's country restriction"
goto discard
}
// Keep track of the closest and farthest mirrors
if len(accepted) == 0 {
closestMirror = m.Distance
} else if m.Distance < closestMirror {
closestMirror = m.Distance
}
if m.Distance > farthestMirror {
farthestMirror = m.Distance
}
accepted = append(accepted, m)
continue
discard:
excluded = append(excluded, m)
}
return
}
// ensureAbsolute returns the url 'as is' if it's absolute (ie. it starts with
// a scheme), otherwise it prepends '<scheme>://' and returns the result.
func ensureAbsolute(url string, scheme string) string {
if utils.HasAnyPrefix(url, "http://", "https://") {
return url
}
return scheme + "://" + url
}
// either returns s if it's not empty, d otherwise
func either(s string, d string) string {
if s != "" {
return s
}
return d
}
|