1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
// Copyright (c) 2017, Shreyas Khare <skhare@rapid7.com>
// See LICENSE for licensing information
package main
import (
"encoding/csv"
"io"
"log"
"net/http"
"os"
"strings"
"text/template"
)
const path = "schemes.go"
var schemesTmpl = template.Must(template.New("schemes").Parse(`// Generated by schemesgen
package xurls
// Schemes is a sorted list of all IANA assigned schemes.
//
// Source: https://www.iana.org/assignments/uri-schemes/uri-schemes-1.csv
var Schemes = []string{
{{range $scheme := .Schemes}}` + "\t`" + `{{$scheme}}` + "`" + `,
{{end}}}
`))
func schemeList() []string {
resp, err := http.Get("https://www.iana.org/assignments/uri-schemes/uri-schemes-1.csv")
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
r := csv.NewReader(resp.Body)
r.Read() // ignore headers
schemes := make([]string, 0)
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
if strings.Contains(record[0], "OBSOLETE") {
continue // skip obsolete schemes; note the scheme column is abused
}
schemes = append(schemes, record[0])
}
return schemes
}
func writeSchemes(schemes []string) error {
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
return schemesTmpl.Execute(f, struct {
Schemes []string
}{
Schemes: schemes,
})
}
func main() {
schemes := schemeList()
log.Printf("Generating %s...", path)
if err := writeSchemes(schemes); err != nil {
log.Fatalf("Could not write path: %v", err)
}
}
|
