1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
package syncpki
import (
"bufio"
"context"
"errors"
"fmt"
"os"
"os/exec"
"regexp"
"strings"
log "github.com/sirupsen/logrus"
)
const (
RsyncProtoPrefix = "rsync://"
)
var (
reDeletion = regexp.MustCompile("^deleting (.*)")
wantedFileExtensionRE = regexp.MustCompile("(.*\\.(cer|mft|crl|roa|gbr))$")
)
func ExtractFoldersPathFromRsyncURL(url string) (string, error) {
if !isRsyncURL(url) {
return "", fmt.Errorf("%q is not an rsync URL", url)
}
filePath := strings.TrimPrefix(url, RsyncProtoPrefix)
parts := strings.Split(filePath, "/")
return strings.Join(parts[0:len(parts)-1], "/"), nil
}
func ExtractFilePathFromRsyncURL(url string) (string, error) {
if !isRsyncURL(url) {
return "", fmt.Errorf("%q is not an rsync URL", url)
}
return strings.TrimPrefix(url, RsyncProtoPrefix), nil
}
func isRsyncURL(url string) bool {
return strings.HasPrefix(url, RsyncProtoPrefix)
}
// Determines if file has been deleted
func FilterMatch(line string) (string, bool, error) {
results := reDeletion.FindAllStringSubmatch(line, -1)
if len(results) > 0 {
return results[0][1], true, nil
}
return line, false, nil
}
type FileStat struct {
Path string
Deleted bool
}
// Runs the rsync binary on a URL
func RunRsync(ctx context.Context, uri string, bin string, dirPath string) ([]*FileStat, error) {
if bin == "" {
return nil, errors.New("rsync binary missing")
}
err := os.MkdirAll(dirPath, os.ModePerm)
if err != nil {
return nil, err
}
cmd := exec.CommandContext(ctx, bin, "-vrlt", uri, dirPath)
log.Debugf("Command ran: %v", cmd)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
if err := cmd.Start(); err != nil {
return nil, err
}
go func() {
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
errorStr := scanner.Text()
log.Error(errorStr)
err = scanner.Err()
if err != nil {
log.Errorf("%v: %v", uri, err)
return
}
}
}()
newuri := uri
uriSplit := strings.Split(newuri[8:], "/")
if uri[len(uri)-1] != '/' && len(uriSplit) > 2 {
newuri = "rsync://" + strings.Join(uriSplit[0:len(uriSplit)-1], "/")
} else {
newuri = strings.TrimSuffix(newuri, "/")
}
files := make([]*FileStat, 0)
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
line := scanner.Text()
match := wantedFileExtensionRE.MatchString(line)
log.Debugf("Rsync received from %v: %v (match=%v)", uri, line, match)
if match {
file, deleted, err := FilterMatch(line)
if err != nil {
return nil, err
}
files = append(files, &FileStat{
Path: fmt.Sprintf("%v/%v", newuri, file),
Deleted: deleted,
})
}
if err != nil {
return files, err
}
}
err = scanner.Err()
err = cmd.Wait()
return files, err
}
|
