1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
package lfs
import (
"bytes"
"crypto/sha256"
"fmt"
"io"
"github.com/git-lfs/git-lfs/v3/config"
"github.com/git-lfs/git-lfs/v3/errors"
"github.com/git-lfs/git-lfs/v3/git"
"github.com/git-lfs/git-lfs/v3/tr"
)
// runCatFileBatch() uses an ObjectDatabase from the
// github.com/git-lfs/gitobj/v2 package to get the contents of Git
// blob objects, given their SHA1s, similar to the behaviour of
// 'git cat-file --batch'.
// Git blob SHA1s are read from the revs channel and fed to an
// ObjectScanner which looks them up in the ObjectDatabase.
// The contents will be decoded as Git LFS pointers and any valid pointers
// will be sent to pointerCh.
// If a Git blob is not an LFS pointer, check the lockableSet to see
// if that blob is for a locked file. Any errors are sent to errCh.
func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error, gitEnv, osEnv config.Environment) error {
scanner, err := NewPointerScanner(gitEnv, osEnv)
if err != nil {
return err
}
go func() {
canScan := true
for r := range revs.Results {
canScan = scanner.Scan(r)
if err := scanner.Err(); err != nil {
errCh <- err
} else if p := scanner.Pointer(); p != nil {
pointerCh <- p
} else if b := scanner.BlobSHA(); git.HasValidObjectIDLength(b) {
if name, ok := lockableSet.Check(b); ok {
lockableCh <- name
}
}
if !canScan {
break
}
}
if canScan {
if err := revs.Wait(); err != nil {
errCh <- err
}
}
if err := scanner.Close(); err != nil {
errCh <- err
}
close(pointerCh)
close(errCh)
close(lockableCh)
}()
return nil
}
type PointerScanner struct {
scanner *git.ObjectScanner
blobSha string
contentsSha string
pointer *WrappedPointer
err error
}
func NewPointerScanner(gitEnv, osEnv config.Environment) (*PointerScanner, error) {
scanner, err := git.NewObjectScanner(gitEnv, osEnv)
if err != nil {
return nil, err
}
return &PointerScanner{scanner: scanner}, nil
}
func (s *PointerScanner) BlobSHA() string {
return s.blobSha
}
func (s *PointerScanner) ContentsSha() string {
return s.contentsSha
}
func (s *PointerScanner) Pointer() *WrappedPointer {
return s.pointer
}
func (s *PointerScanner) Err() error {
return s.err
}
func (s *PointerScanner) Scan(sha string) bool {
s.pointer, s.err = nil, nil
s.blobSha, s.contentsSha = "", ""
b, c, p, err := s.next(sha)
s.blobSha = b
s.contentsSha = c
s.pointer = p
if err != nil {
if err != io.EOF {
s.err = err
}
return false
}
return true
}
func (s *PointerScanner) Close() error {
return s.scanner.Close()
}
func (s *PointerScanner) next(blob string) (string, string, *WrappedPointer, error) {
if !s.scanner.Scan(blob) {
if err := s.scanner.Err(); err != nil {
return "", "", nil, err
}
return "", "", nil, io.EOF
}
blobSha := s.scanner.Sha1()
size := s.scanner.Size()
sha := sha256.New()
var buf *bytes.Buffer
var to io.Writer = sha
if size < blobSizeCutoff {
buf = bytes.NewBuffer(make([]byte, 0, size))
to = io.MultiWriter(to, buf)
}
read, err := io.CopyN(to, s.scanner.Contents(), int64(size))
if err != nil {
return blobSha, "", nil, err
}
if int64(size) != read {
return blobSha, "", nil, errors.New(tr.Tr.Get("expected %d bytes, read %d bytes", size, read))
}
var pointer *WrappedPointer
var contentsSha string
if size < blobSizeCutoff {
if p, err := DecodePointer(bytes.NewReader(buf.Bytes())); err != nil {
contentsSha = fmt.Sprintf("%x", sha.Sum(nil))
} else {
pointer = &WrappedPointer{
Sha1: blobSha,
Pointer: p,
}
contentsSha = p.Oid
}
} else {
contentsSha = fmt.Sprintf("%x", sha.Sum(nil))
}
return blobSha, contentsSha, pointer, err
}
|
