1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
|
package lfs
import (
"encoding/hex"
"sync"
"github.com/git-lfs/git-lfs/v3/config"
"github.com/git-lfs/git-lfs/v3/git"
)
// The nameMap structure provides a goroutine-safe mapping of Git object IDs
// (as SHA hex strings) to their pathspecs, either fully-qualified directory
// paths for trees or file paths for blobs, as returned by "git rev-list".
type nameMap struct {
names map[string]string
mutex *sync.Mutex
}
func (m *nameMap) getName(sha string) (string, bool) {
m.mutex.Lock()
name, ok := m.names[sha]
m.mutex.Unlock()
return name, ok
}
func (m *nameMap) setName(sha, name string) {
m.mutex.Lock()
m.names[sha] = name
m.mutex.Unlock()
}
func newNameMap() *nameMap {
return &nameMap{
names: make(map[string]string, 0),
mutex: &sync.Mutex{},
}
}
type lockableNameSet struct {
nameMap *nameMap
set GitScannerSet
}
// Determines if the given blob sha matches a locked file.
func (s *lockableNameSet) Check(blobSha string) (string, bool) {
if s == nil || s.nameMap == nil || s.set == nil {
return "", false
}
name, ok := s.nameMap.getName(blobSha)
if !ok {
return name, ok
}
if s.set.Contains(name) {
return name, true
}
return name, false
}
func noopFoundLockable(name string) {}
// scanRefsToChan scans through all unique objects reachable from the
// "include" refs and not reachable from any "exclude" refs and invokes the
// provided callback for each pointer file, valid or invalid, that it finds.
// Reports unique OIDs once only, not multiple times if more than one file
// has the same content.
func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude []string, gitEnv, osEnv config.Environment) error {
revs, nameMap, err := revListShas(scanner, include, exclude)
if err != nil {
return err
}
lockableSet := &lockableNameSet{nameMap: nameMap, set: scanner.potentialLockables}
smallShas, batchLockableCh, err := catFileBatchCheck(revs, lockableSet)
if err != nil {
return err
}
lockableCb := scanner.foundLockable
if lockableCb == nil {
lockableCb = noopFoundLockable
}
go func(cb GitScannerFoundLockable, ch chan string) {
for name := range ch {
cb(name)
}
}(lockableCb, batchLockableCh)
pointers, checkLockableCh, err := catFileBatch(smallShas, lockableSet, gitEnv, osEnv)
if err != nil {
return err
}
for p := range pointers.Results {
if name, ok := nameMap.getName(p.Sha1); ok {
p.Name = name
}
if scanner.Filter.Allows(p.Name) {
pointerCb(p, nil)
}
}
for lockableName := range checkLockableCh {
if scanner.Filter.Allows(lockableName) {
lockableCb(lockableName)
}
}
if err := pointers.Wait(); err != nil {
pointerCb(nil, err)
}
return nil
}
// scanRefsToChanSingleIncludeExclude scans through all unique objects
// reachable from the "include" ref and not reachable from the "exclude" ref
// and invokes the provided callback for each pointer file, valid or invalid,
// that it finds.
// Reports unique OIDs once only, not multiple times if more than one file
// has the same content.
func scanRefsToChanSingleIncludeExclude(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude string, gitEnv, osEnv config.Environment) error {
return scanRefsToChan(scanner, pointerCb, []string{include}, []string{exclude}, gitEnv, osEnv)
}
// scanRefsToChanSingleIncludeMultiExclude scans through all unique objects
// reachable from the "include" ref and not reachable from any "exclude" refs
// and invokes the provided callback for each pointer file, valid or invalid,
// that it finds.
// Reports unique OIDs once only, not multiple times if more than one file
// has the same content.
func scanRefsToChanSingleIncludeMultiExclude(scanner *GitScanner, pointerCb GitScannerFoundPointer, include string, exclude []string, gitEnv, osEnv config.Environment) error {
return scanRefsToChan(scanner, pointerCb, []string{include}, exclude, gitEnv, osEnv)
}
// scanRefsByTree scans through all objects reachable from the "include" refs
// and not reachable from any "exclude" refs and invokes the provided callback
// for each pointer file, valid or invalid, that it finds.
// Objects which appear in multiple trees will be visited once per tree.
func scanRefsByTree(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude []string, gitEnv, osEnv config.Environment) error {
revs, _, err := revListShas(scanner, include, exclude)
if err != nil {
return err
}
errchan := make(chan error, 20) // multiple errors possible
wg := &sync.WaitGroup{}
for r := range revs.Results {
wg.Add(1)
go func(rev string) {
defer wg.Done()
err := runScanTreeForPointers(pointerCb, rev, gitEnv, osEnv)
if err != nil {
errchan <- err
}
}(r)
}
wg.Wait()
close(errchan)
for err := range errchan {
if err != nil {
return err
}
}
return revs.Wait()
}
// revListShas uses git rev-list to return the list of object sha1s
// for the given ref. If all is true, ref is ignored. It returns a
// channel from which sha1 strings can be read, and a map of the sha1
// object IDs to their pathspecs, which will be populated as the sha1
// strings are written to the channel.
func revListShas(scanner *GitScanner, include, exclude []string) (*StringChannelWrapper, *nameMap, error) {
nameMap := newNameMap()
revListScanner, err := git.NewRevListScanner(include, exclude, &git.ScanRefsOptions{
Mode: git.ScanningMode(scanner.mode),
SkipDeletedBlobs: scanner.skipDeletedBlobs,
CommitsOnly: scanner.commitsOnly,
Remote: scanner.remote,
SkippedRefs: scanner.skippedRefs,
Names: nameMap.names,
Mutex: nameMap.mutex,
})
if err != nil {
return nil, nil, err
}
revs := make(chan string, chanBufSize)
errs := make(chan error, 5) // may be multiple errors
go func() {
for revListScanner.Scan() {
sha := hex.EncodeToString(revListScanner.OID())
if name := revListScanner.Name(); len(name) > 0 {
nameMap.setName(sha, name)
}
revs <- sha
}
if err = revListScanner.Err(); err != nil {
errs <- err
}
if err = revListScanner.Close(); err != nil {
errs <- err
}
close(revs)
close(errs)
}()
return NewStringChannelWrapper(revs, errs), nameMap, nil
}
|