1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
|
package lfs
import (
"errors"
"sync"
"time"
"github.com/git-lfs/git-lfs/v3/config"
"github.com/git-lfs/git-lfs/v3/filepathfilter"
"github.com/git-lfs/git-lfs/v3/tr"
"github.com/rubyist/tracerx"
)
var missingCallbackErr = errors.New(tr.Tr.Get("no callback given"))
// IsCallbackMissing returns a boolean indicating whether the error is reporting
// that a GitScanner is missing a required GitScannerCallback.
func IsCallbackMissing(err error) bool {
return err == missingCallbackErr
}
// GitScanner scans objects in a Git repository for LFS pointers.
type GitScanner struct {
Filter *filepathfilter.Filter
FoundPointer GitScannerFoundPointer
FoundLockable GitScannerFoundLockable
PotentialLockables GitScannerSet
remote string
skippedRefs []string
closed bool
started time.Time
mu sync.Mutex
cfg *config.Configuration
}
type GitScannerFoundPointer func(*WrappedPointer, error)
type GitScannerFoundLockable func(filename string)
type GitScannerSet interface {
Contains(string) bool
}
// NewGitScanner initializes a *GitScanner for a Git repository in the current
// working directory.
func NewGitScanner(cfg *config.Configuration, cb GitScannerFoundPointer) *GitScanner {
return &GitScanner{started: time.Now(), FoundPointer: cb, cfg: cfg}
}
// Close stops exits once all processing has stopped, and all resources are
// tracked and cleaned up.
func (s *GitScanner) Close() {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return
}
s.closed = true
tracerx.PerformanceSince("scan", s.started)
}
// RemoteForPush sets up this *GitScanner to scan for objects to push to the
// given remote. Needed for ScanRangeToRemote().
func (s *GitScanner) RemoteForPush(r string) error {
s.mu.Lock()
defer s.mu.Unlock()
if len(s.remote) > 0 && s.remote != r {
return errors.New(tr.Tr.Get("trying to set remote to %q, already set to %q", r, s.remote))
}
s.remote = r
s.skippedRefs = calcSkippedRefs(r)
return nil
}
// ScanRangeToRemote scans through all unique objects reachable from the
// "include" ref but not reachable from the "exclude" ref and which the
// given remote does not have. See RemoteForPush().
func (s *GitScanner) ScanRangeToRemote(include, exclude string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
s.mu.Lock()
if len(s.remote) == 0 {
s.mu.Unlock()
return errors.New(tr.Tr.Get("unable to scan starting at %q: no remote set", include))
}
s.mu.Unlock()
return scanRefsToChanSingleIncludeExclude(s, callback, include, exclude, s.cfg.GitEnv(), s.cfg.OSEnv(), s.opts(ScanRangeToRemoteMode))
}
// ScanMultiRangeToRemote scans through all unique objects reachable from the
// "include" ref but not reachable from any "exclude" refs and which the
// given remote does not have. See RemoteForPush().
func (s *GitScanner) ScanMultiRangeToRemote(include string, exclude []string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
s.mu.Lock()
if len(s.remote) == 0 {
s.mu.Unlock()
return errors.New(tr.Tr.Get("unable to scan starting at %q: no remote set", include))
}
s.mu.Unlock()
return scanRefsToChanSingleIncludeMultiExclude(s, callback, include, exclude, s.cfg.GitEnv(), s.cfg.OSEnv(), s.opts(ScanRangeToRemoteMode))
}
// ScanRefs scans through all unique objects reachable from the "include" refs
// but not reachable from any "exclude" refs, including objects that have
// been modified or deleted.
func (s *GitScanner) ScanRefs(include, exclude []string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
opts := s.opts(ScanRefsMode)
opts.SkipDeletedBlobs = false
return scanRefsToChan(s, callback, include, exclude, s.cfg.GitEnv(), s.cfg.OSEnv(), opts)
}
// ScanRefRange scans through all unique objects reachable from the "include"
// ref but not reachable from the "exclude" ref, including objects that have
// been modified or deleted.
func (s *GitScanner) ScanRefRange(include, exclude string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
opts := s.opts(ScanRefsMode)
opts.SkipDeletedBlobs = false
return scanRefsToChanSingleIncludeExclude(s, callback, include, exclude, s.cfg.GitEnv(), s.cfg.OSEnv(), opts)
}
// ScanRefRangeByTree scans through all objects reachable from the "include"
// ref but not reachable from the "exclude" ref, including objects that have
// been modified or deleted. Objects which appear in multiple trees will
// be visited once per tree.
func (s *GitScanner) ScanRefRangeByTree(include, exclude string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
opts := s.opts(ScanRefsMode)
opts.SkipDeletedBlobs = false
opts.CommitsOnly = true
return scanRefsByTree(s, callback, []string{include}, []string{exclude}, s.cfg.GitEnv(), s.cfg.OSEnv(), opts)
}
// ScanRefWithDeleted scans through all unique objects in the given ref,
// including objects that have been modified or deleted.
func (s *GitScanner) ScanRefWithDeleted(ref string, cb GitScannerFoundPointer) error {
return s.ScanRefRange(ref, "", cb)
}
// ScanRef scans through all unique objects in the current ref, excluding
// objects that have been modified or deleted before the ref.
func (s *GitScanner) ScanRef(ref string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
opts := s.opts(ScanRefsMode)
opts.SkipDeletedBlobs = true
return scanRefsToChanSingleIncludeExclude(s, callback, ref, "", s.cfg.GitEnv(), s.cfg.OSEnv(), opts)
}
// ScanRefByTree scans through all objects in the current ref, excluding
// objects that have been modified or deleted before the ref. Objects which
// appear in multiple trees will be visited once per tree.
func (s *GitScanner) ScanRefByTree(ref string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
opts := s.opts(ScanRefsMode)
opts.SkipDeletedBlobs = true
opts.CommitsOnly = true
return scanRefsByTree(s, callback, []string{ref}, []string{}, s.cfg.GitEnv(), s.cfg.OSEnv(), opts)
}
// ScanAll scans through all unique objects in the repository, including
// objects that have been modified or deleted.
func (s *GitScanner) ScanAll(cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
opts := s.opts(ScanAllMode)
opts.SkipDeletedBlobs = false
return scanRefsToChanSingleIncludeExclude(s, callback, "", "", s.cfg.GitEnv(), s.cfg.OSEnv(), opts)
}
// ScanTree takes a ref and returns WrappedPointer objects in the tree at that
// ref. Differs from ScanRefs in that multiple files in the tree with the same
// content are all reported.
func (s *GitScanner) ScanTree(ref string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
return runScanTree(callback, ref, s.Filter, s.cfg.GitEnv(), s.cfg.OSEnv())
}
// ScanUnpushed scans history for all LFS pointers which have been added but not
// pushed to the named remote. remote can be left blank to mean 'any remote'.
func (s *GitScanner) ScanUnpushed(remote string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
return scanUnpushed(callback, remote)
}
// ScanStashed scans for all LFS pointers referenced solely by a stash
func (s *GitScanner) ScanStashed(cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
return scanStashed(callback, s)
}
// ScanPreviousVersions scans changes reachable from ref (commit) back to since.
// Returns channel of pointers for *previous* versions that overlap that time.
// Does not include pointers which were still in use at ref (use ScanRefsToChan
// for that)
func (s *GitScanner) ScanPreviousVersions(ref string, since time.Time, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
return logPreviousSHAs(callback, ref, s.Filter, since)
}
// ScanIndex scans the git index for modified LFS objects.
func (s *GitScanner) ScanIndex(ref string, cb GitScannerFoundPointer) error {
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
if err != nil {
return err
}
return scanIndex(callback, ref, s.Filter, s.cfg.GitEnv(), s.cfg.OSEnv())
}
func (s *GitScanner) opts(mode ScanningMode) *ScanRefsOptions {
s.mu.Lock()
defer s.mu.Unlock()
opts := newScanRefsOptions()
opts.ScanMode = mode
opts.RemoteName = s.remote
opts.skippedRefs = s.skippedRefs
return opts
}
func firstGitScannerCallback(callbacks ...GitScannerFoundPointer) (GitScannerFoundPointer, error) {
for _, cb := range callbacks {
if cb == nil {
continue
}
return cb, nil
}
return nil, missingCallbackErr
}
type ScanningMode int
const (
ScanRefsMode = ScanningMode(iota) // 0 - or default scan mode
ScanAllMode = ScanningMode(iota)
ScanRangeToRemoteMode = ScanningMode(iota)
)
type ScanRefsOptions struct {
ScanMode ScanningMode
RemoteName string
SkipDeletedBlobs bool
CommitsOnly bool
skippedRefs []string
nameMap map[string]string
mutex *sync.Mutex
}
func (o *ScanRefsOptions) GetName(sha string) (string, bool) {
o.mutex.Lock()
name, ok := o.nameMap[sha]
o.mutex.Unlock()
return name, ok
}
func (o *ScanRefsOptions) SetName(sha, name string) {
o.mutex.Lock()
o.nameMap[sha] = name
o.mutex.Unlock()
}
func newScanRefsOptions() *ScanRefsOptions {
return &ScanRefsOptions{
nameMap: make(map[string]string, 0),
mutex: &sync.Mutex{},
}
}
|