File: transfer.go

package info (click to toggle)
git-lfs 3.6.1-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 4,808 kB
  • sloc: sh: 21,256; makefile: 507; ruby: 417
file content (260 lines) | stat: -rw-r--r-- 7,360 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
// Package transfer collects together adapters for uploading and downloading LFS content
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package tq

import (
	"fmt"
	"time"

	"github.com/git-lfs/git-lfs/v3/errors"
	"github.com/git-lfs/git-lfs/v3/lfsapi"
	"github.com/git-lfs/git-lfs/v3/tools"
	"github.com/git-lfs/git-lfs/v3/tr"
)

type Direction int

const (
	Upload   = Direction(iota)
	Download = Direction(iota)
	Checkout = Direction(iota)
)

// Progress returns a string containing the operation in progress.
func (d Direction) Progress() string {
	switch d {
	case Checkout:
		return tr.Tr.Get("Checking out LFS objects")
	case Download:
		return tr.Tr.Get("Downloading LFS objects")
	case Upload:
		return tr.Tr.Get("Uploading LFS objects")
	default:
		return "<unknown>"
	}
}

func (d Direction) String() string {
	switch d {
	case Checkout:
		return "checkout"
	case Download:
		return "download"
	case Upload:
		return "upload"
	default:
		return "<unknown>"
	}
}

type Transfer struct {
	Name          string       `json:"name,omitempty"`
	Oid           string       `json:"oid,omitempty"`
	Size          int64        `json:"size"`
	Authenticated bool         `json:"authenticated,omitempty"`
	Actions       ActionSet    `json:"actions,omitempty"`
	Links         ActionSet    `json:"_links,omitempty"`
	Error         *ObjectError `json:"error,omitempty"`
	Path          string       `json:"path,omitempty"`
	Missing       bool         `json:"-"`
}

func (t *Transfer) Rel(name string) (*Action, error) {
	a, err := t.Actions.Get(name)
	if a != nil || err != nil {
		return a, err
	}

	if t.Links != nil {
		a, err := t.Links.Get(name)
		if a != nil || err != nil {
			return a, err
		}
	}

	return nil, nil
}

type ObjectError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

func (e *ObjectError) Error() string {
	return fmt.Sprintf("[%d] %s", e.Code, e.Message)
}

// newTransfer returns a copy of the given Transfer, with the name and path
// values set.
func newTransfer(tr *Transfer, name string, path string) *Transfer {
	t := &Transfer{
		Name:          name,
		Path:          path,
		Oid:           tr.Oid,
		Size:          tr.Size,
		Authenticated: tr.Authenticated,
		Actions:       make(ActionSet),
	}

	if tr.Error != nil {
		t.Error = &ObjectError{
			Code:    tr.Error.Code,
			Message: tr.Error.Message,
		}
	}

	for rel, action := range tr.Actions {
		t.Actions[rel] = &Action{
			Href:      action.Href,
			Header:    action.Header,
			ExpiresAt: action.ExpiresAt,
			ExpiresIn: action.ExpiresIn,
			Id:        action.Id,
			Token:     action.Token,
			createdAt: action.createdAt,
		}
	}

	if tr.Links != nil {
		t.Links = make(ActionSet)

		for rel, link := range tr.Links {
			t.Links[rel] = &Action{
				Href:      link.Href,
				Header:    link.Header,
				ExpiresAt: link.ExpiresAt,
				ExpiresIn: link.ExpiresIn,
				Id:        link.Id,
				Token:     link.Token,
				createdAt: link.createdAt,
			}
		}
	}

	return t
}

type Action struct {
	Href      string            `json:"href"`
	Header    map[string]string `json:"header,omitempty"`
	ExpiresAt time.Time         `json:"expires_at,omitempty"`
	ExpiresIn int               `json:"expires_in,omitempty"`
	Id        string            `json:"-"`
	Token     string            `json:"-"`

	createdAt time.Time
}

func (a *Action) IsExpiredWithin(d time.Duration) (time.Time, bool) {
	return tools.IsExpiredAtOrIn(a.createdAt, d, a.ExpiresAt, time.Duration(a.ExpiresIn)*time.Second)
}

type ActionSet map[string]*Action

const (
	// objectExpirationToTransfer is the duration we expect to have passed
	// from the time that the object's expires_at (or expires_in) property
	// is checked to when the transfer is executed.
	objectExpirationToTransfer = 5 * time.Second
)

func (as ActionSet) Get(rel string) (*Action, error) {
	a, ok := as[rel]
	if !ok {
		return nil, nil
	}

	if at, expired := a.IsExpiredWithin(objectExpirationToTransfer); expired {
		return nil, errors.NewRetriableError(&ActionExpiredErr{Rel: rel, At: at})
	}

	return a, nil
}

type ActionExpiredErr struct {
	Rel string
	At  time.Time
}

func (e ActionExpiredErr) Error() string {
	return tr.Tr.Get("action %q expires at %s",
		e.Rel, e.At.In(time.Local).Format(time.RFC822))
}

func IsActionExpiredError(err error) bool {
	if _, ok := err.(*ActionExpiredErr); ok {
		return true
	}
	return false
}

// NewAdapterFunc creates new instances of Adapter. Code that wishes
// to provide new Adapter instances should pass an implementation of this
// function to RegisterNewTransferAdapterFunc() on a *Manifest.
// name and dir are to provide context if one func implements many instances
type NewAdapterFunc func(name string, dir Direction) Adapter

type ProgressCallback func(name string, totalSize, readSoFar int64, readSinceLast int) error

type AdapterConfig interface {
	APIClient() *lfsapi.Client
	ConcurrentTransfers() int
	Remote() string
}

type adapterConfig struct {
	apiClient           *lfsapi.Client
	concurrentTransfers int
	remote              string
}

func (c *adapterConfig) ConcurrentTransfers() int {
	return c.concurrentTransfers
}

func (c *adapterConfig) APIClient() *lfsapi.Client {
	return c.apiClient
}

func (c *adapterConfig) Remote() string {
	return c.remote
}

// Adapter is implemented by types which can upload and/or download LFS
// file content to a remote store. Each Adapter accepts one or more requests
// which it may schedule and parallelise in whatever way it chooses, clients of
// this interface will receive notifications of progress and completion asynchronously.
// TransferAdapters support transfers in one direction; if an implementation
// provides support for upload and download, it should be instantiated twice,
// advertising support for each direction separately.
// Note that Adapter only implements the actual upload/download of content
// itself; organising the wider process including calling the API to get URLs,
// handling progress reporting and retries is the job of the core TransferQueue.
// This is so that the orchestration remains core & standard but Adapter
// can be changed to physically transfer to different hosts with less code.
type Adapter interface {
	// Name returns the name of this adapter, which is the same for all instances
	// of this type of adapter
	Name() string
	// Direction returns whether this instance is an upload or download instance
	// Adapter instances can only be one or the other, although the same
	// type may be instantiated for each direction
	Direction() Direction
	// Begin a new batch of uploads or downloads. Call this first, followed by one
	// or more Add calls. The passed in callback will receive updates on progress.
	Begin(cfg AdapterConfig, cb ProgressCallback) error
	// Add queues a download/upload, which will complete asynchronously and
	// notify the callbacks given to Begin()
	Add(transfers ...*Transfer) (results <-chan TransferResult)
	// Indicate that all transfers have been scheduled and resources can be released
	// once the queued items have completed.
	// This call blocks until all items have been processed
	End()
}

// Result of a transfer returned through CompletionChannel()
type TransferResult struct {
	Transfer *Transfer
	// This will be non-nil if there was an error transferring this item
	Error error
}