File: progress.go

package info (click to toggle)
golang-github-openshift-imagebuilder 1.2.15%2Bds1-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 1,776 kB
  • sloc: makefile: 18; sh: 3; ansic: 1
file content (315 lines) | stat: -rw-r--r-- 7,699 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
package imageprogress

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	"github.com/containers/storage/pkg/regexp"
)

const (
	defaultProgressTimeThreshhold = 30 * time.Second
	defaultStableThreshhold       = 10
)

// progressLine is a structure representation of a Docker pull progress line
type progressLine struct {
	ID     string          `json:"id"`
	Status string          `json:"status"`
	Detail *progressDetail `json:"progressDetail"`
	Error  string          `json:"error"`
}

// progressDetail is the progressDetail structure in a Docker pull progress line
type progressDetail struct {
	Current int64 `json:"current"`
	Total   int64 `json:"total"`
}

// layerDetail is layer information associated with a specific layerStatus
type layerDetail struct {
	Count   int
	Current int64
	Total   int64
}

// layerStatus is one of different possible status for layers detected by
// the ProgressWriter
type layerStatus int

const (
	statusPending layerStatus = iota
	statusDownloading
	statusExtracting
	statusComplete
	statusPushing
)

// layerStatusFromDockerString translates a string in a Docker status
// line to a layerStatus
func layerStatusFromDockerString(dockerStatus string) layerStatus {
	switch dockerStatus {
	case "Pushing":
		return statusPushing
	case "Downloading":
		return statusDownloading
	case "Extracting", "Verifying Checksum", "Download complete":
		return statusExtracting
	case "Pull complete", "Already exists", "Pushed", "Layer already exists":
		return statusComplete
	default:
		return statusPending
	}
}

type report map[layerStatus]*layerDetail

func (r report) count(status layerStatus) int {
	detail, ok := r[status]
	if !ok {
		return 0
	}
	return detail.Count
}

func (r report) percentProgress(status layerStatus) float32 {
	detail, ok := r[status]
	if !ok {
		return 0
	}
	if detail.Total == 0 {
		return 0
	}
	pct := float32(detail.Current) / float32(detail.Total) * 100.0
	if pct > 100.0 {
		pct = 100.0
	}
	if pct < 0.0 {
		pct = 0.0
	}
	return pct
}

func (r report) totalCount() int {
	cnt := 0
	for _, detail := range r {
		cnt += detail.Count
	}
	return cnt
}

// String is used for test output
func (r report) String() string {
	result := &bytes.Buffer{}
	fmt.Fprintf(result, "{")
	for k := range r {
		var status string
		switch k {
		case statusPending:
			status = "pending"
		case statusDownloading:
			status = "downloading"
		case statusExtracting:
			status = "extracting"
		case statusComplete:
			status = "complete"
		}
		fmt.Fprintf(result, "%s:{Count: %d, Current: %d, Total: %d}, ", status, r[k].Count, r[k].Current, r[k].Total)
	}
	fmt.Fprintf(result, "}")
	return result.String()
}

// newWriter creates a writer that periodically reports
// on pull/push progress of a Docker image. It only reports when the state of the
// different layers has changed and uses time thresholds to limit the
// rate of the reports.
func newWriter(reportFn func(report), layersChangedFn func(report, report) bool) io.WriteCloser {
	writer := &imageProgressWriter{
		mutex:                  &sync.Mutex{},
		layerStatus:            map[string]*progressLine{},
		reportFn:               reportFn,
		layersChangedFn:        layersChangedFn,
		progressTimeThreshhold: defaultProgressTimeThreshhold,
		stableThreshhold:       defaultStableThreshhold,
	}
	return writer
}

type imageProgressWriter struct {
	mutex                  *sync.Mutex
	internalWriter         io.WriteCloser
	readingGoroutineStatus <-chan error // Exists if internalWriter != nil
	layerStatus            map[string]*progressLine
	lastLayerCount         int
	stableLines            int
	stableThreshhold       int
	progressTimeThreshhold time.Duration
	lastReport             report
	lastReportTime         time.Time
	reportFn               func(report)
	layersChangedFn        func(report, report) bool
}

func (w *imageProgressWriter) Write(data []byte) (int, error) {
	w.mutex.Lock()
	defer w.mutex.Unlock()
	if w.internalWriter == nil {
		var pipeIn *io.PipeReader
		statusChannel := make(chan error, 1) // Buffered, so that sending a value after this or our caller has failed and exited does not block.
		pipeIn, w.internalWriter = io.Pipe()
		w.readingGoroutineStatus = statusChannel
		go w.readingGoroutine(statusChannel, pipeIn)
	}
	return w.internalWriter.Write(data)
}

func (w *imageProgressWriter) Close() error {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	if w.internalWriter == nil {
		return nil
	}

	err := w.internalWriter.Close() // As of Go 1.9 and 1.10, PipeWriter.Close() always returns nil
	readingErr := <-w.readingGoroutineStatus
	if err == nil && readingErr != nil {
		err = readingErr
	}
	return err
}

func (w *imageProgressWriter) readingGoroutine(statusChannel chan<- error, pipeIn *io.PipeReader) {
	err := errors.New("Internal error: unexpected panic in imageProgressWriter.readingGoroutine")
	defer func() { statusChannel <- err }()
	defer func() {
		if err != nil {
			pipeIn.CloseWithError(err)
		} else {
			pipeIn.Close()
		}
	}()

	err = w.readProgress(pipeIn)
	// err is nil on reaching EOF
}

func (w *imageProgressWriter) readProgress(pipeIn *io.PipeReader) error {
	decoder := json.NewDecoder(pipeIn)
	for {
		line := &progressLine{}
		err := decoder.Decode(line)
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		err = w.processLine(line)
		if err != nil {
			return err
		}
	}
	return nil
}

func (w *imageProgressWriter) processLine(line *progressLine) error {

	if err := getError(line); err != nil {
		return err
	}

	// determine if it's a line we want to process
	if !islayerStatus(line) {
		return nil
	}

	w.layerStatus[line.ID] = line

	// if the number of layers has not stabilized yet, return and wait for more
	// progress
	if !w.isStableLayerCount() {
		return nil
	}

	r := createReport(w.layerStatus)

	// check if the count of layers in each state has changed
	if w.layersChangedFn(w.lastReport, r) {
		w.lastReport = r
		w.lastReportTime = time.Now()
		w.reportFn(r)
		return nil
	}
	// If layer counts haven't changed, but enough time has passed (30 sec by default),
	// at least report on download/push progress
	if time.Since(w.lastReportTime) > w.progressTimeThreshhold {
		w.lastReport = r
		w.lastReportTime = time.Now()
		w.reportFn(r)
	}
	return nil
}

func (w *imageProgressWriter) isStableLayerCount() bool {
	// If the number of layers has changed since last status, we're not stable
	if w.lastLayerCount != len(w.layerStatus) {
		w.lastLayerCount = len(w.layerStatus)
		w.stableLines = 0
		return false
	}
	// Only proceed after we've received status for the same number
	// of layers at least stableThreshhold times. If not, they're still increasing
	w.stableLines++
	return w.stableLines >= w.stableThreshhold
}

var layerIDRegexp = regexp.Delayed("^[a-f0-9]*$")

func islayerStatus(line *progressLine) bool {
	// ignore status lines with no layer id
	if len(line.ID) == 0 {
		return false
	}
	// ignore layer ids that are not hex string
	if !layerIDRegexp.MatchString(line.ID) {
		return false
	}
	// ignore retrying status
	if strings.HasPrefix(line.Status, "Retrying") {
		return false
	}
	return true
}

func getError(line *progressLine) error {
	if len(line.Error) > 0 {
		return errors.New(line.Error)
	}
	return nil
}

func createReport(dockerProgress map[string]*progressLine) report {
	r := report{}
	for _, line := range dockerProgress {
		layerStatus := layerStatusFromDockerString(line.Status)
		detail, exists := r[layerStatus]
		if !exists {
			detail = &layerDetail{}
			r[layerStatus] = detail
		}
		detail.Count++
		if line.Detail != nil {
			detail.Current += line.Detail.Current
			detail.Total += line.Detail.Total
		}
	}
	return r
}