File: squashfs.go

package info (click to toggle)
singularity-container 4.0.3%2Bds1-1
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 21,672 kB
  • sloc: asm: 3,857; sh: 2,125; ansic: 1,677; awk: 414; makefile: 110; python: 99
file content (302 lines) | stat: -rw-r--r-- 7,686 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
// Copyright 2023 Sylabs Inc. All rights reserved.
//
// SPDX-License-Identifier: Apache-2.0

package mutate

import (
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"sync"

	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/types"
)

const layerMediaType types.MediaType = "application/vnd.sylabs.image.layer.v1.squashfs"

type squashfsConverter struct {
	converter       string   // Path to converter program.
	args            []string // Arguments required for converter program.
	dir             string   // Working directory.
	convertWhiteout bool     // Convert whiteout markers from AUFS -> OverlayFS
}

// SquashfsConverterOpt are used to specify squashfs converter options.
type SquashfsConverterOpt func(*squashfsConverter) error

// OptSquashfsLayerConverter specifies the converter program to use when converting from TAR to
// SquashFS format.
func OptSquashfsLayerConverter(converter string) SquashfsConverterOpt {
	return func(c *squashfsConverter) error {
		path, err := exec.LookPath(converter)
		if err != nil {
			return err
		}

		c.converter = path

		return nil
	}
}

var errSquashfsConverterNotSupported = errors.New("squashfs converter not supported")

// OptSquashfsSkipWhiteoutConversion is set to skip the default conversion of whiteout /
// opaque markers from AUFS to OverlayFS format.
func OptSquashfsSkipWhiteoutConversion(b bool) SquashfsConverterOpt {
	return func(c *squashfsConverter) error {
		c.convertWhiteout = !b
		return nil
	}
}

// SquashfsLayer converts the base layer into a layer using the squashfs format. A dir must be
// specified, which is used as a working directory during conversion. The caller is responsible for
// cleaning up dir.
//
// By default, this will attempt to locate a suitable TAR to SquashFS converter such as 'tar2sqfs'
// or `sqfstar` via exec.LookPath. To specify a path to a specific converter program, consider
// using OptSquashfsLayerConverter.
//
// By default, AUFS whiteout markers in the base TAR layer will be converted to OverlayFS whiteout
// markers in the SquashFS layer. This can be disabled, e.g. where it is known that the layer is
// part of a squashed image that will not have any whiteouts, using OptSquashfsSkipWhiteoutConversion.
//
// Note - when whiteout conversion is performed the base layer will be read twice. Callers should
// ensure it is cached, and is not a streaming layer.
func SquashfsLayer(base v1.Layer, dir string, opts ...SquashfsConverterOpt) (v1.Layer, error) {
	c := squashfsConverter{
		dir:             dir,
		convertWhiteout: true,
	}

	for _, opt := range opts {
		if err := opt(&c); err != nil {
			return nil, err
		}
	}

	if c.converter == "" {
		path, err := exec.LookPath("tar2sqfs")
		if err != nil {
			if path, err = exec.LookPath("sqfstar"); err != nil {
				return nil, err
			}
		}

		c.converter = path
	}

	switch base := filepath.Base(c.converter); base {
	case "tar2sqfs":
		// Use gzip compression instead of the default (xz).
		c.args = []string{
			"--compressor", "gzip",
		}

	case "sqfstar":
		// The `sqfstar` binary by default creates a root directory that is owned by the
		// uid/gid of the user running it, and uses the current time for the root directory
		// inode as well as the modification_time field of the superblock.
		//
		// The options below modify this behaviour to instead use predictable values, but
		// unfortunately they do not function correctly with squashfs-tools v4.5.
		c.args = []string{
			"-mkfs-time", "0",
			"-root-time", "0",
			"-root-uid", "0",
			"-root-gid", "0",
			"-root-mode", "0755",
		}

	default:
		return nil, fmt.Errorf("%v: %w", base, errSquashfsConverterNotSupported)
	}

	return c.layer(base)
}

// makeSquashfs returns the path to a squashfs file that contains the contents of the uncompressed
// TAR stream from r.
func (c *squashfsConverter) makeSquashfs(r io.Reader) (string, error) {
	dir, err := os.MkdirTemp(c.dir, "")
	if err != nil {
		return "", err
	}

	path := filepath.Join(dir, "layer.sqfs")

	//nolint:gosec // Arguments are created programatically.
	cmd := exec.Command(c.converter, append(c.args, path)...)
	cmd.Stdin = r

	if out, err := cmd.CombinedOutput(); err != nil {
		return "", fmt.Errorf("%s error: %w, output: %s", c.converter, err, out)
	}

	return path, nil
}

// Uncompressed returns an io.ReadCloser for the uncompressed layer contents. If
// c.convertWhiteout is true it will convert whiteout markers from AUFS ->
// OverlayFS format. Note that when conversion is performed, the underlying
// layer TAR is read twice.
func (c *squashfsConverter) Uncompressed(l v1.Layer) (io.ReadCloser, error) {
	rc, err := l.Uncompressed()
	if err != nil {
		return nil, err
	}

	// No conversion - direct tar stream from the layer.
	if !c.convertWhiteout {
		return rc, nil
	}

	// Conversion - first, scan for opaque directories and presence of file
	// whiteout markers.
	opaquePaths, fileWhiteout, err := scanAUFSWhiteouts(rc)
	if err != nil {
		return nil, err
	}
	rc.Close()

	rc, err = l.Uncompressed()
	if err != nil {
		return nil, err
	}

	// Nothing found to filter
	if len(opaquePaths) == 0 && !fileWhiteout {
		return rc, nil
	}

	pr, pw := io.Pipe()
	go func() {
		defer rc.Close()
		pw.CloseWithError(whiteoutFilter(rc, pw, opaquePaths))
	}()
	return pr, nil
}

type squashfsLayer struct {
	base      v1.Layer
	converter *squashfsConverter

	computed bool
	path     string
	hash     v1.Hash
	size     int64

	sync.Mutex
}

var errUnsupportedLayerType = errors.New("unsupported layer type")

// layer converts base to squashfs format.
func (c *squashfsConverter) layer(base v1.Layer) (v1.Layer, error) {
	mt, err := base.MediaType()
	if err != nil {
		return nil, err
	}

	//nolint:exhaustive // Exhaustive cases not appropriate.
	switch mt {
	case layerMediaType:
		return base, nil

	case types.DockerLayer, types.DockerUncompressedLayer, types.OCILayer, types.OCIUncompressedLayer:
		return &squashfsLayer{
			base:      base,
			converter: c,
		}, nil

	default:
		return nil, fmt.Errorf("%w: %v", errUnsupportedLayerType, mt)
	}
}

// populate populates various fields in l.
func (l *squashfsLayer) populate() error {
	l.Lock()
	defer l.Unlock()

	if l.computed {
		return nil
	}

	rc, err := l.converter.Uncompressed(l.base)
	if err != nil {
		return err
	}
	defer rc.Close()

	path, err := l.converter.makeSquashfs(rc)
	if err != nil {
		return err
	}

	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	h, n, err := v1.SHA256(f)
	if err != nil {
		return err
	}

	l.computed = true
	l.path = path
	l.hash = h
	l.size = n

	return nil
}

// Digest returns the Hash of the compressed layer.
func (l *squashfsLayer) Digest() (v1.Hash, error) {
	return l.DiffID()
}

// DiffID returns the Hash of the uncompressed layer.
func (l *squashfsLayer) DiffID() (v1.Hash, error) {
	if err := l.populate(); err != nil {
		return v1.Hash{}, err
	}

	return l.hash, nil
}

// Compressed returns an io.ReadCloser for the compressed layer contents.
func (l *squashfsLayer) Compressed() (io.ReadCloser, error) {
	return l.Uncompressed()
}

// Uncompressed returns an io.ReadCloser for the uncompressed layer contents.
func (l *squashfsLayer) Uncompressed() (io.ReadCloser, error) {
	if err := l.populate(); err != nil {
		return nil, err
	}

	return os.Open(l.path)
}

// Size returns the compressed size of the Layer.
func (l *squashfsLayer) Size() (int64, error) {
	if err := l.populate(); err != nil {
		return 0, err
	}

	return l.size, nil
}

// MediaType returns the media type of the Layer.
func (l *squashfsLayer) MediaType() (types.MediaType, error) {
	return layerMediaType, nil
}