File: cache.go

package info (click to toggle)
singularity-container 4.0.3%2Bds1-1
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 21,672 kB
  • sloc: asm: 3,857; sh: 2,125; ansic: 1,677; awk: 414; makefile: 110; python: 99
file content (333 lines) | stat: -rw-r--r-- 9,778 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
// Copyright (c) 2020, Sylabs Inc. All rights reserved.
// This software is licensed under a 3-clause BSD license. Please consult the
// LICENSE.md file distributed with the sources of this project regarding your
// rights to use or distribute this software.

package cache

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"net/http"
	"net/http/httptest"
	"path"
	"path/filepath"
	"testing"

	"github.com/sylabs/singularity/v4/e2e/internal/e2e"
	"github.com/sylabs/singularity/v4/e2e/internal/testhelper"
	"github.com/sylabs/singularity/v4/internal/pkg/cache"
	"github.com/sylabs/singularity/v4/internal/pkg/util/fs"
)

type cacheTests struct {
	env e2e.TestEnv
}

const imgName = "alpine_latest.sif"

func prepTest(t *testing.T, testEnv e2e.TestEnv, testName string, cacheParentDir string, imagePath string) (imageURL string, cleanup func()) {
	// We will pull images from a temporary http server
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, testEnv.ImagePath)
	}))
	cleanup = srv.Close

	// If the test imageFile is already present check it's not also in the cache
	// at the start of our test - we expect to pull it again and then see it
	// appear in the cache.
	if fs.IsFile(imagePath) {
		ensureNotCached(t, testName, srv.URL, cacheParentDir)
	}

	testEnv.UnprivCacheDir = cacheParentDir
	testEnv.RunSingularity(
		t,
		e2e.WithProfile(e2e.UserProfile),
		e2e.WithCommand("pull"),
		e2e.WithArgs([]string{"--force", imagePath, srv.URL}...),
		e2e.ExpectExit(0),
	)

	ensureCached(t, testName, srv.URL, cacheParentDir)
	return srv.URL, cleanup
}

func (c cacheTests) testNoninteractiveCacheCmds(t *testing.T) {
	tests := []struct {
		name               string
		options            []string
		needImage          bool
		expectedEmptyCache bool
		expectedOutput     string
		exit               int
	}{
		{
			name:               "clean force",
			options:            []string{"clean", "--force"},
			expectedOutput:     "",
			needImage:          true,
			expectedEmptyCache: true,
			exit:               0,
		},
		{
			name:               "clean force days beyond age",
			options:            []string{"clean", "--force", "--days", "30"},
			expectedOutput:     "",
			needImage:          true,
			expectedEmptyCache: false,
			exit:               0,
		},
		{
			name:               "clean force days within age",
			options:            []string{"clean", "--force", "--days", "0"},
			expectedOutput:     "",
			needImage:          true,
			expectedEmptyCache: true,
			exit:               0,
		},
		{
			name:           "clean help",
			options:        []string{"clean", "--help"},
			expectedOutput: "Clean your local Singularity cache",
			needImage:      false,
			exit:           0,
		},
		{
			name:           "list help",
			options:        []string{"list", "--help"},
			expectedOutput: "List your local Singularity cache",
			needImage:      false,
			exit:           0,
		},
		{
			name:               "list type",
			options:            []string{"list", "--type", "net"},
			needImage:          true,
			expectedOutput:     "There are 1 container file",
			expectedEmptyCache: false,
			exit:               0,
		},
		{
			name:               "list verbose",
			needImage:          true,
			options:            []string{"list", "--verbose"},
			expectedOutput:     "NAME",
			expectedEmptyCache: false,
			exit:               0,
		},
	}
	// A directory where we store the image and used by separate commands
	tempDir, imgStoreCleanup := e2e.MakeTempDir(t, "", "", "image store")
	defer imgStoreCleanup(t)
	imagePath := filepath.Join(tempDir, imgName)

	for _, tt := range tests {
		// Each test get its own clean cache directory
		cacheDir, cleanup := e2e.MakeCacheDir(t, "")
		defer cleanup(t)
		_, err := cache.New(cache.Config{ParentDir: cacheDir})
		if err != nil {
			t.Fatalf("Could not create image cache handle: %v", err)
		}

		imageURL := ""
		if tt.needImage {
			var srvCleanup func()
			imageURL, srvCleanup = prepTest(t, c.env, tt.name, cacheDir, imagePath)
			defer srvCleanup()
		}

		c.env.UnprivCacheDir = cacheDir
		c.env.RunSingularity(
			t,
			e2e.AsSubtest(tt.name),
			e2e.WithProfile(e2e.UserProfile),
			e2e.WithCommand("cache"),
			e2e.WithArgs(tt.options...),
			e2e.ExpectExit(tt.exit),
		)

		if tt.needImage && tt.expectedEmptyCache {
			ensureNotCached(t, tt.name, imageURL, cacheDir)
		}
	}
}

func (c cacheTests) testInteractiveCacheCmds(t *testing.T) {
	tt := []struct {
		name               string
		options            []string
		expect             string
		send               string
		exit               int
		expectedEmptyCache bool // Is the cache supposed to be empty after the command is executed
	}{
		{
			name:               "clean normal confirmed",
			options:            []string{"clean"},
			expect:             "Do you want to continue? [y/N]",
			send:               "y",
			expectedEmptyCache: true,
			exit:               0,
		},
		{
			name:               "clean normal not confirmed",
			options:            []string{"clean"},
			expect:             "Do you want to continue? [y/N]",
			send:               "n",
			expectedEmptyCache: false,
			exit:               0,
		},
		{
			name:               "clean normal force",
			options:            []string{"clean", "--force"},
			expectedEmptyCache: true,
			exit:               0,
		},
		{
			name:               "clean dry-run confirmed",
			options:            []string{"clean", "--dry-run"},
			expectedEmptyCache: false,
			exit:               0,
		},
		{
			name:               "clean type confirmed",
			options:            []string{"clean", "--type", "net"},
			expect:             "Do you want to continue? [y/N]",
			send:               "y",
			expectedEmptyCache: true,
			exit:               0,
		},
		{
			name:               "clean type not confirmed",
			options:            []string{"clean", "--type", "net"},
			expect:             "Do you want to continue? [y/N]",
			send:               "n",
			expectedEmptyCache: false,
			exit:               0,
		},
		{
			name:               "clean days beyond age",
			options:            []string{"clean", "--days", "30"},
			expect:             "Do you want to continue? [y/N]",
			send:               "y",
			expectedEmptyCache: false,
			exit:               0,
		},
		{
			name:               "clean days within age",
			options:            []string{"clean", "--days", "0"},
			expect:             "Do you want to continue? [y/N]",
			send:               "y",
			expectedEmptyCache: true,
			exit:               0,
		},
	}

	// A directory where we store the image and used by separate commands
	tempDir, imgStoreCleanup := e2e.MakeTempDir(t, "", "", "image store")
	defer imgStoreCleanup(t)
	imagePath := filepath.Join(tempDir, imgName)

	for _, tc := range tt {
		// Each test get its own clean cache directory
		cacheDir, cleanup := e2e.MakeCacheDir(t, "")
		defer cleanup(t)
		_, err := cache.New(cache.Config{ParentDir: cacheDir})
		if err != nil {
			t.Fatalf("Could not create image cache handle: %v", err)
		}

		c.env.UnprivCacheDir = cacheDir
		imageURL, srvCleanup := prepTest(t, c.env, tc.name, cacheDir, imagePath)
		defer srvCleanup()

		c.env.RunSingularity(
			t,
			e2e.AsSubtest(tc.name),
			e2e.WithProfile(e2e.UserProfile),
			e2e.WithCommand("cache"),
			e2e.WithArgs(tc.options...),
			e2e.ConsoleRun(
				e2e.ConsoleExpect(tc.expect),
				e2e.ConsoleSendLine(tc.send),
			),
			e2e.ExpectExit(tc.exit),
		)

		// Check the content of the cache
		if tc.expectedEmptyCache {
			ensureNotCached(t, tc.name, imageURL, cacheDir)
		} else {
			ensureCached(t, tc.name, imageURL, cacheDir)
		}
	}
}

// ensureNotCached checks the entry related to an image is not in the cache
func ensureNotCached(t *testing.T, testName string, imageURL string, cacheParentDir string) {
	shasum, err := netHash(imageURL)
	if err != nil {
		t.Fatalf("couldn't compute hash of image %s: %v", imageURL, err)
	}

	// Where the cached image should be
	cacheImagePath := path.Join(cacheParentDir, "cache", "net", shasum)

	// The image file shouldn't be present
	if e2e.PathExists(t, cacheImagePath) {
		t.Fatalf("%s failed: %s is still in the cache (%s)", testName, imgName, cacheImagePath)
	}
}

// ensureCached checks the entry related to an image is really in the cache
func ensureCached(t *testing.T, testName string, imageURL string, cacheParentDir string) {
	shasum, err := netHash(imageURL)
	if err != nil {
		t.Fatalf("couldn't compute hash of image %s: %v", imageURL, err)
	}

	// Where the cached image should be
	cacheImagePath := path.Join(cacheParentDir, "cache", "net", shasum)

	// The image file shouldn't be present
	if !e2e.PathExists(t, cacheImagePath) {
		t.Fatalf("%s failed: %s is not in the cache (%s)", testName, imgName, cacheImagePath)
	}
}

// netHash computes the expected cache hash for the image at url
func netHash(url string) (hash string, err error) {
	req, err := http.NewRequest("HEAD", url, nil)
	if err != nil {
		return "", fmt.Errorf("error constructing http request: %w", err)
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", fmt.Errorf("error making http request: %w", err)
	}
	defer res.Body.Close()

	headerDate := res.Header.Get("Last-Modified")
	h := sha256.New()
	h.Write([]byte(url + headerDate))
	return hex.EncodeToString(h.Sum(nil)), nil
}

// E2ETests is the main func to trigger the test suite
func E2ETests(env e2e.TestEnv) testhelper.Tests {
	c := cacheTests{
		env: env,
	}

	np := testhelper.NoParallel

	return testhelper.Tests{
		"interactive commands":     np(c.testInteractiveCacheCmds),
		"non-interactive commands": np(c.testNoninteractiveCacheCmds),
		"issue5097":                np(c.issue5097),
		"issue5350":                np(c.issue5350),
	}
}