File: cache.go

package info (click to toggle)
snapd 2.71-3
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 79,536 kB
  • sloc: ansic: 16,114; sh: 16,105; python: 9,941; makefile: 1,890; exp: 190; awk: 40; xml: 22
file content (227 lines) | stat: -rw-r--r-- 6,643 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
// -*- Mode: Go; indent-tabs-mode: t -*-

/*
 * Copyright (C) 2017 Canonical Ltd
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 3 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package store

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"syscall"
	"time"

	"github.com/snapcore/snapd/logger"
	"github.com/snapcore/snapd/osutil"
)

// overridden in the unit tests
var osRemove = os.Remove

// downloadCache is the interface that a store download cache must provide
type downloadCache interface {
	// Get retrieves the given cacheKey content and puts it into targetPath. Returns
	// true if a cached file was moved to targetPath or if one was already there.
	Get(cacheKey, targetPath string) bool
	// Put adds a new file to the cache
	Put(cacheKey, sourcePath string) error
	// Get full path of the file in cache
	GetPath(cacheKey string) string
}

// nullCache is cache that does not cache
type nullCache struct{}

func (cm *nullCache) Get(cacheKey, targetPath string) bool {
	return false
}
func (cm *nullCache) GetPath(cacheKey string) string {
	return ""
}
func (cm *nullCache) Put(cacheKey, sourcePath string) error { return nil }

// changesByMtime sorts by the mtime of files
type changesByMtime []os.FileInfo

func (s changesByMtime) Len() int           { return len(s) }
func (s changesByMtime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s changesByMtime) Less(i, j int) bool { return s[i].ModTime().Before(s[j].ModTime()) }

// cacheManager implements a downloadCache via content based hard linking
type CacheManager struct {
	cacheDir string
	maxItems int
}

// NewCacheManager returns a new CacheManager with the given cacheDir
// and the given maximum amount of items. The idea behind it is the
// following algorithm:
//
//  1. When starting a download, check if it exists in $cacheDir
//  2. If found, update its mtime, hardlink into target location, and
//     return success
//  3. If not found, download the snap
//  4. On success, hardlink into $cacheDir/<digest>
//  5. If cache dir has more than maxItems entries, remove oldest mtimes
//     until it has maxItems
//
// The caching part is done here, the downloading happens in the store.go
// code.
func NewCacheManager(cacheDir string, maxItems int) *CacheManager {
	return &CacheManager{
		cacheDir: cacheDir,
		maxItems: maxItems,
	}
}

// GetPath returns the full path of the given content in the cache
// or empty string
func (cm *CacheManager) GetPath(cacheKey string) string {
	if _, err := os.Stat(cm.path(cacheKey)); os.IsNotExist(err) {
		return ""
	}
	return cm.path(cacheKey)
}

// Get retrieves the given cacheKey content and puts it into targetPath. Returns
// true if a cached file was moved to targetPath or if one was already there.
func (cm *CacheManager) Get(cacheKey, targetPath string) bool {
	if err := os.Link(cm.path(cacheKey), targetPath); err != nil && !errors.Is(err, os.ErrExist) {
		return false
	}

	logger.Debugf("using cache for %s", targetPath)
	now := time.Now()
	// the modification time is updated on a best-effort basis
	_ = os.Chtimes(targetPath, now, now)
	return true
}

// Put adds a new file to the cache with the given cacheKey
func (cm *CacheManager) Put(cacheKey, sourcePath string) error {
	// always try to create the cache dir first or the following
	// osutil.IsWritable will always fail if the dir is missing
	_ = os.MkdirAll(cm.cacheDir, 0700)

	// happens on e.g. `snap download` which runs as the user
	if !osutil.IsWritable(cm.cacheDir) {
		return nil
	}

	err := os.Link(sourcePath, cm.path(cacheKey))
	if os.IsExist(err) {
		now := time.Now()
		err := os.Chtimes(cm.path(cacheKey), now, now)
		// this can happen if a cleanup happens in parallel, ie.
		// the file was there but cleanup() removed it between
		// the os.Link/os.Chtimes - no biggie, just link it again
		if os.IsNotExist(err) {
			return os.Link(sourcePath, cm.path(cacheKey))
		}
		return err
	}
	if err != nil {
		return err
	}
	return cm.cleanup()
}

// count returns the number of items in the cache
func (cm *CacheManager) count() int {
	// TODO: Use something more effective than a list of all entries
	//       here. This will waste a lot of memory on large dirs.
	if l, err := os.ReadDir(cm.cacheDir); err == nil {
		return len(l)
	}
	return 0
}

// path returns the full path of the given content in the cache
func (cm *CacheManager) path(cacheKey string) string {
	return filepath.Join(cm.cacheDir, cacheKey)
}

// cleanup ensures that only maxItems are stored in the cache
func (cm *CacheManager) cleanup() error {
	entries, err := os.ReadDir(cm.cacheDir)
	if err != nil {
		return err
	}

	if len(entries) <= cm.maxItems {
		return nil
	}

	// most of the entries will have more than one hardlink, but a minority may
	// be referenced only the cache and thus be a candidate for pruning
	pruneCandidates := make([]os.FileInfo, 0, len(entries)/5)

	for _, entry := range entries {
		fi, err := entry.Info()
		if err != nil {
			return err
		}

		n, err := hardLinkCount(fi)
		if err != nil {
			logger.Noticef("cannot inspect cache: %s", err)
		}
		// If the file is referenced in the filesystem somewhere else our copy
		// is "free" so skip it.
		if n <= 1 {
			pruneCandidates = append(pruneCandidates, fi)
		}
	}

	if len(pruneCandidates) <= cm.maxItems {
		// nothing to prune
		return nil
	}

	var lastErr error
	sort.Sort(changesByMtime(pruneCandidates))
	numOwned := len(pruneCandidates)
	deleted := 0
	for _, fi := range pruneCandidates {
		path := cm.path(fi.Name())
		if err := osRemove(path); err != nil {
			if !os.IsNotExist(err) {
				// If there is any error we cleanup the file (it is just a cache
				// after all).
				logger.Noticef("cannot cleanup cache: %s", err)
				lastErr = err
			}
			continue
		}
		deleted++
		if numOwned-deleted <= cm.maxItems {
			break
		}
	}
	return lastErr
}

// hardLinkCount returns the number of hardlinks for the given path
func hardLinkCount(fi os.FileInfo) (uint64, error) {
	if stat, ok := fi.Sys().(*syscall.Stat_t); ok && stat != nil {
		return uint64(stat.Nlink), nil
	}
	return 0, fmt.Errorf("internal error: cannot read hardlink count from %s", fi.Name())
}