File: pwalk_test.go

package info (click to toggle)
golang-github-opencontainers-selinux 1.11.1-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 280 kB
  • sloc: makefile: 31
file content (241 lines) | stat: -rw-r--r-- 5,734 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
package pwalk

import (
	"errors"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"sync/atomic"
	"testing"
	"time"
)

func TestWalk(t *testing.T) {
	var count uint32
	concurrency := runtime.NumCPU() * 2

	dir, total := prepareTestSet(t, 3, 2, 1)

	err := WalkN(dir,
		func(_ string, _ os.FileInfo, _ error) error {
			atomic.AddUint32(&count, 1)
			return nil
		},
		concurrency)
	if err != nil {
		t.Errorf("Walk failed: %v", err)
	}
	if count != uint32(total) {
		t.Errorf("File count mismatch: found %d, expected %d", count, total)
	}

	t.Logf("concurrency: %d, files found: %d", concurrency, count)
}

func TestWalkTopLevelErrNotExistNotIgnored(t *testing.T) {
	if WalkN("non-existent-directory", cbEmpty, 8) == nil {
		t.Fatal("expected ErrNotExist, got nil")
	}
}

// https://github.com/opencontainers/selinux/issues/199
func TestWalkRaceWithRemoval(t *testing.T) {
	var count uint32
	concurrency := runtime.NumCPU() * 2
	// This test is still on a best-effort basis, meaning it can still pass
	// when there is a bug in the code, but the larger the test set is, the
	// higher the probability that this test fails (without a fix).
	//
	// With this set (4, 5, 6), and the fix commented out, it fails
	// 100 out of 100 runs on my machine.
	dir, total := prepareTestSet(t, 4, 5, 6)

	// Race walk with removal.
	go os.RemoveAll(dir)
	err := WalkN(dir,
		func(_ string, _ os.FileInfo, _ error) error {
			atomic.AddUint32(&count, 1)
			return nil
		},
		concurrency)
	t.Logf("found %d of %d files", count, total)
	if err != nil {
		t.Fatalf("expected nil, got %v", err)
	}
}

func TestWalkDirManyErrors(t *testing.T) {
	var count uint32

	dir, total := prepareTestSet(t, 3, 3, 2)

	max := uint32(total / 2)
	e42 := errors.New("42")
	err := Walk(dir,
		func(_ string, _ os.FileInfo, _ error) error {
			if atomic.AddUint32(&count, 1) > max {
				return e42
			}
			return nil
		})
	t.Logf("found %d of %d files", count, total)

	if err == nil {
		t.Errorf("Walk succeeded, but error is expected")
		if count != uint32(total) {
			t.Errorf("File count mismatch: found %d, expected %d", count, total)
		}
	}
}

func makeManyDirs(prefix string, levels, dirs, files int) (count int, err error) {
	for d := 0; d < dirs; d++ {
		var dir string
		dir, err = os.MkdirTemp(prefix, "d-")
		if err != nil {
			return
		}
		count++
		for f := 0; f < files; f++ {
			var fi *os.File
			fi, err = os.CreateTemp(dir, "f-")
			if err != nil {
				return count, err
			}
			_ = fi.Close()
			count++
		}
		if levels == 0 {
			continue
		}
		var c int
		if c, err = makeManyDirs(dir, levels-1, dirs, files); err != nil {
			return
		}
		count += c
	}

	return
}

// prepareTestSet() creates a directory tree of shallow files,
// to be used for testing or benchmarking.
//
// Total dirs: dirs^levels + dirs^(levels-1) + ... + dirs^1
// Total files: total_dirs * files
func prepareTestSet(tb testing.TB, levels, dirs, files int) (dir string, total int) {
	tb.Helper()
	var err error

	dir, err = os.MkdirTemp(".", "pwalk-test-")
	if err != nil {
		tb.Fatal(err)
	}
	tb.Cleanup(func() {
		if err := os.RemoveAll(dir); err != nil && !errors.Is(err, os.ErrNotExist) {
			tb.Errorf("cleanup error: %v", err)
		}
	})
	total, err = makeManyDirs(dir, levels, dirs, files)
	if err != nil {
		tb.Fatal(err)
	}
	total++ // this dir

	return
}

type walkerFunc func(root string, walkFn WalkFunc) error

func genWalkN(n int) walkerFunc {
	return func(root string, walkFn WalkFunc) error {
		return WalkN(root, walkFn, n)
	}
}

func BenchmarkWalk(b *testing.B) {
	const (
		levels = 5 // how deep
		dirs   = 3 // dirs on each levels
		files  = 8 // files on each levels
	)

	benchmarks := []struct {
		walk filepath.WalkFunc
		name string
	}{
		{name: "Empty", walk: cbEmpty},
		{name: "ReadFile", walk: cbReadFile},
		{name: "ChownChmod", walk: cbChownChmod},
		{name: "RandomSleep", walk: cbRandomSleep},
	}

	walkers := []struct {
		walker walkerFunc
		name   string
	}{
		{name: "filepath.Walk", walker: filepath.Walk},
		{name: "pwalk.Walk", walker: Walk},
		// test WalkN with various values of N
		{name: "pwalk.Walk1", walker: genWalkN(1)},
		{name: "pwalk.Walk2", walker: genWalkN(2)},
		{name: "pwalk.Walk4", walker: genWalkN(4)},
		{name: "pwalk.Walk8", walker: genWalkN(8)},
		{name: "pwalk.Walk16", walker: genWalkN(16)},
		{name: "pwalk.Walk32", walker: genWalkN(32)},
		{name: "pwalk.Walk64", walker: genWalkN(64)},
		{name: "pwalk.Walk128", walker: genWalkN(128)},
		{name: "pwalk.Walk256", walker: genWalkN(256)},
	}

	dir, total := prepareTestSet(b, levels, dirs, files)
	b.Logf("dataset: %d levels x %d dirs x %d files, total entries: %d", levels, dirs, files, total)

	for _, bm := range benchmarks {
		for _, w := range walkers {
			walker := w.walker
			walkFn := bm.walk
			// preheat
			if err := w.walker(dir, bm.walk); err != nil {
				b.Errorf("walk failed: %v", err)
			}
			// benchmark
			b.Run(bm.name+"/"+w.name, func(b *testing.B) {
				for i := 0; i < b.N; i++ {
					if err := walker(dir, walkFn); err != nil {
						b.Errorf("walk failed: %v", err)
					}
				}
			})
		}
	}
}

func cbEmpty(_ string, _ os.FileInfo, _ error) error {
	return nil
}

func cbChownChmod(path string, info os.FileInfo, _ error) error {
	_ = os.Chown(path, 0, 0)
	mode := os.FileMode(0o644)
	if info.Mode().IsDir() {
		mode = os.FileMode(0o755)
	}
	_ = os.Chmod(path, mode)

	return nil
}

func cbReadFile(path string, info os.FileInfo, _ error) error {
	var err error
	if info.Mode().IsRegular() {
		_, err = os.ReadFile(path)
	}
	return err
}

func cbRandomSleep(_ string, _ os.FileInfo, _ error) error {
	time.Sleep(time.Duration(rand.Intn(500)) * time.Microsecond) //nolint:gosec // ignore G404: Use of weak random number generator
	return nil
}