File: makefiles.go

package info (click to toggle)
rclone 1.60.1%2Bdfsg-2
  • links: PTS, VCS
  • area: main
  • in suites: bookworm
  • size: 34,820 kB
  • sloc: sh: 957; xml: 857; python: 655; javascript: 612; makefile: 264; ansic: 101; php: 74
file content (293 lines) | stat: -rw-r--r-- 8,625 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
// Package makefiles builds a directory structure with the required
// number of files in of the required size.
package makefiles

import (
	"io"
	"log"
	"math"
	"math/rand"
	"os"
	"path/filepath"
	"time"

	"github.com/rclone/rclone/cmd"
	"github.com/rclone/rclone/cmd/test"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/config/flags"
	"github.com/rclone/rclone/lib/file"
	"github.com/rclone/rclone/lib/random"
	"github.com/rclone/rclone/lib/readers"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

var (
	// Flags
	numberOfFiles            = 1000
	averageFilesPerDirectory = 10
	maxDepth                 = 10
	minFileSize              = fs.SizeSuffix(0)
	maxFileSize              = fs.SizeSuffix(100)
	minFileNameLength        = 4
	maxFileNameLength        = 12
	seed                     = int64(1)
	zero                     = false
	sparse                   = false
	ascii                    = false
	pattern                  = false
	chargen                  = false

	// Globals
	randSource          *rand.Rand
	source              io.Reader
	directoriesToCreate int
	totalDirectories    int
	fileNames           = map[string]struct{}{} // keep a note of which file name we've used already
)

func init() {
	test.Command.AddCommand(makefilesCmd)
	makefilesFlags := makefilesCmd.Flags()
	flags.IntVarP(makefilesFlags, &numberOfFiles, "files", "", numberOfFiles, "Number of files to create")
	flags.IntVarP(makefilesFlags, &averageFilesPerDirectory, "files-per-directory", "", averageFilesPerDirectory, "Average number of files per directory")
	flags.IntVarP(makefilesFlags, &maxDepth, "max-depth", "", maxDepth, "Maximum depth of directory hierarchy")
	flags.FVarP(makefilesFlags, &minFileSize, "min-file-size", "", "Minimum size of file to create")
	flags.FVarP(makefilesFlags, &maxFileSize, "max-file-size", "", "Maximum size of files to create")
	flags.IntVarP(makefilesFlags, &minFileNameLength, "min-name-length", "", minFileNameLength, "Minimum size of file names")
	flags.IntVarP(makefilesFlags, &maxFileNameLength, "max-name-length", "", maxFileNameLength, "Maximum size of file names")

	test.Command.AddCommand(makefileCmd)
	makefileFlags := makefileCmd.Flags()

	// Common flags to makefiles and makefile
	for _, f := range []*pflag.FlagSet{makefilesFlags, makefileFlags} {
		flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)")
		flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00")
		flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)")
		flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only")
		flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern")
		flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern")
	}
}

var makefilesCmd = &cobra.Command{
	Use:   "makefiles <dir>",
	Short: `Make a random file hierarchy in a directory`,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1, command, args)
		commonInit()
		outputDirectory := args[0]
		directoriesToCreate = numberOfFiles / averageFilesPerDirectory
		averageSize := (minFileSize + maxFileSize) / 2
		start := time.Now()
		fs.Logf(nil, "Creating %d files of average size %v in %d directories in %q.", numberOfFiles, averageSize, directoriesToCreate, outputDirectory)
		root := &dir{name: outputDirectory, depth: 1}
		for totalDirectories < directoriesToCreate {
			root.createDirectories()
		}
		dirs := root.list("", []string{})
		totalBytes := int64(0)
		for i := 0; i < numberOfFiles; i++ {
			dir := dirs[randSource.Intn(len(dirs))]
			size := int64(minFileSize)
			if maxFileSize > minFileSize {
				size += randSource.Int63n(int64(maxFileSize - minFileSize))
			}
			writeFile(dir, fileName(), size)
			totalBytes += size
		}
		dt := time.Since(start)
		fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
	},
}

var makefileCmd = &cobra.Command{
	Use:   "makefile <size> [<file>]+ [flags]",
	Short: `Make files with random contents of the size given`,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1e6, command, args)
		commonInit()
		var size fs.SizeSuffix
		err := size.Set(args[0])
		if err != nil {
			log.Fatalf("Failed to parse size %q: %v", args[0], err)
		}
		start := time.Now()
		fs.Logf(nil, "Creating %d files of size %v.", len(args[1:]), size)
		totalBytes := int64(0)
		for _, filePath := range args[1:] {
			dir := filepath.Dir(filePath)
			name := filepath.Base(filePath)
			writeFile(dir, name, int64(size))
			totalBytes += int64(size)
		}
		dt := time.Since(start)
		fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
	},
}

func bool2int(b bool) int {
	if b {
		return 1
	}
	return 0
}

// common initialisation for makefiles and makefile
func commonInit() {
	if seed == 0 {
		seed = time.Now().UnixNano()
		fs.Logf(nil, "Using random seed = %d", seed)
	}
	randSource = rand.New(rand.NewSource(seed))
	if bool2int(zero)+bool2int(sparse)+bool2int(ascii)+bool2int(pattern)+bool2int(chargen) > 1 {
		log.Fatal("Can only supply one of --zero, --sparse, --ascii, --pattern or --chargen")
	}
	switch {
	case zero, sparse:
		source = zeroReader{}
	case ascii:
		source = asciiReader{}
	case pattern:
		source = readers.NewPatternReader(math.MaxInt64)
	case chargen:
		source = &chargenReader{}
	default:
		source = randSource
	}
	if minFileSize > maxFileSize {
		maxFileSize = minFileSize
	}
}

type zeroReader struct{}

// Read a chunk of zeroes
func (zeroReader) Read(p []byte) (n int, err error) {
	for i := range p {
		p[i] = 0
	}
	return len(p), nil
}

type asciiReader struct{}

// Read a chunk of printable ASCII characters
func (asciiReader) Read(p []byte) (n int, err error) {
	n, err = randSource.Read(p)
	for i := range p[:n] {
		p[i] = (p[i] % (0x7F - 0x20)) + 0x20
	}
	return n, err
}

type chargenReader struct {
	start   byte // offset from startChar to start line with
	written byte // chars in line so far
}

// Read a chunk of printable ASCII characters in chargen format
func (r *chargenReader) Read(p []byte) (n int, err error) {
	const (
		startChar    = 0x20 // ' '
		endChar      = 0x7E // '~' inclusive
		charsPerLine = 72
	)
	for i := range p {
		if r.written >= charsPerLine {
			r.start++
			if r.start > endChar-startChar {
				r.start = 0
			}
			p[i] = '\n'
			r.written = 0
		} else {
			c := r.start + r.written + startChar
			if c > endChar {
				c -= endChar - startChar + 1
			}
			p[i] = c
			r.written++
		}
	}
	return len(p), err
}

// fileName creates a unique random file or directory name
func fileName() (name string) {
	for {
		length := randSource.Intn(maxFileNameLength-minFileNameLength) + minFileNameLength
		name = random.StringFn(length, randSource.Intn)
		if _, found := fileNames[name]; !found {
			break
		}
	}
	fileNames[name] = struct{}{}
	return name
}

// dir is a directory in the directory hierarchy being built up
type dir struct {
	name     string
	depth    int
	children []*dir
	parent   *dir
}

// Create a random directory hierarchy under d
func (d *dir) createDirectories() {
	for totalDirectories < directoriesToCreate {
		newDir := &dir{
			name:   fileName(),
			depth:  d.depth + 1,
			parent: d,
		}
		d.children = append(d.children, newDir)
		totalDirectories++
		switch randSource.Intn(4) {
		case 0:
			if d.depth < maxDepth {
				newDir.createDirectories()
			}
		case 1:
			return
		}
	}
}

// list the directory hierarchy
func (d *dir) list(path string, output []string) []string {
	dirPath := filepath.Join(path, d.name)
	output = append(output, dirPath)
	for _, subDir := range d.children {
		output = subDir.list(dirPath, output)
	}
	return output
}

// writeFile writes a random file at dir/name
func writeFile(dir, name string, size int64) {
	err := file.MkdirAll(dir, 0777)
	if err != nil {
		log.Fatalf("Failed to make directory %q: %v", dir, err)
	}
	path := filepath.Join(dir, name)
	fd, err := os.Create(path)
	if err != nil {
		log.Fatalf("Failed to open file %q: %v", path, err)
	}
	if sparse {
		err = fd.Truncate(size)
	} else {
		_, err = io.CopyN(fd, source, size)
	}
	if err != nil {
		log.Fatalf("Failed to write %v bytes to file %q: %v", size, path, err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatalf("Failed to close file %q: %v", path, err)
	}
	fs.Infof(path, "Written file size %v", fs.SizeSuffix(size))
}