1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
|
package godirwalk
import (
"errors"
"fmt"
"os"
"path/filepath"
)
// Options provide parameters for how the Walk function operates.
type Options struct {
// ErrorCallback specifies a function to be invoked in the case of an error
// that could potentially be ignored while walking a file system
// hierarchy. When set to nil or left as its zero-value, any error condition
// causes Walk to immediately return the error describing what took
// place. When non-nil, this user supplied function is invoked with the OS
// pathname of the file system object that caused the error along with the
// error that took place. The return value of the supplied ErrorCallback
// function determines whether the error will cause Walk to halt immediately
// as it would were no ErrorCallback value provided, or skip this file
// system node yet continue on with the remaining nodes in the file system
// hierarchy.
//
// ErrorCallback is invoked both for errors that are returned by the
// runtime, and for errors returned by other user supplied callback
// functions.
ErrorCallback func(string, error) ErrorAction
// FollowSymbolicLinks specifies whether Walk will follow symbolic links
// that refer to directories. When set to false or left as its zero-value,
// Walk will still invoke the callback function with symbolic link nodes,
// but if the symbolic link refers to a directory, it will not recurse on
// that directory. When set to true, Walk will recurse on symbolic links
// that refer to a directory.
FollowSymbolicLinks bool
// Unsorted controls whether or not Walk will sort the immediate descendants
// of a directory by their relative names prior to visiting each of those
// entries.
//
// When set to false or left at its zero-value, Walk will get the list of
// immediate descendants of a particular directory, sort that list by
// lexical order of their names, and then visit each node in the list in
// sorted order. This will cause Walk to always traverse the same directory
// tree in the same order, however may be inefficient for directories with
// many immediate descendants.
//
// When set to true, Walk skips sorting the list of immediate descendants
// for a directory, and simply visits each node in the order the operating
// system enumerated them. This will be more fast, but with the side effect
// that the traversal order may be different from one invocation to the
// next.
Unsorted bool
// Callback is a required function that Walk will invoke for every file
// system node it encounters.
Callback WalkFunc
// PostChildrenCallback is an option function that Walk will invoke for
// every file system directory it encounters after its children have been
// processed.
PostChildrenCallback WalkFunc
// ScratchBuffer is an optional byte slice to use as a scratch buffer for
// Walk to use when reading directory entries, to reduce amount of garbage
// generation. Not all architectures take advantage of the scratch
// buffer. If omitted or the provided buffer has fewer bytes than
// MinimumScratchBufferSize, then a buffer with MinimumScratchBufferSize
// bytes will be created and used once per Walk invocation.
ScratchBuffer []byte
// AllowNonDirectory causes Walk to bypass the check that ensures it is
// being called on a directory node, or when FollowSymbolicLinks is true, a
// symbolic link that points to a directory. Leave this value false to have
// Walk return an error when called on a non-directory. Set this true to
// have Walk run even when called on a non-directory node.
AllowNonDirectory bool
}
// ErrorAction defines a set of actions the Walk function could take based on
// the occurrence of an error while walking the file system. See the
// documentation for the ErrorCallback field of the Options structure for more
// information.
type ErrorAction int
const (
// Halt is the ErrorAction return value when the upstream code wants to halt
// the walk process when a runtime error takes place. It matches the default
// action the Walk function would take were no ErrorCallback provided.
Halt ErrorAction = iota
// SkipNode is the ErrorAction return value when the upstream code wants to
// ignore the runtime error for the current file system node, skip
// processing of the node that caused the error, and continue walking the
// file system hierarchy with the remaining nodes.
SkipNode
)
// WalkFunc is the type of the function called for each file system node visited
// by Walk. The pathname argument will contain the argument to Walk as a prefix;
// that is, if Walk is called with "dir", which is a directory containing the
// file "a", the provided WalkFunc will be invoked with the argument "dir/a",
// using the correct os.PathSeparator for the Go Operating System architecture,
// GOOS. The directory entry argument is a pointer to a Dirent for the node,
// providing access to both the basename and the mode type of the file system
// node.
//
// If an error is returned by the Callback or PostChildrenCallback functions,
// and no ErrorCallback function is provided, processing stops. If an
// ErrorCallback function is provided, then it is invoked with the OS pathname
// of the node that caused the error along along with the error. The return
// value of the ErrorCallback function determines whether to halt processing, or
// skip this node and continue processing remaining file system nodes.
//
// The exception is when the function returns the special value
// filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a
// directory, Walk skips the directory's contents entirely. If the function
// returns filepath.SkipDir when invoked on a non-directory file system node,
// Walk skips the remaining files in the containing directory. Note that any
// supplied ErrorCallback function is not invoked with filepath.SkipDir when the
// Callback or PostChildrenCallback functions return that special value.
type WalkFunc func(osPathname string, directoryEntry *Dirent) error
// Walk walks the file tree rooted at the specified directory, calling the
// specified callback function for each file system node in the tree, including
// root, symbolic links, and other node types.
//
// This function is often much faster than filepath.Walk because it does not
// invoke os.Stat for every node it encounters, but rather obtains the file
// system node type when it reads the parent directory.
//
// If a runtime error occurs, either from the operating system or from the
// upstream Callback or PostChildrenCallback functions, processing typically
// halts. However, when an ErrorCallback function is provided in the provided
// Options structure, that function is invoked with the error along with the OS
// pathname of the file system node that caused the error. The ErrorCallback
// function's return value determines the action that Walk will then take.
//
// func main() {
// dirname := "."
// if len(os.Args) > 1 {
// dirname = os.Args[1]
// }
// err := godirwalk.Walk(dirname, &godirwalk.Options{
// Callback: func(osPathname string, de *godirwalk.Dirent) error {
// fmt.Printf("%s %s\n", de.ModeType(), osPathname)
// return nil
// },
// ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction {
// // Your program may want to log the error somehow.
// fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
//
// // For the purposes of this example, a simple SkipNode will suffice,
// // although in reality perhaps additional logic might be called for.
// return godirwalk.SkipNode
// },
// })
// if err != nil {
// fmt.Fprintf(os.Stderr, "%s\n", err)
// os.Exit(1)
// }
// }
func Walk(pathname string, options *Options) error {
if options == nil || options.Callback == nil {
return errors.New("cannot walk without non-nil options and Callback function")
}
pathname = filepath.Clean(pathname)
var fi os.FileInfo
var err error
if options.FollowSymbolicLinks {
fi, err = os.Stat(pathname)
} else {
fi, err = os.Lstat(pathname)
}
if err != nil {
return err
}
mode := fi.Mode()
if !options.AllowNonDirectory && mode&os.ModeDir == 0 {
return fmt.Errorf("cannot Walk non-directory: %s", pathname)
}
dirent := &Dirent{
name: filepath.Base(pathname),
path: filepath.Dir(pathname),
modeType: mode & os.ModeType,
}
if len(options.ScratchBuffer) < MinimumScratchBufferSize {
options.ScratchBuffer = newScratchBuffer()
}
// If ErrorCallback is nil, set to a default value that halts the walk
// process on all operating system errors. This is done to allow error
// handling to be more succinct in the walk code.
if options.ErrorCallback == nil {
options.ErrorCallback = defaultErrorCallback
}
if err = walk(pathname, dirent, options); err != filepath.SkipDir {
return err
}
return nil // silence SkipDir for top level
}
// defaultErrorCallback always returns Halt because if the upstream code did not
// provide an ErrorCallback function, walking the file system hierarchy ought to
// halt upon any operating system error.
func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt }
// walk recursively traverses the file system node specified by pathname and the
// Dirent.
func walk(osPathname string, dirent *Dirent, options *Options) error {
err := options.Callback(osPathname, dirent)
if err != nil {
if err == filepath.SkipDir {
return err
}
if action := options.ErrorCallback(osPathname, err); action == SkipNode {
return nil
}
return err
}
if dirent.IsSymlink() {
if !options.FollowSymbolicLinks {
return nil
}
// Does this symlink point to a directory?
info, err := os.Stat(osPathname)
if err != nil {
if action := options.ErrorCallback(osPathname, err); action == SkipNode {
return nil
}
return err
}
if !info.IsDir() {
return nil
}
} else if !dirent.IsDir() {
return nil
}
// If get here, then specified pathname refers to a directory or a
// symbolic link to a directory.
var ds scanner
if options.Unsorted {
// When upstream does not request a sorted iteration, it's more memory
// efficient to read a single child at a time from the file system.
ds, err = NewScanner(osPathname)
} else {
// When upstream wants a sorted iteration, we must read the entire
// directory and sort through the child names, and then iterate on each
// child.
ds, err = newSortedScanner(osPathname, options.ScratchBuffer)
}
if err != nil {
if action := options.ErrorCallback(osPathname, err); action == SkipNode {
return nil
}
return err
}
for ds.Scan() {
deChild, err := ds.Dirent()
osChildname := filepath.Join(osPathname, deChild.name)
if err != nil {
if action := options.ErrorCallback(osChildname, err); action == SkipNode {
return nil
}
return err
}
err = walk(osChildname, deChild, options)
debug("osChildname: %q; error: %v\n", osChildname, err)
if err == nil {
continue
}
if err != filepath.SkipDir {
return err
}
// When received SkipDir on a directory or a symbolic link to a
// directory, stop processing that directory but continue processing
// siblings. When received on a non-directory, stop processing
// remaining siblings.
isDir, err := deChild.IsDirOrSymlinkToDir()
if err != nil {
if action := options.ErrorCallback(osChildname, err); action == SkipNode {
continue // ignore and continue with next sibling
}
return err // caller does not approve of this error
}
if !isDir {
break // stop processing remaining siblings, but allow post children callback
}
// continue processing remaining siblings
}
if err = ds.Err(); err != nil {
return err
}
if options.PostChildrenCallback == nil {
return nil
}
err = options.PostChildrenCallback(osPathname, dirent)
if err == nil || err == filepath.SkipDir {
return err
}
if action := options.ErrorCallback(osPathname, err); action == SkipNode {
return nil
}
return err
}
|