1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
|
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package kio
import (
"fmt"
"os"
"path/filepath"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/filesys"
"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
"sigs.k8s.io/kustomize/kyaml/sets"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// requiredResourcePackageAnnotations are annotations that are required to write resources back to
// files.
var requiredResourcePackageAnnotations = []string{kioutil.IndexAnnotation, kioutil.PathAnnotation}
// PackageBuffer implements Reader and Writer, storing Resources in a local field.
type PackageBuffer struct {
Nodes []*yaml.RNode
}
func (r *PackageBuffer) Read() ([]*yaml.RNode, error) {
return r.Nodes, nil
}
func (r *PackageBuffer) Write(nodes []*yaml.RNode) error {
r.Nodes = nodes
return nil
}
// LocalPackageReadWriter reads and writes Resources from / to a local directory.
// When writing, LocalPackageReadWriter will delete files if all of the Resources from
// that file have been removed from the output.
type LocalPackageReadWriter struct {
Kind string `yaml:"kind,omitempty"`
KeepReaderAnnotations bool `yaml:"keepReaderAnnotations,omitempty"`
// PreserveSeqIndent if true adds kioutil.SeqIndentAnnotation to each resource
PreserveSeqIndent bool
// PackagePath is the path to the package directory.
PackagePath string `yaml:"path,omitempty"`
// PackageFileName is the name of file containing package metadata.
// It will be used to identify package.
PackageFileName string `yaml:"packageFileName,omitempty"`
// MatchFilesGlob configures Read to only read Resources from files matching any of the
// provided patterns.
// Defaults to ["*.yaml", "*.yml"] if empty. To match all files specify ["*"].
MatchFilesGlob []string `yaml:"matchFilesGlob,omitempty"`
// IncludeSubpackages will configure Read to read Resources from subpackages.
// Subpackages are identified by presence of PackageFileName.
IncludeSubpackages bool `yaml:"includeSubpackages,omitempty"`
// ErrorIfNonResources will configure Read to throw an error if yaml missing missing
// apiVersion or kind is read.
ErrorIfNonResources bool `yaml:"errorIfNonResources,omitempty"`
// OmitReaderAnnotations will cause the reader to skip annotating Resources with the file
// path and mode.
OmitReaderAnnotations bool `yaml:"omitReaderAnnotations,omitempty"`
// SetAnnotations are annotations to set on the Resources as they are read.
SetAnnotations map[string]string `yaml:"setAnnotations,omitempty"`
// NoDeleteFiles if set to true, LocalPackageReadWriter won't delete any files
NoDeleteFiles bool `yaml:"noDeleteFiles,omitempty"`
files sets.String
// FileSkipFunc is a function which returns true if reader should ignore
// the file
FileSkipFunc LocalPackageSkipFileFunc
// FileSystem can be used to mock the disk file system.
FileSystem filesys.FileSystemOrOnDisk
// WrapBareSeqNode wraps the bare sequence node document with map node,
// kyaml uses reader annotations to track resources, it is not possible to
// add them to bare sequence nodes, this option enables wrapping such bare
// sequence nodes into map node with key yaml.BareSeqNodeWrappingKey
// note that this wrapping is different and not related to ResourceList wrapping
WrapBareSeqNode bool
}
func (r *LocalPackageReadWriter) Read() ([]*yaml.RNode, error) {
nodes, err := LocalPackageReader{
PackagePath: r.PackagePath,
MatchFilesGlob: r.MatchFilesGlob,
IncludeSubpackages: r.IncludeSubpackages,
ErrorIfNonResources: r.ErrorIfNonResources,
SetAnnotations: r.SetAnnotations,
PackageFileName: r.PackageFileName,
FileSkipFunc: r.FileSkipFunc,
PreserveSeqIndent: r.PreserveSeqIndent,
FileSystem: r.FileSystem,
WrapBareSeqNode: r.WrapBareSeqNode,
}.Read()
if err != nil {
return nil, errors.Wrap(err)
}
// keep track of all the files
if !r.NoDeleteFiles {
r.files, err = r.getFiles(nodes)
if err != nil {
return nil, errors.Wrap(err)
}
}
return nodes, nil
}
func (r *LocalPackageReadWriter) Write(nodes []*yaml.RNode) error {
newFiles, err := r.getFiles(nodes)
if err != nil {
return errors.Wrap(err)
}
var clear []string
for k := range r.SetAnnotations {
clear = append(clear, k)
}
err = LocalPackageWriter{
PackagePath: r.PackagePath,
ClearAnnotations: clear,
KeepReaderAnnotations: r.KeepReaderAnnotations,
FileSystem: r.FileSystem,
}.Write(nodes)
if err != nil {
return errors.Wrap(err)
}
deleteFiles := r.files.Difference(newFiles)
for f := range deleteFiles {
if err = r.FileSystem.RemoveAll(filepath.Join(r.PackagePath, f)); err != nil {
return errors.Wrap(err)
}
}
return nil
}
func (r *LocalPackageReadWriter) getFiles(nodes []*yaml.RNode) (sets.String, error) {
val := sets.String{}
for _, n := range nodes {
path, _, err := kioutil.GetFileAnnotations(n)
if err != nil {
return nil, errors.Wrap(err)
}
val.Insert(path)
}
return val, nil
}
// LocalPackageSkipFileFunc is a function which returns true if the file
// in the package should be ignored by reader.
// relPath is an OS specific relative path
type LocalPackageSkipFileFunc func(relPath string) bool
// LocalPackageReader reads ResourceNodes from a local package.
type LocalPackageReader struct {
Kind string `yaml:"kind,omitempty"`
// PackagePath is the path to the package directory.
PackagePath string `yaml:"path,omitempty"`
// PackageFileName is the name of file containing package metadata.
// It will be used to identify package.
PackageFileName string `yaml:"packageFileName,omitempty"`
// MatchFilesGlob configures Read to only read Resources from files matching any of the
// provided patterns.
// Defaults to ["*.yaml", "*.yml"] if empty. To match all files specify ["*"].
MatchFilesGlob []string `yaml:"matchFilesGlob,omitempty"`
// IncludeSubpackages will configure Read to read Resources from subpackages.
// Subpackages are identified by presence of PackageFileName.
IncludeSubpackages bool `yaml:"includeSubpackages,omitempty"`
// ErrorIfNonResources will configure Read to throw an error if yaml missing missing
// apiVersion or kind is read.
ErrorIfNonResources bool `yaml:"errorIfNonResources,omitempty"`
// OmitReaderAnnotations will cause the reader to skip annotating Resources with the file
// path and mode.
OmitReaderAnnotations bool `yaml:"omitReaderAnnotations,omitempty"`
// SetAnnotations are annotations to set on the Resources as they are read.
SetAnnotations map[string]string `yaml:"setAnnotations,omitempty"`
// FileSkipFunc is a function which returns true if reader should ignore
// the file
FileSkipFunc LocalPackageSkipFileFunc
// PreserveSeqIndent if true adds kioutil.SeqIndentAnnotation to each resource
PreserveSeqIndent bool
// FileSystem can be used to mock the disk file system.
FileSystem filesys.FileSystemOrOnDisk
// WrapBareSeqNode wraps the bare sequence node document with map node,
// kyaml uses reader annotations to track resources, it is not possible to
// add them to bare sequence nodes, this option enables wrapping such bare
// sequence nodes into map node with key yaml.BareSeqNodeWrappingKey
// note that this wrapping is different and not related to ResourceList wrapping
WrapBareSeqNode bool
}
var _ Reader = LocalPackageReader{}
var DefaultMatch = []string{"*.yaml", "*.yml"}
var JSONMatch = []string{"*.json"}
var MatchAll = append(DefaultMatch, JSONMatch...)
// Read reads the Resources.
func (r LocalPackageReader) Read() ([]*yaml.RNode, error) {
if r.PackagePath == "" {
return nil, fmt.Errorf("must specify package path")
}
// use slash for path
r.PackagePath = filepath.ToSlash(r.PackagePath)
if len(r.MatchFilesGlob) == 0 {
r.MatchFilesGlob = DefaultMatch
}
var operand ResourceNodeSlice
var pathRelativeTo string
var err error
ignoreFilesMatcher := &ignoreFilesMatcher{
fs: r.FileSystem,
}
dir, file, err := r.FileSystem.CleanedAbs(r.PackagePath)
if err != nil {
return nil, errors.Wrap(err)
}
r.PackagePath = filepath.Join(string(dir), file)
err = r.FileSystem.Walk(r.PackagePath, func(
path string, info os.FileInfo, err error) error {
if err != nil {
return errors.Wrap(err)
}
// is this the user specified path?
if path == r.PackagePath {
if info.IsDir() {
// skip the root package directory, but check for a
// .krmignore file first.
pathRelativeTo = r.PackagePath
return ignoreFilesMatcher.readIgnoreFile(path)
}
// user specified path is a file rather than a directory.
// make its path relative to its parent so it can be written to another file.
pathRelativeTo = filepath.Dir(r.PackagePath)
}
// check if we should skip the directory or file
if info.IsDir() {
return r.shouldSkipDir(path, ignoreFilesMatcher)
}
// get the relative path to file within the package so we can write the files back out
// to another location.
relPath, err := filepath.Rel(pathRelativeTo, path)
if err != nil {
return errors.WrapPrefixf(err, pathRelativeTo)
}
if match, err := r.shouldSkipFile(path, relPath, ignoreFilesMatcher); err != nil {
return err
} else if match {
// skip this file
return nil
}
r.initReaderAnnotations(relPath, info)
nodes, err := r.readFile(path, info)
if err != nil {
return errors.WrapPrefixf(err, path)
}
operand = append(operand, nodes...)
return nil
})
return operand, err
}
// readFile reads the ResourceNodes from a file
func (r *LocalPackageReader) readFile(path string, _ os.FileInfo) ([]*yaml.RNode, error) {
f, err := r.FileSystem.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
rr := &ByteReader{
DisableUnwrapping: true,
Reader: f,
OmitReaderAnnotations: r.OmitReaderAnnotations,
SetAnnotations: r.SetAnnotations,
PreserveSeqIndent: r.PreserveSeqIndent,
WrapBareSeqNode: r.WrapBareSeqNode,
}
return rr.Read()
}
// shouldSkipFile returns true if the file should be skipped
func (r *LocalPackageReader) shouldSkipFile(path, relPath string, matcher *ignoreFilesMatcher) (bool, error) {
// check if the file is covered by a .krmignore file.
if matcher.matchFile(path) {
return true, nil
}
if r.FileSkipFunc != nil && r.FileSkipFunc(relPath) {
return true, nil
}
// check if the files are in scope
for _, g := range r.MatchFilesGlob {
if match, err := filepath.Match(g, filepath.Base(path)); err != nil {
return true, errors.Wrap(err)
} else if match {
return false, nil
}
}
return true, nil
}
// initReaderAnnotations adds the LocalPackageReader Annotations to r.SetAnnotations
func (r *LocalPackageReader) initReaderAnnotations(path string, _ os.FileInfo) {
if r.SetAnnotations == nil {
r.SetAnnotations = map[string]string{}
}
if !r.OmitReaderAnnotations {
r.SetAnnotations[kioutil.PathAnnotation] = path
r.SetAnnotations[kioutil.LegacyPathAnnotation] = path
}
}
// shouldSkipDir returns a filepath.SkipDir if the directory should be skipped
func (r *LocalPackageReader) shouldSkipDir(path string, matcher *ignoreFilesMatcher) error {
if matcher.matchDir(path) {
return filepath.SkipDir
}
if r.PackageFileName == "" {
return nil
}
// check if this is a subpackage
if !r.FileSystem.Exists(filepath.Join(path, r.PackageFileName)) {
return nil
}
if !r.IncludeSubpackages {
return filepath.SkipDir
}
return matcher.readIgnoreFile(path)
}
|