1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
|
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// Package kio contains low-level libraries for reading, modifying and writing
// Resource Configuration and packages.
package kio
import (
"fmt"
"strconv"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// Reader reads ResourceNodes. Analogous to io.Reader.
type Reader interface {
Read() ([]*yaml.RNode, error)
}
// ResourceNodeSlice is a collection of ResourceNodes.
// While ResourceNodeSlice has no inherent constraints on ordering or uniqueness, specific
// Readers, Filters or Writers may have constraints.
type ResourceNodeSlice []*yaml.RNode
var _ Reader = ResourceNodeSlice{}
func (o ResourceNodeSlice) Read() ([]*yaml.RNode, error) {
return o, nil
}
// Writer writes ResourceNodes. Analogous to io.Writer.
type Writer interface {
Write([]*yaml.RNode) error
}
// WriterFunc implements a Writer as a function.
type WriterFunc func([]*yaml.RNode) error
func (fn WriterFunc) Write(o []*yaml.RNode) error {
return fn(o)
}
// ReaderWriter implements both Reader and Writer interfaces
type ReaderWriter interface {
Reader
Writer
}
// Filter modifies a collection of Resource Configuration by returning the modified slice.
// When possible, Filters should be serializable to yaml so that they can be described
// as either data or code.
//
// Analogous to http://www.linfo.org/filters.html
type Filter interface {
Filter([]*yaml.RNode) ([]*yaml.RNode, error)
}
// TrackableFilter is an extension of Filter which is also capable of tracking
// which fields were mutated by the filter.
type TrackableFilter interface {
Filter
WithMutationTracker(func(key, value, tag string, node *yaml.RNode))
}
// FilterFunc implements a Filter as a function.
type FilterFunc func([]*yaml.RNode) ([]*yaml.RNode, error)
func (fn FilterFunc) Filter(o []*yaml.RNode) ([]*yaml.RNode, error) {
return fn(o)
}
// Pipeline reads Resource Configuration from a set of Inputs, applies some
// transformation filters, and writes the results to a set of Outputs.
//
// Analogous to http://www.linfo.org/pipes.html
type Pipeline struct {
// Inputs provide sources for Resource Configuration to be read.
Inputs []Reader `yaml:"inputs,omitempty"`
// Filters are transformations applied to the Resource Configuration.
// They are applied in the order they are specified.
// Analogous to http://www.linfo.org/filters.html
Filters []Filter `yaml:"filters,omitempty"`
// Outputs are where the transformed Resource Configuration is written.
Outputs []Writer `yaml:"outputs,omitempty"`
// ContinueOnEmptyResult configures what happens when a filter in the pipeline
// returns an empty result.
// If it is false (default), subsequent filters will be skipped and the result
// will be returned immediately. This is useful as an optimization when you
// know that subsequent filters will not alter the empty result.
// If it is true, the empty result will be provided as input to the next
// filter in the list. This is useful when subsequent functions in the
// pipeline may generate new resources.
ContinueOnEmptyResult bool `yaml:"continueOnEmptyResult,omitempty"`
}
// Execute executes each step in the sequence, returning immediately after encountering
// any error as part of the Pipeline.
func (p Pipeline) Execute() error {
return p.ExecuteWithCallback(nil)
}
// PipelineExecuteCallbackFunc defines a callback function that will be called each time a step in the pipeline succeeds.
type PipelineExecuteCallbackFunc = func(op Filter)
// ExecuteWithCallback executes each step in the sequence, returning immediately after encountering
// any error as part of the Pipeline. The callback will be called each time a step succeeds.
func (p Pipeline) ExecuteWithCallback(callback PipelineExecuteCallbackFunc) error {
var result []*yaml.RNode
// read from the inputs
for _, i := range p.Inputs {
nodes, err := i.Read()
if err != nil {
return errors.Wrap(err)
}
result = append(result, nodes...)
}
// apply operations
for i := range p.Filters {
// Not all RNodes passed through kio.Pipeline have metadata nor should
// they all be required to.
nodeAnnos, err := PreprocessResourcesForInternalAnnotationMigration(result)
if err != nil {
return err
}
op := p.Filters[i]
if callback != nil {
callback(op)
}
result, err = op.Filter(result)
// TODO (issue 2872): This len(result) == 0 should be removed and empty result list should be
// handled by outputs. However currently some writer like LocalPackageReadWriter
// will clear the output directory and which will cause unpredictable results
if len(result) == 0 && !p.ContinueOnEmptyResult || err != nil {
return errors.Wrap(err)
}
// If either the internal annotations for path, index, and id OR the legacy
// annotations for path, index, and id are changed, we have to update the other.
err = ReconcileInternalAnnotations(result, nodeAnnos)
if err != nil {
return err
}
}
// write to the outputs
for _, o := range p.Outputs {
if err := o.Write(result); err != nil {
return errors.Wrap(err)
}
}
return nil
}
// FilterAll runs the yaml.Filter against all inputs
func FilterAll(filter yaml.Filter) Filter {
return FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
for i := range nodes {
_, err := filter.Filter(nodes[i])
if err != nil {
return nil, errors.Wrap(err)
}
}
return nodes, nil
})
}
// PreprocessResourcesForInternalAnnotationMigration returns a mapping from id to all
// internal annotations, so that we can use it to reconcile the annotations
// later. This is necessary because currently both internal-prefixed annotations
// and legacy annotations are currently supported, and a change to one must be
// reflected in the other if needed.
func PreprocessResourcesForInternalAnnotationMigration(result []*yaml.RNode) (map[string]map[string]string, error) {
idToAnnosMap := make(map[string]map[string]string)
for i := range result {
idStr := strconv.Itoa(i)
err := result[i].PipeE(yaml.SetAnnotation(kioutil.InternalAnnotationsMigrationResourceIDAnnotation, idStr))
if err != nil {
return nil, err
}
idToAnnosMap[idStr] = kioutil.GetInternalAnnotations(result[i])
if err = kioutil.CopyLegacyAnnotations(result[i]); err != nil {
return nil, err
}
meta, _ := result[i].GetMeta()
if err = checkMismatchedAnnos(meta.Annotations); err != nil {
return nil, err
}
}
return idToAnnosMap, nil
}
func checkMismatchedAnnos(annotations map[string]string) error {
path := annotations[kioutil.PathAnnotation]
index := annotations[kioutil.IndexAnnotation]
id := annotations[kioutil.IdAnnotation]
legacyPath := annotations[kioutil.LegacyPathAnnotation]
legacyIndex := annotations[kioutil.LegacyIndexAnnotation]
legacyId := annotations[kioutil.LegacyIdAnnotation]
// if prior to running the functions, the legacy and internal annotations differ,
// throw an error as we cannot infer the user's intent.
if path != "" && legacyPath != "" && path != legacyPath {
return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
}
if index != "" && legacyIndex != "" && index != legacyIndex {
return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
}
if id != "" && legacyId != "" && id != legacyId {
return fmt.Errorf("resource input to function has mismatched legacy and internal id annotations")
}
return nil
}
type nodeAnnotations struct {
path string
index string
id string
}
// ReconcileInternalAnnotations reconciles the annotation format for path, index and id annotations.
// It will ensure the output annotation format matches the format in the input. e.g. if the input
// format uses the legacy format and the output will be converted to the legacy format if it's not.
func ReconcileInternalAnnotations(result []*yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
useInternal, useLegacy, err := determineAnnotationsFormat(nodeAnnosMap)
if err != nil {
return err
}
for i := range result {
// if only one annotation is set, set the other.
err = missingInternalOrLegacyAnnotations(result[i])
if err != nil {
return err
}
// we must check to see if the function changed either the new internal annotations
// or the old legacy annotations. If one is changed, the change must be reflected
// in the other.
err = checkAnnotationsAltered(result[i], nodeAnnosMap)
if err != nil {
return err
}
// We invoke determineAnnotationsFormat to find out if the original annotations
// use the internal or (and) the legacy format. We format the resources to
// make them consistent with original format.
err = formatInternalAnnotations(result[i], useInternal, useLegacy)
if err != nil {
return err
}
// if the annotations are still somehow out of sync, throw an error
meta, _ := result[i].GetMeta()
err = checkMismatchedAnnos(meta.Annotations)
if err != nil {
return err
}
if _, err = result[i].Pipe(yaml.ClearAnnotation(kioutil.InternalAnnotationsMigrationResourceIDAnnotation)); err != nil {
return err
}
}
return nil
}
// determineAnnotationsFormat determines if the resources are using one of the internal and legacy annotation format or both of them.
func determineAnnotationsFormat(nodeAnnosMap map[string]map[string]string) (bool, bool, error) {
var useInternal, useLegacy bool
var err error
if len(nodeAnnosMap) == 0 {
return true, true, nil
}
var internal, legacy *bool
for _, annos := range nodeAnnosMap {
_, foundPath := annos[kioutil.PathAnnotation]
_, foundIndex := annos[kioutil.IndexAnnotation]
_, foundId := annos[kioutil.IdAnnotation]
_, foundLegacyPath := annos[kioutil.LegacyPathAnnotation]
_, foundLegacyIndex := annos[kioutil.LegacyIndexAnnotation]
_, foundLegacyId := annos[kioutil.LegacyIdAnnotation]
if !(foundPath || foundIndex || foundId || foundLegacyPath || foundLegacyIndex || foundLegacyId) {
continue
}
foundOneOf := foundPath || foundIndex || foundId
if internal == nil {
f := foundOneOf
internal = &f
}
if (foundOneOf && !*internal) || (!foundOneOf && *internal) {
err = fmt.Errorf("the annotation formatting in the input resources is not consistent")
return useInternal, useLegacy, err
}
foundOneOf = foundLegacyPath || foundLegacyIndex || foundLegacyId
if legacy == nil {
f := foundOneOf
legacy = &f
}
if (foundOneOf && !*legacy) || (!foundOneOf && *legacy) {
err = fmt.Errorf("the annotation formatting in the input resources is not consistent")
return useInternal, useLegacy, err
}
}
if internal != nil {
useInternal = *internal
}
if legacy != nil {
useLegacy = *legacy
}
return useInternal, useLegacy, err
}
func missingInternalOrLegacyAnnotations(rn *yaml.RNode) error {
if err := missingInternalOrLegacyAnnotation(rn, kioutil.PathAnnotation, kioutil.LegacyPathAnnotation); err != nil {
return err
}
if err := missingInternalOrLegacyAnnotation(rn, kioutil.IndexAnnotation, kioutil.LegacyIndexAnnotation); err != nil {
return err
}
if err := missingInternalOrLegacyAnnotation(rn, kioutil.IdAnnotation, kioutil.LegacyIdAnnotation); err != nil {
return err
}
return nil
}
func missingInternalOrLegacyAnnotation(rn *yaml.RNode, newKey string, legacyKey string) error {
meta, _ := rn.GetMeta()
annotations := meta.Annotations
value := annotations[newKey]
legacyValue := annotations[legacyKey]
if value == "" && legacyValue == "" {
// do nothing
return nil
}
if value == "" {
// new key is not set, copy from legacy key
if err := rn.PipeE(yaml.SetAnnotation(newKey, legacyValue)); err != nil {
return err
}
} else if legacyValue == "" {
// legacy key is not set, copy from new key
if err := rn.PipeE(yaml.SetAnnotation(legacyKey, value)); err != nil {
return err
}
}
return nil
}
func checkAnnotationsAltered(rn *yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
meta, _ := rn.GetMeta()
annotations := meta.Annotations
// get the resource's current path, index, and ids from the new annotations
internal := nodeAnnotations{
path: annotations[kioutil.PathAnnotation],
index: annotations[kioutil.IndexAnnotation],
id: annotations[kioutil.IdAnnotation],
}
// get the resource's current path, index, and ids from the legacy annotations
legacy := nodeAnnotations{
path: annotations[kioutil.LegacyPathAnnotation],
index: annotations[kioutil.LegacyIndexAnnotation],
id: annotations[kioutil.LegacyIdAnnotation],
}
rid := annotations[kioutil.InternalAnnotationsMigrationResourceIDAnnotation]
originalAnnotations, found := nodeAnnosMap[rid]
if !found {
return nil
}
originalPath, found := originalAnnotations[kioutil.PathAnnotation]
if !found {
originalPath = originalAnnotations[kioutil.LegacyPathAnnotation]
}
if originalPath != "" {
switch {
case originalPath != internal.path && originalPath != legacy.path && internal.path != legacy.path:
return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
case originalPath != internal.path:
if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.LegacyPathAnnotation, internal.path)); err != nil {
return err
}
case originalPath != legacy.path:
if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.PathAnnotation, legacy.path)); err != nil {
return err
}
}
}
originalIndex, found := originalAnnotations[kioutil.IndexAnnotation]
if !found {
originalIndex = originalAnnotations[kioutil.LegacyIndexAnnotation]
}
if originalIndex != "" {
switch {
case originalIndex != internal.index && originalIndex != legacy.index && internal.index != legacy.index:
return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
case originalIndex != internal.index:
if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.LegacyIndexAnnotation, internal.index)); err != nil {
return err
}
case originalIndex != legacy.index:
if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.IndexAnnotation, legacy.index)); err != nil {
return err
}
}
}
return nil
}
func formatInternalAnnotations(rn *yaml.RNode, useInternal, useLegacy bool) error {
if !useInternal {
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.IdAnnotation)); err != nil {
return err
}
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.PathAnnotation)); err != nil {
return err
}
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.IndexAnnotation)); err != nil {
return err
}
}
if !useLegacy {
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyIdAnnotation)); err != nil {
return err
}
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyPathAnnotation)); err != nil {
return err
}
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyIndexAnnotation)); err != nil {
return err
}
}
return nil
}
|