1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394
|
package copy
import (
"context"
"errors"
"fmt"
"io"
"os"
"time"
"github.com/containers/image/v5/docker/reference"
internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache"
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/internal/imagedestination"
"github.com/containers/image/v5/internal/imagesource"
internalManifest "github.com/containers/image/v5/internal/manifest"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/blobinfocache"
compression "github.com/containers/image/v5/pkg/compression/types"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/signature/signer"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
encconfig "github.com/containers/ocicrypt/config"
digest "github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"golang.org/x/sync/semaphore"
"golang.org/x/term"
)
var (
// ErrDecryptParamsMissing is returned if there is missing decryption parameters
ErrDecryptParamsMissing = errors.New("Necessary DecryptParameters not present")
// maxParallelDownloads is used to limit the maximum number of parallel
// downloads. Let's follow Firefox by limiting it to 6.
maxParallelDownloads = uint(6)
)
const (
// CopySystemImage is the default value which, when set in
// Options.ImageListSelection, indicates that the caller expects only one
// image to be copied, so if the source reference refers to a list of
// images, one that matches the current system will be selected.
CopySystemImage ImageListSelection = iota
// CopyAllImages is a value which, when set in Options.ImageListSelection,
// indicates that the caller expects to copy multiple images, and if
// the source reference refers to a list, that the list and every image
// to which it refers will be copied. If the source reference refers
// to a list, the target reference can not accept lists, an error
// should be returned.
CopyAllImages
// CopySpecificImages is a value which, when set in
// Options.ImageListSelection, indicates that the caller expects the
// source reference to be either a single image or a list of images,
// and if the source reference is a list, wants only specific instances
// from it copied (or none of them, if the list of instances to copy is
// empty), along with the list itself. If the target reference can
// only accept one image (i.e., it cannot accept lists), an error
// should be returned.
CopySpecificImages
)
// ImageListSelection is one of CopySystemImage, CopyAllImages, or
// CopySpecificImages, to control whether, when the source reference is a list,
// copy.Image() copies only an image which matches the current runtime
// environment, or all images which match the supplied reference, or only
// specific images from the source reference.
type ImageListSelection int
// Options allows supplying non-default configuration modifying the behavior of CopyImage.
type Options struct {
RemoveSignatures bool // Remove any pre-existing signatures. Signers and SignBy… will still add a new signature.
// Signers to use to add signatures during the copy.
// Callers are still responsible for closing these Signer objects; they can be reused for multiple copy.Image operations in a row.
Signers []*signer.Signer
SignBy string // If non-empty, asks for a signature to be added during the copy, and specifies a key ID, as accepted by signature.NewGPGSigningMechanism().SignDockerManifest(),
SignPassphrase string // Passphrase to use when signing with the key ID from `SignBy`.
SignBySigstorePrivateKeyFile string // If non-empty, asks for a signature to be added during the copy, using a sigstore private key file at the provided path.
SignSigstorePrivateKeyPassphrase []byte // Passphrase to use when signing with `SignBySigstorePrivateKeyFile`.
SignIdentity reference.Named // Identify to use when signing, defaults to the docker reference of the destination
ReportWriter io.Writer
SourceCtx *types.SystemContext
DestinationCtx *types.SystemContext
ProgressInterval time.Duration // time to wait between reports to signal the progress channel
Progress chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset.
// Preserve digests, and fail if we cannot.
PreserveDigests bool
// manifest MIME type of image set by user. "" is default and means use the autodetection to the manifest MIME type
ForceManifestMIMEType string
ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list
Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances and the list itself
// Give priority to pulling gzip images if multiple images are present when configured to OptionalBoolTrue,
// prefers the best compression if this is configured as OptionalBoolFalse. Choose automatically (and the choice may change over time)
// if this is set to OptionalBoolUndefined (which is the default behavior, and recommended for most callers).
// This only affects CopySystemImage.
PreferGzipInstances types.OptionalBool
// If OciEncryptConfig is non-nil, it indicates that an image should be encrypted.
// The encryption options is derived from the construction of EncryptConfig object.
OciEncryptConfig *encconfig.EncryptConfig
// OciEncryptLayers represents the list of layers to encrypt.
// If nil, don't encrypt any layers.
// If non-nil and len==0, denotes encrypt all layers.
// integers in the slice represent 0-indexed layer indices, with support for negative
// indexing. i.e. 0 is the first layer, -1 is the last (top-most) layer.
OciEncryptLayers *[]int
// OciDecryptConfig contains the config that can be used to decrypt an image if it is
// encrypted if non-nil. If nil, it does not attempt to decrypt an image.
OciDecryptConfig *encconfig.DecryptConfig
// A weighted semaphore to limit the amount of concurrently copied layers and configs. Applies to all copy operations using the semaphore. If set, MaxParallelDownloads is ignored.
ConcurrentBlobCopiesSemaphore *semaphore.Weighted
// MaxParallelDownloads indicates the maximum layers to pull at the same time. Applies to a single copy operation. A reasonable default is used if this is left as 0. Ignored if ConcurrentBlobCopiesSemaphore is set.
MaxParallelDownloads uint
// When OptimizeDestinationImageAlreadyExists is set, optimize the copy assuming that the destination image already
// exists (and is equivalent). Making the eventual (no-op) copy more performant for this case. Enabling the option
// is slightly pessimistic if the destination image doesn't exist, or is not equivalent.
OptimizeDestinationImageAlreadyExists bool
// Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type
// to not indicate "nondistributable".
DownloadForeignLayers bool
// Contains slice of OptionCompressionVariant, where copy will ensure that for each platform
// in the manifest list, a variant with the requested compression will exist.
// Invalid when copying a non-multi-architecture image. That will probably
// change in the future.
EnsureCompressionVariantsExist []OptionCompressionVariant
// ForceCompressionFormat ensures that the compression algorithm set in
// DestinationCtx.CompressionFormat is used exclusively, and blobs of other
// compression algorithms are not reused.
ForceCompressionFormat bool
}
// OptionCompressionVariant allows to supply information about
// selected compression algorithm and compression level by the
// end-user. Refer to EnsureCompressionVariantsExist to know
// more about its usage.
type OptionCompressionVariant struct {
Algorithm compression.Algorithm
Level *int // Only used when we are creating a new image instance using the specified algorithm, not when the image already contains such an instance
}
// copier allows us to keep track of diffID values for blobs, and other
// data shared across one or more images in a possible manifest list.
// The owner must call close() when done.
type copier struct {
policyContext *signature.PolicyContext
dest private.ImageDestination
rawSource private.ImageSource
options *Options // never nil
reportWriter io.Writer
progressOutput io.Writer
unparsedToplevel *image.UnparsedImage // for rawSource
blobInfoCache internalblobinfocache.BlobInfoCache2
concurrentBlobCopiesSemaphore *semaphore.Weighted // Limits the amount of concurrently copied blobs
signers []*signer.Signer // Signers to use to create new signatures for the image
signersToClose []*signer.Signer // Signers that should be closed when this copier is destroyed.
}
// Internal function to validate `requireCompressionFormatMatch` for copySingleImageOptions
func shouldRequireCompressionFormatMatch(options *Options) (bool, error) {
if options.ForceCompressionFormat && (options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil) {
return false, fmt.Errorf("cannot use ForceCompressionFormat with undefined default compression format")
}
return options.ForceCompressionFormat, nil
}
// Image copies image from srcRef to destRef, using policyContext to validate
// source image admissibility. It returns the manifest which was written to
// the new copy of the image.
func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (copiedManifest []byte, retErr error) {
if options == nil {
options = &Options{}
}
if err := validateImageListSelection(options.ImageListSelection); err != nil {
return nil, err
}
reportWriter := io.Discard
if options.ReportWriter != nil {
reportWriter = options.ReportWriter
}
publicDest, err := destRef.NewImageDestination(ctx, options.DestinationCtx)
if err != nil {
return nil, fmt.Errorf("initializing destination %s: %w", transports.ImageName(destRef), err)
}
dest := imagedestination.FromPublic(publicDest)
defer func() {
if err := dest.Close(); err != nil {
if retErr != nil {
retErr = fmt.Errorf(" (dest: %v): %w", err, retErr)
} else {
retErr = fmt.Errorf(" (dest: %v)", err)
}
}
}()
publicRawSource, err := srcRef.NewImageSource(ctx, options.SourceCtx)
if err != nil {
return nil, fmt.Errorf("initializing source %s: %w", transports.ImageName(srcRef), err)
}
rawSource := imagesource.FromPublic(publicRawSource)
defer func() {
if err := rawSource.Close(); err != nil {
if retErr != nil {
retErr = fmt.Errorf(" (src: %v): %w", err, retErr)
} else {
retErr = fmt.Errorf(" (src: %v)", err)
}
}
}()
// If reportWriter is not a TTY (e.g., when piping to a file), do not
// print the progress bars to avoid long and hard to parse output.
// Instead use printCopyInfo() to print single line "Copying ..." messages.
progressOutput := reportWriter
if !isTTY(reportWriter) {
progressOutput = io.Discard
}
c := &copier{
policyContext: policyContext,
dest: dest,
rawSource: rawSource,
options: options,
reportWriter: reportWriter,
progressOutput: progressOutput,
unparsedToplevel: image.UnparsedInstance(rawSource, nil),
// FIXME? The cache is used for sources and destinations equally, but we only have a SourceCtx and DestinationCtx.
// For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more).
// Conceptually the cache settings should be in copy.Options instead.
blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)),
}
defer c.close()
c.blobInfoCache.Open()
defer c.blobInfoCache.Close()
// Set the concurrentBlobCopiesSemaphore if we can copy layers in parallel.
if dest.HasThreadSafePutBlob() && rawSource.HasThreadSafeGetBlob() {
c.concurrentBlobCopiesSemaphore = c.options.ConcurrentBlobCopiesSemaphore
if c.concurrentBlobCopiesSemaphore == nil {
max := c.options.MaxParallelDownloads
if max == 0 {
max = maxParallelDownloads
}
c.concurrentBlobCopiesSemaphore = semaphore.NewWeighted(int64(max))
}
} else {
c.concurrentBlobCopiesSemaphore = semaphore.NewWeighted(int64(1))
if c.options.ConcurrentBlobCopiesSemaphore != nil {
if err := c.options.ConcurrentBlobCopiesSemaphore.Acquire(ctx, 1); err != nil {
return nil, fmt.Errorf("acquiring semaphore for concurrent blob copies: %w", err)
}
defer c.options.ConcurrentBlobCopiesSemaphore.Release(1)
}
}
if err := c.setupSigners(); err != nil {
return nil, err
}
multiImage, err := isMultiImage(ctx, c.unparsedToplevel)
if err != nil {
return nil, fmt.Errorf("determining manifest MIME type for %s: %w", transports.ImageName(srcRef), err)
}
if !multiImage {
if len(options.EnsureCompressionVariantsExist) > 0 {
return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
}
requireCompressionFormatMatch, err := shouldRequireCompressionFormatMatch(options)
if err != nil {
return nil, err
}
// The simple case: just copy a single image.
single, err := c.copySingleImage(ctx, c.unparsedToplevel, nil, copySingleImageOptions{requireCompressionFormatMatch: requireCompressionFormatMatch})
if err != nil {
return nil, err
}
copiedManifest = single.manifest
} else if c.options.ImageListSelection == CopySystemImage {
if len(options.EnsureCompressionVariantsExist) > 0 {
return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
}
requireCompressionFormatMatch, err := shouldRequireCompressionFormatMatch(options)
if err != nil {
return nil, err
}
// This is a manifest list, and we weren't asked to copy multiple images. Choose a single image that
// matches the current system to copy, and copy it.
mfest, manifestType, err := c.unparsedToplevel.Manifest(ctx)
if err != nil {
return nil, fmt.Errorf("reading manifest for %s: %w", transports.ImageName(srcRef), err)
}
manifestList, err := internalManifest.ListFromBlob(mfest, manifestType)
if err != nil {
return nil, fmt.Errorf("parsing primary manifest as list for %s: %w", transports.ImageName(srcRef), err)
}
instanceDigest, err := manifestList.ChooseInstanceByCompression(c.options.SourceCtx, c.options.PreferGzipInstances) // try to pick one that matches c.options.SourceCtx
if err != nil {
return nil, fmt.Errorf("choosing an image from manifest list %s: %w", transports.ImageName(srcRef), err)
}
logrus.Debugf("Source is a manifest list; copying (only) instance %s for current system", instanceDigest)
unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest)
single, err := c.copySingleImage(ctx, unparsedInstance, nil, copySingleImageOptions{requireCompressionFormatMatch: requireCompressionFormatMatch})
if err != nil {
return nil, fmt.Errorf("copying system image from manifest list: %w", err)
}
copiedManifest = single.manifest
} else { /* c.options.ImageListSelection == CopyAllImages or c.options.ImageListSelection == CopySpecificImages, */
// If we were asked to copy multiple images and can't, that's an error.
if !supportsMultipleImages(c.dest) {
return nil, fmt.Errorf("copying multiple images: destination transport %q does not support copying multiple images as a group", destRef.Transport().Name())
}
// Copy some or all of the images.
switch c.options.ImageListSelection {
case CopyAllImages:
logrus.Debugf("Source is a manifest list; copying all instances")
case CopySpecificImages:
logrus.Debugf("Source is a manifest list; copying some instances")
}
if copiedManifest, err = c.copyMultipleImages(ctx); err != nil {
return nil, err
}
}
if err := c.dest.Commit(ctx, c.unparsedToplevel); err != nil {
return nil, fmt.Errorf("committing the finished image: %w", err)
}
return copiedManifest, nil
}
// Printf writes a formatted string to c.reportWriter.
// Note that the method name Printf is not entirely arbitrary: (go tool vet)
// has a built-in list of functions/methods (whatever object they are for)
// which have their format strings checked; for other names we would have
// to pass a parameter to every (go tool vet) invocation.
func (c *copier) Printf(format string, a ...any) {
fmt.Fprintf(c.reportWriter, format, a...)
}
// close tears down state owned by copier.
func (c *copier) close() {
for i, s := range c.signersToClose {
if err := s.Close(); err != nil {
logrus.Warnf("Error closing per-copy signer %d: %v", i+1, err)
}
}
}
// validateImageListSelection returns an error if the passed-in value is not one that we recognize as a valid ImageListSelection value
func validateImageListSelection(selection ImageListSelection) error {
switch selection {
case CopySystemImage, CopyAllImages, CopySpecificImages:
return nil
default:
return fmt.Errorf("Invalid value for options.ImageListSelection: %d", selection)
}
}
// Checks if the destination supports accepting multiple images by checking if it can support
// manifest types that are lists of other manifests.
func supportsMultipleImages(dest types.ImageDestination) bool {
mtypes := dest.SupportedManifestMIMETypes()
if len(mtypes) == 0 {
// Anything goes!
return true
}
return slices.ContainsFunc(mtypes, manifest.MIMETypeIsMultiImage)
}
// isTTY returns true if the io.Writer is a file and a tty.
func isTTY(w io.Writer) bool {
if f, ok := w.(*os.File); ok {
return term.IsTerminal(int(f.Fd()))
}
return false
}
|