1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
|
// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved.
// This software is licensed under a 3-clause BSD license. Please consult the
// LICENSE.md file distributed with the sources of this project regarding your
// rights to use or distribute this software.
package library
import (
"context"
"fmt"
"os"
"sort"
"strconv"
"strings"
"github.com/sylabs/singularity/v4/internal/pkg/buildcfg"
"github.com/sylabs/singularity/v4/pkg/sylog"
"github.com/sylabs/singularity/v4/pkg/util/singularityconf"
scslibrary "github.com/sylabs/scs-library-client/client"
)
const defaultTag = "latest"
// Default empty description set by older pushes, which is not informative to display
// Comparison will be lower case as the GUI / code has used different capitalisation through time.
const noDescription = "no description"
func splitHostNameAndPath(ref string) (string, string) {
ref = strings.TrimPrefix(ref, "library://")
if strings.Count(ref, "/") <= 2 {
// handle "library://container[:tag]", "library://collection/container[:tag]", or "library://entity/collection/container[:tag]"
return "", ref
}
// handle "library://hostname/entity/collection/container[:tag]"
c := strings.SplitN(ref, "/", 2)
return c[0], c[1]
}
// NormalizeLibraryRef parses library ref and sets default tag, if necessary.
func NormalizeLibraryRef(ref string) (*scslibrary.Ref, error) {
host, pathref := splitHostNameAndPath(ref)
elem := strings.SplitN(pathref, ":", 2)
var tags []string
if len(elem) == 2 {
tags = strings.Split(elem[1], ",")
} else {
tags = []string{defaultTag}
}
return &scslibrary.Ref{Host: host, Path: elem[0], Tags: tags}, nil
}
func getEnvInt(key string, defval int64) int64 {
if env := os.Getenv(key); env != "" {
if n, err := strconv.ParseInt(env, 10, 0); err == nil {
return n
}
sylog.Warningf("Error parsing %s; using default (%d)", key, defval)
}
return defval
}
func getDownloadConfig() (scslibrary.Downloader, error) {
// get downloader parameters from config
conf := singularityconf.GetCurrentConfig()
if conf == nil {
// sylog.Fatalf("Unable to get singularity configuration")
var err error
conf, err = singularityconf.Parse(buildcfg.SINGULARITY_CONF_FILE)
if err != nil {
sylog.Fatalf("unable to parse singularity.conf file: %s", err)
}
}
concurrency := int64(getEnvInt("SINGULARITY_DOWNLOAD_CONCURRENCY", int64(conf.DownloadConcurrency)))
partSize := int64(getEnvInt("SINGULARITY_DOWNLOAD_PART_SIZE", int64(conf.DownloadPartSize)))
bufferSize := int64(getEnvInt("SINGULARITY_DOWNLOAD_BUFFER_SIZE", int64(conf.DownloadBufferSize)))
if concurrency < 1 {
return scslibrary.Downloader{}, fmt.Errorf("invalid download concurrency value (%v)", concurrency)
}
if partSize < 1 {
return scslibrary.Downloader{}, fmt.Errorf("invalid concurrent download part size (%v)", partSize)
}
if bufferSize < 1 {
return scslibrary.Downloader{}, fmt.Errorf("invalid concurrent download buffer size (%v)", bufferSize)
}
return scslibrary.Downloader{
Concurrency: uint(concurrency),
PartSize: partSize,
BufferSize: bufferSize,
}, nil
}
// DownloadImage is a helper function to wrap library image download operation
func DownloadImage(ctx context.Context, c *scslibrary.Client, imagePath, arch string, libraryRef *scslibrary.Ref, pb scslibrary.ProgressBar) error {
// open destination file for writing
f, err := os.OpenFile(imagePath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o777)
if err != nil {
return fmt.Errorf("error opening file %s for writing: %v", imagePath, err)
}
defer f.Close()
var tag string
if len(libraryRef.Tags) > 0 {
tag = libraryRef.Tags[0]
}
spec, err := getDownloadConfig()
if err != nil {
return err
}
// call library client to download image
if err := c.ConcurrentDownloadImage(ctx, f, arch, libraryRef.Path, tag, &spec, pb); err != nil {
// Delete incomplete image file in the event of failure
// we get here e.g. if the context is canceled by Ctrl-C
sylog.Debugf("Cleaning up incomplete download: %s", imagePath)
if err := os.Remove(imagePath); err != nil {
sylog.Errorf("Error while removing incomplete download: %v", err)
}
return fmt.Errorf("error downloading image: %v", err)
}
return nil
}
// DownloadImageNoProgress downloads an image from the library without
// displaying a progress bar while doing so
func DownloadImageNoProgress(ctx context.Context, c *scslibrary.Client, imagePath, arch string, libraryRef *scslibrary.Ref) error {
return DownloadImage(ctx, c, imagePath, arch, libraryRef, nil)
}
// SearchLibrary searches the library and outputs results to stdout
func SearchLibrary(ctx context.Context, c *scslibrary.Client, value, arch string, signed bool) error {
if len(value) < 3 {
return fmt.Errorf("bad query '%s'. You must search for at least 3 characters", value)
}
searchSpec := map[string]string{
"value": value,
"arch": arch,
}
if signed {
searchSpec["signed"] = "true"
}
results, err := c.Search(ctx, searchSpec)
if err != nil {
return err
}
numImages := len(results.Images)
if numImages > 0 {
imageList := []string{}
for _, img := range results.Images {
tagSpec := img.Hash
if len(img.Tags) > 0 {
tagSpec = strings.Join(img.Tags, ",")
}
imageItem := fmt.Sprintf("\tlibrary://%s/%s/%s:%s", img.EntityName, img.CollectionName, img.ContainerName, tagSpec)
if img.Description != "" && strings.ToLower(img.Description) != noDescription {
imageItem = imageItem + fmt.Sprintf("\n\t\t%s", img.Description)
}
if len(img.Fingerprints) > 0 {
imageItem = imageItem + fmt.Sprintf("\n\t\tSigned by: %s", strings.Join(img.Fingerprints, ","))
}
imageList = append(imageList, imageItem)
}
sort.Strings(imageList)
fmt.Printf("Found %d container images for %s matching %q:\n\n", numImages, arch, value)
fmt.Println(strings.Join(imageList, "\n\n"))
fmt.Printf("\n")
} else {
fmt.Printf("No container images found for %s matching %q.\n\n", arch, value)
}
return nil
}
|