File: pull.py

package info (click to toggle)
charliecloud 0.43-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 3,084 kB
  • sloc: python: 6,021; sh: 4,284; ansic: 3,863; makefile: 598
file content (331 lines) | stat: -rw-r--r-- 12,893 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
import json
import os
import os.path

import charliecloud as ch
import build_cache as bu
import image as im
import registry as rg


## Constants ##

# Internal library of manifests, e.g. for “FROM scratch” (issue #1013).
manifests_internal = {
   "scratch": {  # magic empty image
      "schemaVersion": 2,
      "config": { "digest": None },
      "layers": []
   }
}


## Main ##

def main(cli):
   # Set things up.
   src_ref = im.Reference(cli.source_ref)
   dst_ref = src_ref if cli.dest_ref is None else im.Reference(cli.dest_ref)
   if (cli.parse_only):
      print(src_ref.as_verbose_str)
      ch.exit(0)
   if (ch.xattrs_save):
      ch.WARNING("--xattrs unsupported for “ch-image pull” (see FAQ)")
   dst_img = im.Image(dst_ref)
   ch.INFO("pulling image:    %s" % src_ref)
   if (src_ref != dst_ref):
      ch.INFO("destination:      %s" % dst_ref)
   ch.INFO("requesting arch:  %s" % ch.arch)
   bu.cache.pull_eager(dst_img, src_ref, cli.last_layer)
   ch.done_notify()


## Classes ##

class Image_Puller:

   __slots__ = ("architectures",  # key: architecture, value: manifest digest
                "config_hash",
                "digests",
                "image",
                "layer_hashes",
                "registry",
                "sid_input",
                "src_ref")

   def __init__(self, image, src_ref):
      self.architectures = None
      self.config_hash = None
      self.digests = dict()
      self.image = image
      self.layer_hashes = None
      self.registry = rg.HTTP(src_ref)
      self.sid_input = None
      self.src_ref = src_ref

   @property
   def config_path(self):
      if (self.config_hash is None):
         return None
      else:
         return ch.storage.download_cache // (self.config_hash + ".json")

   @property
   def fatman_path(self):
      return ch.storage.fatman_for_download(self.image.ref)

   @property
   def manifest_path(self):
      if (str(self.image.ref) in manifests_internal):
         return "[internal library]"
      else:
         if (ch.arch == "yolo" or self.architectures is None):
            digest = None
         else:
            digest = self.architectures[ch.arch]
         return ch.storage.manifest_for_download(self.image.ref, digest)

   def done(self):
      self.registry.close()

   def download(self):
      "Download image metadata and layers and put them in the download cache."
      # Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
      ch.VERBOSE("downloading image: %s" % self.image)
      have_skinny = False
      try:
         # fat manifest
         if (ch.arch != "yolo"):
            try:
               self.fatman_load()
               if (not self.architectures.in_warn(ch.arch)):
                  ch.FATAL("requested arch unavailable: %s" % ch.arch,
                           ("available: %s"
                            % " ".join(sorted(self.architectures.keys()))))
            except ch.No_Fatman_Error:
               # currently, this error is only raised if we’ve downloaded the
               # skinny manifest.
               have_skinny = True
               if (ch.arch == "amd64"):
                  # We’re guessing that enough arch-unaware images are amd64 to
                  # barge ahead if requested architecture is amd64.
                  ch.arch = "yolo"
                  ch.WARNING("image is architecture-unaware")
                  ch.WARNING("requested arch is amd64; using --arch=yolo")
               else:
                  ch.FATAL("image is architecture-unaware",
                           "consider --arch=yolo")
         # manifest
         self.manifest_load(have_skinny)
      except ch.Image_Unavailable_Error:
         if (ch.user() == "qwofford"):
            h = "Quincy, use --auth!!"
         else:
            h = "if your registry needs authentication, use --auth"
         ch.FATAL("unauthorized or not in registry: %s" % self.registry.ref, h)
      # config
      ch.VERBOSE("config path: %s" % self.config_path)
      if (self.config_path is not None):
         if (os.path.exists(self.config_path) and ch.dlcache_p):
            ch.INFO("config: using existing file")
         else:
            self.registry.blob_to_file(self.config_hash, self.config_path,
                                       "config: downloading")
      # layers
      for (i, lh) in enumerate(self.layer_hashes, start=1):
         path = self.layer_path(lh)
         ch.VERBOSE("layer path: %s" % path)
         msg = "layer %d/%d: %s" % (i, len(self.layer_hashes), lh[:7])
         if (os.path.exists(path) and ch.dlcache_p):
            ch.INFO("%s: using existing file" % msg)
         else:
            self.registry.blob_to_file(lh, path, "%s: downloading" % msg)
      # done
      self.registry.close()

   def error_decode(self, data):
      """Decode first error message in registry error blob and return a tuple
         (code, message)."""
      try:
         code = data["errors"][0]["code"]
         msg = data["errors"][0]["message"]
      except (IndexError, KeyError):
         ch.FATAL("malformed error data", "yes, this is ironic")
      return (code, msg)

   def fatman_load(self):
      """Download the fat manifest and load it. If the image has a fat manifest
         populate self.architectures; this may be an empty dictionary if no
         valid architectures were found.

         Raises:

           * Image_Unavailable_Error if the image does not exist or we are not
             authorized to have it.

           * No_Fatman_Error if the image exists but has no fat manifest,
             i.e., is architecture-unaware. In this case self.architectures is
             set to None."""
      self.architectures = None
      if (str(self.src_ref) in manifests_internal):
         # cheat; internal manifest library matches every architecture
         self.architectures = ch.Arch_Dict({ ch.arch_host: None })
         # Assume that image has no digest. This is a kludge, but it makes my
         # solution to issue #1365 work so ¯\_(ツ)_/¯
         self.digests[ch.arch_host] = "no digest"
         return
      # raises Image_Unavailable_Error if needed
      self.registry.fatman_to_file(self.fatman_path,
                                   "manifest list: downloading")
      fm = self.fatman_path.json_from_file("fat manifest")
      if ("layers" in fm or "fsLayers" in fm):
         # Check for skinny manifest. If not present, create a symlink to the
         # “fat manifest” with the conventional name for a skinny manifest.
         # This works because the file we just saved as the “fat manifest” is
         # actually a misleadingly named skinny manifest. Link is relative to
         # avoid embedding the storage directory path within the storage
         # directory (see PR #1657).
         if (not self.manifest_path.exists()):
            self.manifest_path.symlink_to(self.fatman_path.name)
         raise ch.No_Fatman_Error()
      if ("errors" in fm):
         # fm is an error blob.
         (code, msg) = self.error_decode(fm)
         if (code == "MANIFEST_UNKNOWN"):
            ch.INFO("manifest list: no such image")
            return
         else:
            ch.FATAL("manifest list: error: %s" % msg)
      self.architectures = ch.Arch_Dict()
      if ("manifests" not in fm):
         ch.FATAL("manifest list has no key 'manifests'")
      for m in fm["manifests"]:
         try:
            if (m["platform"]["os"] != "linux"):
               continue
            arch = m["platform"]["architecture"]
            if ("variant" in m["platform"]):
               arch = "%s/%s" % (arch, m["platform"]["variant"])
            digest = m["digest"]
         except KeyError:
            ch.FATAL("manifest lists missing a required key")
         if (arch in self.architectures):
            ch.FATAL("manifest list: duplicate architecture: %s" % arch)
         self.architectures[arch] = ch.digest_trim(digest)
         self.digests[arch] = digest.split(":")[1]
      if (len(self.architectures) == 0):
         ch.WARNING("no valid architectures found")

   def layer_path(self, layer_hash):
      "Return the path to tarball for layer layer_hash."
      return ch.storage.download_cache // (layer_hash + ".tar.gz")

   def manifest_digest_by_arch(self):
      "Return skinny manifest digest for target architecture."
      fatman  = self.fat_manifest_path.json_from_file()
      arch    = None
      digest  = None
      variant = None
      try:
         arch, variant = ch.arch.split("/", maxsplit=1)
      except ValueError:
         arch = ch.arch
      if ("manifests" not in fatman):
         ch.FATAL("manifest list has no manifests")
      for k in fatman["manifests"]:
         if (k.get('platform').get('os') != 'linux'):
            continue
         elif (    k.get('platform').get('architecture') == arch
               and (   variant is None
                    or k.get('platform').get('variant') == variant)):
            digest = k.get('digest')
      if (digest is None):
         ch.FATAL("arch not found for image: %s" % arch,
                  'try "ch-image list IMAGE_REF"')
      return digest

   def manifest_load(self, have_skinny=False):
      """Download the manifest file, parse it, and set self.config_hash and
         self.layer_hashes. If the image does not exist,
         exit with error."""
      def bad_key(key):
         ch.FATAL("manifest: %s: no key: %s" % (self.manifest_path, key))
      self.config_hash = None
      self.layer_hashes = None
      # obtain the manifest
      try:
         # internal manifest library, e.g. for “FROM scratch”
         manifest = manifests_internal[str(self.src_ref)]
         ch.INFO("manifest: using internal library")
      except KeyError:
         # download the file and parse it
         if (ch.arch == "yolo" or self.architectures is None):
            digest = None
         else:
            digest = self.architectures[ch.arch]
         ch.DEBUG("manifest digest: %s" % digest)
         if (not have_skinny):
            self.registry.manifest_to_file(self.manifest_path,
                                          "manifest: downloading",
                                          digest=digest)
         manifest = self.manifest_path.json_from_file("manifest")
      # validate schema version
      try:
         version = manifest['schemaVersion']
      except KeyError:
         bad_key("schemaVersion")
      if (version not in {1,2}):
         ch.FATAL("unsupported manifest schema version: %s" % repr(version))
      # load config hash
      #
      # FIXME: Manifest version 1 does not list a config blob. It does have
      # things (plural) that look like a config at history/v1Compatibility as
      # an embedded JSON string :P but I haven’t dug into it.
      if (version == 1):
         ch.VERBOSE("no config; manifest schema version 1")
         self.config_hash = None
      else:  # version == 2
         try:
            self.config_hash = manifest["config"]["digest"]
            if (self.config_hash is not None):
               self.config_hash = ch.digest_trim(self.config_hash)
         except KeyError:
            bad_key("config/digest")
      # load layer hashes
      if (version == 1):
         key1 = "fsLayers"
         key2 = "blobSum"
      else:  # version == 2
         key1 = "layers"
         key2 = "digest"
      if (key1 not in manifest):
         bad_key(key1)
      self.layer_hashes = list()
      for i in manifest[key1]:
         if (key2 not in i):
            bad_key("%s/%s" % (key1, key2))
         self.layer_hashes.append(ch.digest_trim(i[key2]))
      if (version == 1):
         self.layer_hashes.reverse()
      # Remember State_ID input. We can’t rely on the manifest existing in
      # serialized form (e.g. for internal manifests), so re-serialize.
      self.sid_input = json.dumps(manifest, sort_keys=True)

   def unpack(self, last_layer=None):
      layer_paths = [self.layer_path(h) for h in self.layer_hashes]
      bu.cache.unpack_delete(self.image, missing_ok=True)
      self.image.unpack(layer_paths, last_layer)
      self.image.metadata_replace(self.config_path)
      # Check architecture we got. This is limited because image metadata does
      # not store the variant. Move fast and break things, I guess.
      arch_image = self.image.metadata["arch"] or "unknown"
      arch_short = ch.arch.split("/")[0]
      arch_host_short = ch.arch_host.split("/")[0]
      if (arch_image != "unknown" and arch_image != arch_host_short):
         host_mismatch = " (may not match host %s)" % ch.arch_host
      else:
         host_mismatch = ""
      ch.INFO("image arch: %s%s" % (arch_image, host_mismatch))
      if (ch.arch != "yolo" and arch_short != arch_image):
         ch.WARNING("image architecture does not match requested: %s ≠ %s"
                    % (ch.arch, arch_image))