1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
|
#!/usr/bin/python
# Copyright (c) 2014 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A package is a JSON file describing a list of package archives."""
import json
import os
import posixpath
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
import pynacl.file_tools
import pynacl.gsd_storage
import archive_info
import error
PACKAGE_KEY_ARCHIVES = 'archives'
PACKAGE_KEY_VERSION = 'version'
CURRENT_PACKAGE_VERSION = 1
def ReadPackageFile(package_file):
"""Returns a PackageInfoTuple representation of a JSON package file."""
with open(package_file, 'rt') as f:
json_value = json.load(f)
# TODO(dyen): Support old format temporarily when it was a list of archives.
if isinstance(json_value, list):
return { PACKAGE_KEY_ARCHIVES: json_value, PACKAGE_KEY_VERSION: 0 }
else:
return json_value
def GetFileBaseName(filename):
"""Removes all extensions from a file.
(Note: os.path.splitext() only removes the last extension).
"""
first_ext = filename.find('.')
if first_ext != -1:
filename = filename[:first_ext]
return filename
def GetLocalPackageName(local_package_file):
"""Returns the package name given a local package file."""
package_basename = os.path.basename(local_package_file)
return GetFileBaseName(package_basename)
def GetRemotePackageName(remote_package_file):
"""Returns the package name given a remote posix based package file."""
package_basename = posixpath.basename(remote_package_file)
return GetFileBaseName(package_basename)
def DownloadPackageInfoFiles(local_package_file, remote_package_file,
downloader=None):
"""Downloads all package info files from a downloader.
Downloads a package file from the cloud along with all of the archive
info files. Archive info files are expected to be in a directory with the
name of the package along side the package file. Files will be downloaded
in the same structure.
Args:
local_package_file: Local package file where root file will live.
remote_package_file: Remote package URL to download from.
downloader: Optional downloader if standard HTTP one should not be used.
"""
if downloader is None:
downloader = pynacl.gsd_storage.HttpDownload
pynacl.file_tools.MakeParentDirectoryIfAbsent(local_package_file)
downloader(remote_package_file, local_package_file)
if not os.path.isfile(local_package_file):
raise error.Error('Could not download package file: %s.' %
remote_package_file)
package_data = ReadPackageFile(local_package_file)
archive_list = package_data[PACKAGE_KEY_ARCHIVES]
local_package_name = GetLocalPackageName(local_package_file)
remote_package_name = GetRemotePackageName(remote_package_file)
local_archive_dir = os.path.join(os.path.dirname(local_package_file),
local_package_name)
remote_archive_dir = posixpath.join(posixpath.dirname(remote_package_file),
remote_package_name)
pynacl.file_tools.MakeDirectoryIfAbsent(local_archive_dir)
for archive in archive_list:
archive_file = archive + '.json'
local_archive_file = os.path.join(local_archive_dir, archive_file)
remote_archive_file = posixpath.join(remote_archive_dir, archive_file)
downloader(remote_archive_file, local_archive_file)
if not os.path.isfile(local_archive_file):
raise error.Error('Could not download archive file: %s.' %
remote_archive_file)
def UploadPackageInfoFiles(storage, package_target, package_name,
remote_package_file, local_package_file,
skip_missing=False, annotate=False):
"""Uploads all package info files from a downloader.
Uploads a package file to the cloud along with all of the archive info
files. Archive info files are expected to be in a directory with the
name of the package along side the package file. Files will be uploaded
using the same file structure.
Args:
storage: Cloud storage object to store the files to.
remote_package_file: Remote package URL to upload to.
local_package_file: Local package file where root file lives.
skip_missing: Whether to skip missing archive files or error.
annotate: Whether to annotate build bot links.
Returns:
The URL where the root package file is located.
"""
package_data = ReadPackageFile(local_package_file)
archive_list = package_data[PACKAGE_KEY_ARCHIVES]
local_package_name = GetLocalPackageName(local_package_file)
remote_package_name = GetRemotePackageName(remote_package_file)
local_archive_dir = os.path.join(os.path.dirname(local_package_file),
local_package_name)
remote_archive_dir = posixpath.join(posixpath.dirname(remote_package_file),
remote_package_name)
num_archives = len(archive_list)
for index, archive in enumerate(archive_list):
archive_file = archive + '.json'
local_archive_file = os.path.join(local_archive_dir, archive_file)
remote_archive_file = posixpath.join(remote_archive_dir, archive_file)
if skip_missing and not os.path.isfile(local_archive_file):
continue
archive_url = storage.PutFile(local_archive_file, remote_archive_file)
if annotate:
print ('@@@STEP_LINK@download (%s/%s/%s.json [%d/%d])@%s@@@' %
(package_target, package_name, archive, index+1, num_archives,
archive_url))
package_url = storage.PutFile(local_package_file, remote_package_file)
if annotate:
print ('@@@STEP_LINK@download (%s/%s.json)@%s@@@' %
(package_target, package_name, package_url))
return package_url
class PackageInfo(object):
"""A package file is a list of package archives (usually .tar or .tgz files).
PackageInfo will contain a list of ArchiveInfo objects, ArchiveInfo will
contain all the necessary information for an archive (name, URL, hash...etc.).
"""
def __init__(self, package_file=None, skip_missing=False):
self._archive_list = []
self._package_version = CURRENT_PACKAGE_VERSION
if package_file is not None:
self.LoadPackageFile(package_file, skip_missing)
def __eq__(self, other):
if type(self) != type(other):
return False
elif self.GetPackageVersion() != other.GetPackageVersion():
return False
archives1 = [archive.GetArchiveData() for archive in self.GetArchiveList()]
archives2 = [archive.GetArchiveData() for archive in other.GetArchiveList()]
return set(archives1) == set(archives2)
def __repr__(self):
return 'PackageInfo(%s)' % self.DumpPackageJson()
def LoadPackageFile(self, package_file, skip_missing=False):
"""Loads a package file into this object.
Args:
package_file: Filename or JSON dictionary.
"""
archive_names = None
self._archive_list = []
# TODO(dyen): Support old format temporarily when it was a list of archives.
if isinstance(package_file, list) or isinstance(package_file, dict):
if isinstance(package_file, list):
self._package_version = 0
archive_list = package_file
else:
self._package_version = package_file[PACKAGE_KEY_VERSION]
archive_list = package_file[PACKAGE_KEY_ARCHIVES]
if archive_list:
if isinstance(archive_list[0], archive_info.ArchiveInfo):
# Setting a list of ArchiveInfo objects, no need to interpret JSON.
self._archive_list = archive_list
else:
# Assume to be JSON.
for archive_json in archive_list:
archive = archive_info.ArchiveInfo(archive_info_file=archive_json)
self._archive_list.append(archive)
elif isinstance(package_file, basestring):
package_data = ReadPackageFile(package_file)
self._package_version = package_data[PACKAGE_KEY_VERSION]
archive_names = package_data[PACKAGE_KEY_ARCHIVES]
package_name = GetLocalPackageName(package_file)
archive_dir = os.path.join(os.path.dirname(package_file), package_name)
for archive in archive_names:
arch_file = archive + '.json'
arch_path = os.path.join(archive_dir, arch_file)
if not os.path.isfile(arch_path):
if not skip_missing:
raise error.Error(
'Package (%s) points to invalid archive file (%s).' %
(package_file, arch_path))
archive_desc = archive_info.ArchiveInfo(name=archive)
else:
archive_desc = archive_info.ArchiveInfo(archive_info_file=arch_path)
self._archive_list.append(archive_desc)
else:
raise error.Error('Invalid load package file type (%s): %s.' %
(type(package_file), package_file))
def SavePackageFile(self, package_file):
"""Saves this object as a serialized JSON file.
Args:
package_file: File path where JSON file will be saved.
"""
package_name = GetLocalPackageName(package_file)
archive_dir = os.path.join(os.path.dirname(package_file), package_name)
pynacl.file_tools.RemoveDir(archive_dir)
os.makedirs(archive_dir)
archive_list = []
for archive in self.GetArchiveList():
archive_data = archive.GetArchiveData()
archive_list.append(archive_data.name)
archive_file = archive_data.name + '.json'
archive_path = os.path.join(archive_dir, archive_file)
archive.SaveArchiveInfoFile(archive_path)
package_json = {
PACKAGE_KEY_ARCHIVES: archive_list,
PACKAGE_KEY_VERSION: self._package_version
}
with open(package_file, 'wt') as f:
json.dump(package_json, f, sort_keys=True,
indent=2, separators=(',', ': '))
def DumpPackageJson(self):
"""Returns a dictionary representation of the JSON of this object."""
archives = [archive.DumpArchiveJson() for archive in self.GetArchiveList()]
return {
PACKAGE_KEY_ARCHIVES: archives,
PACKAGE_KEY_VERSION: self._package_version
}
def ClearArchiveList(self):
"""Clears this object so it represents no archives."""
self._archive_list = []
def AppendArchive(self, archive_info):
"""Append a package archive into this object"""
self._archive_list.append(archive_info)
def GetArchiveList(self):
"""Returns the sorted list of ARCHIVE_INFOs this object represents."""
return sorted(self._archive_list,
key=lambda archive : archive.GetArchiveData().name)
def GetPackageVersion(self):
"""Returns the version of this package."""
return self._package_version
|