1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Replace localized parts of a packaged directory with data from a langpack
directory.
"""
import json
import os
from createprecomplete import generate_precomplete
import mozpack.path as mozpath
from mozpack.chrome.manifest import (
Manifest,
ManifestChrome,
ManifestEntryWithRelPath,
ManifestLocale,
is_manifest,
)
from mozpack.copier import FileCopier, Jarrer
from mozpack.errors import errors
from mozpack.files import ComposedFinder, GeneratedFile, ManifestFile
from mozpack.mozjar import JAR_DEFLATED
from mozpack.packager import Component, SimpleManifestSink, SimplePackager
from mozpack.packager.formats import FlatFormatter, JarFormatter, OmniJarFormatter
from mozpack.packager.unpack import UnpackFinder
class LocaleManifestFinder:
def __init__(self, finder):
entries = self.entries = []
bases = self.bases = []
class MockFormatter:
def add_interfaces(self, path, content):
pass
def add(self, path, content):
pass
def add_manifest(self, entry):
if entry.localized:
entries.append(entry)
def add_base(self, base, addon=False):
bases.append(base)
# SimplePackager rejects "manifest foo.manifest" entries with
# additional flags (such as "manifest foo.manifest application=bar").
# Those type of entries are used by language packs to work as addons,
# but are not necessary for the purpose of l10n repacking. So we wrap
# the finder in order to remove those entries.
class WrapFinder:
def __init__(self, finder):
self._finder = finder
def find(self, pattern):
for p, f in self._finder.find(pattern):
if isinstance(f, ManifestFile):
unwanted = [
e for e in f._entries if isinstance(e, Manifest) and e.flags
]
if unwanted:
f = ManifestFile(
f._base, [e for e in f._entries if e not in unwanted]
)
yield p, f
sink = SimpleManifestSink(WrapFinder(finder), MockFormatter())
sink.add(Component(""), "*")
sink.close(False)
# Find unique locales used in these manifest entries.
self.locales = list(
set(e.id for e in self.entries if isinstance(e, ManifestLocale))
)
class L10NRepackFormatterMixin:
def __init__(self, *args, **kwargs):
super(L10NRepackFormatterMixin, self).__init__(*args, **kwargs)
self._dictionaries = {}
def add(self, path, file):
base, relpath = self._get_base(path)
if path.endswith(".dic"):
if relpath.startswith("dictionaries/"):
root, ext = mozpath.splitext(mozpath.basename(path))
self._dictionaries[root] = path
elif path.endswith("/built_in_addons.json"):
data = json.loads(file.open().read())
data["dictionaries"] = self._dictionaries
# The GeneratedFile content is only really generated after
# all calls to formatter.add.
file = GeneratedFile(lambda: json.dumps(data))
elif relpath.startswith("META-INF/"):
# Ignore signatures inside omnijars. We drop these items: if we
# don't treat them as omnijar resources, they will be included in
# the top-level package, and that's not how omnijars are signed (Bug
# 1750676). If we treat them as omnijar resources, they will stay
# in the omnijar, as expected -- but the signatures won't be valid
# after repacking. Therefore, drop them.
return
super(L10NRepackFormatterMixin, self).add(path, file)
def L10NRepackFormatter(klass):
class L10NRepackFormatter(L10NRepackFormatterMixin, klass):
pass
return L10NRepackFormatter
FlatFormatter = L10NRepackFormatter(FlatFormatter)
JarFormatter = L10NRepackFormatter(JarFormatter)
OmniJarFormatter = L10NRepackFormatter(OmniJarFormatter)
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1:
errors.fatal("Multiple app locales aren't supported: " + ",".join(app.locales))
if len(l10n.locales) > 1:
errors.fatal(
"Multiple l10n locales aren't supported: " + ",".join(l10n.locales)
)
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
# As there may be multiple locale entries with the same base, but with
# different flags, that tracking takes the flags into account when there
# are some. Example:
# locale foo en-US path/to/files/win os=Win
# locale foo en-US path/to/files/mac os=Darwin
def key(entry):
if entry.flags:
return "%s %s" % (entry.name, entry.flags)
return entry.name
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][key(e)] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if key(e) not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][key(e)]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base], subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not files:
if base not in non_chrome:
finderBase = ""
if hasattr(l10n_finder, "base"):
finderBase = l10n_finder.base
errors.error("Missing file: %s" % os.path.join(finderBase, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, "chrome.%s.manifest" % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, "chrome.manifest"), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Resources in `localization` directories are packaged from the source and then
# if localized versions are present in the l10n dir, we package them as well
# keeping the source dir resources as a runtime fallback.
for p, f in l10n_finder.find("**/localization"):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.items():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def repack(
source, l10n, extra_l10n={}, non_resources=[], non_chrome=set(), minify=False
):
"""
Replace localized data from the `source` directory with localized data
from `l10n` and `extra_l10n`.
The `source` argument points to a directory containing a packaged
application (in omnijar, jar or flat form).
The `l10n` argument points to a directory containing the main localized
data (usually in the form of a language pack addon) to use to replace
in the packaged application.
The `extra_l10n` argument contains a dict associating relative paths in
the source to separate directories containing localized data for them.
This can be used to point at different language pack addons for different
parts of the package application.
The `non_resources` argument gives a list of relative paths in the source
that should not be added in an omnijar in case the packaged application
is in that format.
The `non_chrome` argument gives a list of file/directory patterns for
localized files that are not listed in a chrome.manifest.
If `minify`, `.properties` files are minified.
"""
app_finder = UnpackFinder(source, minify=minify)
l10n_finder = UnpackFinder(l10n, minify=minify)
if extra_l10n:
finders = {
"": l10n_finder,
}
for base, path in extra_l10n.items():
finders[base] = UnpackFinder(path, minify=minify)
l10n_finder = ComposedFinder(finders)
copier = FileCopier()
compress = min(app_finder.compressed, JAR_DEFLATED)
if app_finder.kind == "flat":
formatter = FlatFormatter(copier)
elif app_finder.kind == "jar":
formatter = JarFormatter(copier, compress=compress)
elif app_finder.kind == "omni":
formatter = OmniJarFormatter(
copier, app_finder.omnijar, compress=compress, non_resources=non_resources
)
with errors.accumulate():
_repack(app_finder, l10n_finder, copier, formatter, non_chrome)
copier.copy(source, skip_if_older=False)
generate_precomplete(source)
|