1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import subprocess
from datetime import datetime, timedelta
from pathlib import Path
from compare_locales import parser
from compare_locales.lint.linter import L10nLinter
from compare_locales.lint.util import l10n_base_reference_and_tests
from compare_locales.paths import ProjectFiles, TOMLParser
from filelock import FileLock, Timeout
from mach import util as mach_util
from mozfile import which
from mozlint import pathutils, result
from mozpack import path as mozpath
from mozversioncontrol import MissingVCSTool
L10N_SOURCE_NAME = "l10n-source"
L10N_SOURCE_REPO = "https://github.com/mozilla-l10n/firefox-l10n-source.git"
PULL_AFTER = timedelta(days=2)
# Wrapper to call lint_strings with mozilla-central configuration
# comm-central defines its own wrapper since comm-central strings are
# in separate repositories
def lint(paths, lintconfig, **lintargs):
return lint_strings(L10N_SOURCE_NAME, paths, lintconfig, **lintargs)
def lint_strings(name, paths, lintconfig, **lintargs):
l10n_base = mach_util.get_state_dir()
root = lintargs["root"]
exclude = lintconfig.get("exclude")
extensions = lintconfig.get("extensions")
# Load l10n.toml configs
l10nconfigs = load_configs(lintconfig["l10n_configs"], root, l10n_base, name)
# If l10n.yml is included in the provided paths, validate it against the
# TOML files, then remove it to avoid parsing it as a localizable resource.
if lintconfig["path"] in paths:
results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
paths.remove(lintconfig["path"])
lintconfig["include"].remove(mozpath.relpath(lintconfig["path"], root))
else:
results = []
all_files = []
for p in paths:
fp = pathutils.FilterPath(p)
if fp.isdir:
for _, fileobj in fp.finder:
all_files.append(fileobj.path)
if fp.isfile:
all_files.append(p)
# Filter out files explicitly excluded in the l10n.yml configuration.
# `browser/locales/en-US/firefox-l10n.js` is a good example.
all_files, _ = pathutils.filterpaths(
lintargs["root"],
all_files,
lintconfig["include"],
exclude=exclude,
extensions=extensions,
)
# Filter again, our directories might have picked up files that should be
# excluded in l10n.yml
skips = {p for p in all_files if not parser.hasParser(p)}
results.extend(
result.from_config(
lintconfig,
level="warning",
path=path,
message="file format not supported in compare-locales",
)
for path in skips
)
all_files = [p for p in all_files if p not in skips]
files = ProjectFiles(name, l10nconfigs)
get_reference_and_tests = l10n_base_reference_and_tests(files)
linter = MozL10nLinter(lintconfig)
results += linter.lint(all_files, get_reference_and_tests)
return results
# Similar to the lint/lint_strings wrapper setup, for comm-central support.
def source_repo_setup(**lint_args):
state_dir = mach_util.get_state_dir()
lock_file = (Path(state_dir) / L10N_SOURCE_NAME).with_suffix(".lock")
timeout = 60
# In the scenario where multiple processes run this same linter where the l10n source
# has not yet been cloned, we need a lock to prevent them from all trying to clone
# to the same place at the same time.
try:
with FileLock(lock_file, timeout=timeout):
gs = mozpath.join(state_dir, L10N_SOURCE_NAME)
marker = mozpath.join(gs, ".git", "l10n_pull_marker")
try:
last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
skip_clone = datetime.now() < last_pull + PULL_AFTER
except OSError:
skip_clone = False
if skip_clone:
return 0
git = which("git")
if not git:
if os.environ.get("MOZ_AUTOMATION"):
raise MissingVCSTool("Unable to obtain git path.")
print("warning: l10n linter requires Git but was unable to find 'git'")
return 1
# If this is called from a source hook on a git repo, there might be a index
# file listed in the environment as a git operation is ongoing. This seems
# to confuse the git call here into thinking that it is actually operating
# on the main repository, rather than the l10n-source repo. Therefore,
# we remove this environment flag.
if "GIT_INDEX_FILE" in os.environ:
os.environ.pop("GIT_INDEX_FILE")
kwargs = {
"check": False,
"stdout": subprocess.PIPE,
"stderr": subprocess.STDOUT,
}
if os.path.exists(gs):
proc = subprocess.run([git, "pull", L10N_SOURCE_REPO], cwd=gs, **kwargs)
else:
proc = subprocess.run([git, "clone", L10N_SOURCE_REPO, gs], **kwargs)
if proc.returncode != 0:
lint_args["log"].error(
f"Failed to pull {L10N_SOURCE_REPO}:\n{proc.stdout}"
)
return 1
with open(marker, "w") as fh:
fh.flush()
except Timeout:
print(
f"Could not acquire the lock at {lock_file} for the {L10N_SOURCE_NAME} checkout after {timeout} seconds."
)
return 0
def load_configs(l10n_configs, root, l10n_base, locale):
"""Load l10n configuration files specified in the linter configuration."""
configs = []
env = {"l10n_base": l10n_base}
for toml in l10n_configs:
cfg = TOMLParser().parse(
mozpath.join(root, toml), env=env, ignore_missing_includes=True
)
cfg.set_locales([locale], deep=True)
configs.append(cfg)
return configs
def validate_linter_includes(lintconfig, l10nconfigs, lintargs):
"""Check l10n.yml config against l10n.toml configs."""
reference_paths = set(
mozpath.relpath(p["reference"].prefix, lintargs["root"])
for project in l10nconfigs
for config in project.configs
for p in config.paths
)
# Just check for directories
reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p))
missing_in_yml = [
refd for refd in reference_dirs if refd not in lintconfig["include"]
]
# These might be subdirectories in the config, though
missing_in_yml = [
d
for d in missing_in_yml
if not any(d.startswith(parent + "/") for parent in lintconfig["include"])
]
if missing_in_yml:
dirs = ", ".join(missing_in_yml)
return [
result.from_config(
lintconfig,
path=lintconfig["path"],
message="l10n.yml out of sync with l10n.toml, add: " + dirs,
)
]
return []
class MozL10nLinter(L10nLinter):
"""Subclass linter to generate the right result type."""
def __init__(self, lintconfig):
super(MozL10nLinter, self).__init__()
self.lintconfig = lintconfig
def lint(self, files, get_reference_and_tests):
return [
result.from_config(self.lintconfig, **result_data)
for result_data in super(MozL10nLinter, self).lint(
files, get_reference_and_tests
)
]
|
