1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import datetime
import logging
import mozpack.path as mozpath
from mozbuild.base import MozbuildObject
from mozbuild.util import memoize
from taskgraph.optimize.base import OptimizationStrategy, register_strategy
from taskgraph.optimize.strategies import IndexSearch
from taskgraph.util.parameterization import resolve_timestamps
from taskgraph.util.path import match as match_path
from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint
logger = logging.getLogger(__name__)
@register_strategy("skip-unless-schedules")
class SkipUnlessSchedules(OptimizationStrategy):
@memoize
def scheduled_by_push(self, files_changed):
mbo = MozbuildObject.from_environment()
# the decision task has a sparse checkout, so, mozbuild_reader will use
# a MercurialRevisionFinder with revision '.', which should be the same
# as `revision`; in other circumstances, it will use a default reader
rdr = mbo.mozbuild_reader(config_mode="empty")
components = set()
for p, m in rdr.files_info(files_changed).items():
components |= set(m["SCHEDULES"].components)
return components
def should_remove_task(self, task, params, conditions):
if params.get("pushlog_id") == -1:
return False
scheduled = self.scheduled_by_push(frozenset(params["files_changed"]))
conditions = set(conditions)
# if *any* of the condition components are scheduled, do not optimize
if conditions & scheduled:
return False
return True
@register_strategy("skip-unless-has-relevant-tests")
class SkipUnlessHasRelevantTests(OptimizationStrategy):
"""Optimizes tasks that don't run any tests that were
in child directories of a modified file.
"""
@memoize
def get_changed_dirs(self, files_changed):
changed = map(mozpath.dirname, files_changed)
# Filter out empty directories (from files modified in the root).
# Otherwise all tasks would be scheduled.
return {d for d in changed if d}
def should_remove_task(self, task, params, _):
if not task.attributes.get("test_manifests"):
return True
for d in self.get_changed_dirs(frozenset(params["files_changed"])):
for t in task.attributes["test_manifests"]:
if t.startswith(d):
logger.debug(
f"{task.label} runs a test path ({t}) contained by a modified file ({d})"
)
return False
return True
# TODO: This overwrites upstream Taskgraph's `skip-unless-changed`
# optimization. Once the firefox-android migration is landed and we upgrade
# upstream Taskgraph to a version that doesn't call files_changed.check`, this
# class can be deleted. Also remove the `taskgraph.optimize.base.registry` tweak
# in `gecko_taskgraph.register` at the same time.
@register_strategy("skip-unless-changed")
class SkipUnlessChanged(OptimizationStrategy):
@memoize
def _match_path(self, path, pattern):
return match_path(path, pattern)
def check(self, files_changed, patterns):
"""Optimized check using memoized path matching"""
# Check if any path matches any pattern
# short-circuits on first match via generator
return any(
self._match_path(path, pattern)
for path in files_changed
for pattern in patterns
)
def should_remove_task(self, task, params, file_patterns):
# pushlog_id == -1 - this is the case when run from a cron.yml job or on a git repository
if params.get("repository_type") == "hg" and params.get("pushlog_id") == -1:
return False
changed = self.check(params["files_changed"], file_patterns)
if not changed:
logger.debug(
f'no files found matching a pattern in `skip-unless-changed` for "{task.label}"'
)
return True
return False
register_strategy("skip-unless-mozlint", args=("tools/lint",))(SkipUnlessMozlint)
@register_strategy("skip-unless-missing")
class SkipUnlessMissing(OptimizationStrategy):
"""Skips a task unless it is missing from a specified index.
This simply defers to Taskgraph's `index-search` optimization. The reason
we need this shim is because replacement and removal optimizations can't be
joined together in a composite strategy as removal and replacement happen
at different times.
"""
index_search = IndexSearch()
def _convert_datetime_str(self, dt):
if dt.endswith("Z"):
dt = dt[:-1]
return datetime.datetime.fromisoformat(dt).strftime(self.index_search.fmt)
def should_remove_task(self, task, params, index):
now = datetime.datetime.now(datetime.timezone.utc)
deadline = self._convert_datetime_str(
resolve_timestamps(now, task.task["deadline"])
)
return bool(
self.index_search.should_replace_task(task, params, deadline, [index])
)
|
