1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import importlib
import inspect
import pathlib
BUILTIN_COMPARATORS = {}
class ComparatorNotFound(Exception):
"""Raised when we can't find the specified comparator.
Triggered when either the comparator name is incorrect for a builtin one,
or when a path to a specified comparator cannot be found.
"""
pass
class GithubRequestFailure(Exception):
"""Raised when we hit a failure during PR link parsing."""
pass
class BadComparatorArgs(Exception):
"""Raised when the args given to the comparator are incorrect."""
pass
def comparator(comparator_klass):
BUILTIN_COMPARATORS[comparator_klass.__name__] = comparator_klass
return comparator_klass
@comparator
class BasePerfComparator:
def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args):
"""Initialize the standard/default settings for Comparators.
:param vcs object: Used for updating the local repo.
:param compare_commit str: The base revision found for the local repo.
:param current_revision_ref str: The current revision of the local repo.
:param comparator_args list: List of comparator args in the format NAME=VALUE.
"""
self.vcs = vcs
self.compare_commit = compare_commit
self.current_revision_ref = current_revision_ref
self.comparator_args = comparator_args
# Used to ensure that the local repo gets cleaned up appropriately on failures
self._updated = False
def setup_base_revision(self, extra_args):
"""Setup the base try run/revision.
In this case, we update to the repo to the base revision and
push that to try. The extra_args can be used to set additional
arguments for Raptor (not available for other harnesses).
:param extra_args list: A list of extra arguments to pass to the try tasks.
"""
self.vcs.update(self.compare_commit)
self._updated = True
def teardown_base_revision(self):
"""Teardown the setup for the base revision."""
if self._updated:
self.vcs.update(self.current_revision_ref)
self._updated = False
def setup_new_revision(self, extra_args):
"""Setup the new try run/revision.
Note that the extra_args are reset between the base, and new revision runs.
:param extra_args list: A list of extra arguments to pass to the try tasks.
"""
pass
def teardown_new_revision(self):
"""Teardown the new run/revision setup."""
pass
def teardown(self):
"""Teardown for failures.
This method can be used for ensuring that the repo is cleaned up
when a failure is hit at any point in the process of doing the
new/base revision setups, or the pushes to try.
"""
self.teardown_base_revision()
def get_github_pull_request_info(link):
"""Returns information about a PR link.
This method accepts a Github link in either of these formats:
https://github.com/mozilla-mobile/firefox-android/pull/1627,
https://github.com/mozilla-mobile/firefox-android/pull/1876/commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365 #noqa
and returns the Github link, branch, and revision of the commit.
"""
from urllib.parse import urlparse
import requests
# Parse the url, and get all the necessary info
parsed_url = urlparse(link)
path_parts = parsed_url.path.strip("/").split("/")
owner, repo = path_parts[0], path_parts[1]
pr_number = path_parts[-1]
if "/pull/" not in parsed_url.path:
raise GithubRequestFailure(
f"Link for Github PR is invalid (missing /pull/): {link}"
)
# Get the commit being targeted in the PR
pr_commit = None
if "/commits/" in parsed_url.path:
pr_commit = path_parts[-1]
pr_number = path_parts[-3]
# Make the request, and get the PR info, otherwise,
# raise an exception if the response code is not 200
api_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}"
response = requests.get(api_url)
if response.status_code == 200:
link_info = response.json()
return (
link_info["head"]["repo"]["html_url"],
pr_commit if pr_commit else link_info["head"]["sha"],
link_info["head"]["ref"],
)
raise GithubRequestFailure(
f"The following url returned a non-200 status code: {api_url}"
)
@comparator
class BenchmarkComparator(BasePerfComparator):
def _get_benchmark_info(self, arg_prefix):
# Get the flag from the comparator args
benchmark_info = {"repo": None, "branch": None, "revision": None, "link": None}
for arg in self.comparator_args:
if arg.startswith(arg_prefix):
_, settings = arg.split(arg_prefix)
setting, val = settings.split("=")
if setting not in benchmark_info:
raise BadComparatorArgs(
f"Unknown argument provided `{setting}`. Only the following "
f"are available (prefixed with `{arg_prefix}`): "
f"{list(benchmark_info.keys())}"
)
benchmark_info[setting] = val
# Parse the link for any required information
if benchmark_info.get("link", None) is not None:
(
benchmark_info["repo"],
benchmark_info["revision"],
benchmark_info["branch"],
) = get_github_pull_request_info(benchmark_info["link"])
return benchmark_info
def _setup_benchmark_args(self, extra_args, benchmark_info):
# Setup the arguments for Raptor
extra_args.append(f"benchmark-repository={benchmark_info['repo']}")
extra_args.append(f"benchmark-revision={benchmark_info['revision']}")
if benchmark_info.get("branch", None):
extra_args.append(f"benchmark-branch={benchmark_info['branch']}")
def setup_base_revision(self, extra_args):
"""Sets up the options for a base benchmark revision run.
Checks for a `base-link` in the
command and adds the appropriate commands to the extra_args
which will be added to the PERF_FLAGS environment variable.
If that isn't provided, then you must provide the repo, branch,
and revision directly through these (branch is optional):
base-repo=https://github.com/mozilla-mobile/firefox-android
base-branch=main
base-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365
Otherwise, we'll use the default mach try perf
base behaviour.
TODO: Get the information automatically from a commit link. Github
API doesn't provide the branch name from a link like that.
"""
base_info = self._get_benchmark_info("base-")
# If no options were provided, use the default BasePerfComparator behaviour
if not any(v is not None for v in base_info.values()):
raise BadComparatorArgs(
f"Could not find the correct base-revision arguments in: {self.comparator_args}"
)
self._setup_benchmark_args(extra_args, base_info)
def setup_new_revision(self, extra_args):
"""Sets up the options for a new benchmark revision run.
Same as `setup_base_revision`, except it uses
`new-` as the prefix instead of `base-`.
"""
new_info = self._get_benchmark_info("new-")
# If no options were provided, use the default BasePerfComparator behaviour
if not any(v is not None for v in new_info.values()):
raise BadComparatorArgs(
f"Could not find the correct new-revision arguments in: {self.comparator_args}"
)
self._setup_benchmark_args(extra_args, new_info)
def get_comparator(comparator):
if comparator in BUILTIN_COMPARATORS:
return BUILTIN_COMPARATORS[comparator]
file = pathlib.Path(comparator)
if not file.exists():
raise ComparatorNotFound(
f"Expected either a path to a file containing a comparator, or a "
f"builtin comparator from this list: {BUILTIN_COMPARATORS.keys()}"
)
# Importing a source file directly
spec = importlib.util.spec_from_file_location(name=file.name, location=comparator)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
members = inspect.getmembers(
module,
lambda c: inspect.isclass(c)
and issubclass(c, BasePerfComparator)
and c != BasePerfComparator,
)
if not members:
raise ComparatorNotFound(
f"The path {comparator} was found but it was not a valid comparator. "
f"Ensure it is a subclass of BasePerfComparator and optionally contains the "
f"following methods: "
f"{', '.join(inspect.getmembers(BasePerfComparator, predicate=inspect.ismethod))}"
)
return members[0][-1]
|