1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
|
import importlib
import inspect
import json
import os
import pkgutil
import traceback
from ._aux import update_sys_path
from .benchmarks import benchmark_types
def _get_benchmark(attr_name, module, klass, func):
"""
Retrieves benchmark function based on attribute name, module, class, and
function.
#### Parameters
**attr_name** (`str`)
: The attribute name of the function.
**module** (module)
: The module where the function resides.
**klass** (class or None)
: The class defining the function, or None if not applicable.
**func** (function)
: The function to be benchmarked.
#### Returns
**benchmark** (Benchmark instance or None)
: A benchmark instance with the name of the benchmark, the function to be
benchmarked, and its sources. Returns None if no matching benchmark is found
or the function is marked to be skipped.
#### Notes
The function tries to get the `benchmark_name` from `func`. If it fails, it
uses `attr_name` to match with the name regex in the benchmark types. If a
match is found, it creates a new benchmark instance and returns it. If no
match is found or the function is marked to be skipped, it returns None.
"""
# Check if the function has been marked to be skipped
if getattr(func, "skip_benchmark", False):
return
try:
name = func.benchmark_name
except AttributeError:
name = None
search = attr_name
else:
search = name.split(".")[-1]
for cls in benchmark_types:
if cls.name_regex.match(search):
break
else:
return
# relative to benchmark_dir
mname_parts = module.__name__.split(".", 1)[1:]
if klass is None:
if name is None:
name = ".".join(mname_parts + [func.__name__])
sources = [func, module]
else:
instance = klass()
func = getattr(instance, attr_name)
if name is None:
name = ".".join(mname_parts + [klass.__name__, attr_name])
sources = [func, instance, module]
return cls(name, func, sources)
def disc_modules(module_name, ignore_import_errors=False):
"""
Recursively imports a module and all sub-modules in the package.
#### Parameters
**module_name** (`str`)
: The name of the module to import.
**ignore_import_errors** (`bool`, optional)
: Whether to ignore import errors. Default is False.
#### Yields
**module** (module)
: The imported module in the package tree.
#### Notes
This function imports the given module and yields it. If `ignore_import_errors`
is set to True, the function will continue executing even if the import fails
and will print the traceback. If `ignore_import_errors` is set to False and
the import fails, the function will raise the error. After yielding the
imported module, the function looks for sub-modules within the package of
the imported module and recursively imports and yields them.
"""
if not ignore_import_errors:
module = importlib.import_module(module_name)
else:
try:
module = importlib.import_module(module_name)
except BaseException:
traceback.print_exc()
return
yield module
if getattr(module, "__path__", None):
for _, name, _ in pkgutil.iter_modules(module.__path__, f"{module_name}."):
yield from disc_modules(name, ignore_import_errors)
def disc_benchmarks(root, ignore_import_errors=False):
"""
Discovers all benchmarks in a given directory tree, yielding Benchmark
objects.
#### Parameters
**root** (`str`)
: The root of the directory tree where the function begins to search for
benchmarks.
**ignore_import_errors** (`bool`, optional)
: Specifies if import errors should be ignored. Default is False.
#### Yields
**benchmark** (Benchmark instance or None)
: A benchmark instance containing the benchmark's name, the function to
be benchmarked, and its sources if a matching benchmark is found.
#### Notes
For each class definition, the function searches for methods with a
specific name. For each free function, it yields all functions with a
specific name. The function initially imports all modules and submodules
in the directory tree using the `disc_modules` function. Then, for each
imported module, it searches for classes and functions that might be
benchmarks. If it finds a class, it looks for methods within that class
that could be benchmarks. If it finds a free function, it considers it as
a potential benchmark. A potential benchmark is confirmed by the
`_get_benchmark` function. If this function returns a benchmark instance,
the instance is yielded.
"""
root_name = os.path.basename(root)
for module in disc_modules(root_name, ignore_import_errors=ignore_import_errors):
for attr_name, module_attr in (
(k, v) for k, v in module.__dict__.items() if not k.startswith("_")
):
if inspect.isclass(module_attr) and not inspect.isabstract(module_attr):
for name, class_attr in inspect.getmembers(module_attr):
if inspect.isfunction(class_attr) or inspect.ismethod(class_attr):
benchmark = _get_benchmark(
name, module, module_attr, class_attr
)
if benchmark is not None:
yield benchmark
elif inspect.isfunction(module_attr):
benchmark = _get_benchmark(attr_name, module, None, module_attr)
if benchmark is not None:
yield benchmark
def get_benchmark_from_name(root, name, extra_params=None):
"""
Creates a benchmark from a fully-qualified benchmark name.
#### Parameters
**root** (`str`)
: Path to the root of a benchmark suite.
**name** (`str`)
: Fully-qualified name of a specific benchmark.
**extra_params** (`dict`, optional)
: Extra parameters to be added to the benchmark.
#### Returns
**benchmark** (Benchmark instance)
: A benchmark instance created from the given fully-qualified benchmark name.
#### Raises
**ValueError**
: If the provided benchmark ID is invalid or if the benchmark could not be found.
#### Notes
This function aims to create a benchmark from the given fully-qualified
name. It splits the name using the "-" character. If "-" is present in the
name, the string after the "-" is converted to an integer and is considered as
the parameter index. If "-" is not present, the parameter index is set to
None. The function then tries to directly import the benchmark function by
guessing its import module name. If the benchmark is not found this way, the
function searches for the benchmark in the directory tree root using
`disc_benchmarks`. If the benchmark is still not found, it raises a
ValueError. If extra parameters are provided, they are added to the
benchmark.
"""
if "-" in name:
try:
name, param_idx = name.split("-", 1)
param_idx = int(param_idx)
except ValueError:
raise ValueError(f"Benchmark id {name!r} is invalid")
else:
param_idx = None
update_sys_path(root)
benchmark = None
# try to directly import benchmark function by guessing its import module name
parts = name.split(".")
for i in [1, 2]:
path = f"{os.path.join(root, *parts[:-i])}.py"
if not os.path.isfile(path):
continue
modname = ".".join([os.path.basename(root)] + parts[:-i])
module = importlib.import_module(modname)
try:
module_attr = getattr(module, parts[-i])
except AttributeError:
break
if i == 1 and inspect.isfunction(module_attr):
benchmark = _get_benchmark(parts[-i], module, None, module_attr)
break
elif i == 2 and inspect.isclass(module_attr):
try:
class_attr = getattr(module_attr, parts[-1])
except AttributeError:
break
if inspect.isfunction(class_attr) or inspect.ismethod(class_attr):
benchmark = _get_benchmark(parts[-1], module, module_attr, class_attr)
break
if benchmark is None:
for benchmark in disc_benchmarks(root):
if benchmark.name == name:
break
else:
raise ValueError(f"Could not find benchmark '{name}'")
if param_idx is not None:
benchmark.set_param_idx(param_idx)
if extra_params:
class ExtraBenchmarkAttrs:
pass
for key, value in extra_params.items():
setattr(ExtraBenchmarkAttrs, key, value)
benchmark._attr_sources.insert(0, ExtraBenchmarkAttrs)
return benchmark
def list_benchmarks(root, fp):
"""
Lists all discovered benchmarks to a file pointer as JSON.
#### Parameters
**root** (`str`)
: Path to the root of a benchmark suite.
**fp** (file object)
: File pointer where the JSON list of benchmarks should be written.
#### Notes
The function updates the system path with the root directory of the
benchmark suite. Then, it iterates over all benchmarks discovered in the
root directory. For each benchmark, it creates a dictionary containing all
attributes of the benchmark that are of types `str`, `int`, `float`, `list`,
`dict`, `bool` and don't start with an underscore `_`. These attribute
dictionaries are then dumped as JSON into the file pointed by `fp`.
"""
update_sys_path(root)
# Streaming of JSON back out to the master process
fp.write("[")
first = True
for benchmark in disc_benchmarks(root):
if not first:
fp.write(", ")
clean = {
k: v
for (k, v) in benchmark.__dict__.items()
if isinstance(v, (str, int, float, list, dict, bool))
and not k.startswith("_")
}
json.dump(clean, fp, skipkeys=True)
first = False
fp.write("]")
def _discover(args):
"""
Discovers all benchmarks in the provided benchmark directory and lists them
to a file.
#### Parameters
**args** (`tuple`)
: A tuple containing benchmark directory and result file path.
#### Notes
The function takes a tuple as an argument. The first element of the tuple
should be the path to the benchmark directory, and the second element should
be the path to the result file. It opens the result file for writing and
calls the `list_benchmarks` function with the benchmark directory and the
file pointer of the result file.
"""
benchmark_dir, result_file = args
with open(result_file, "w") as fp:
list_benchmarks(benchmark_dir, fp)
|