1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
|
"""Abstraction for optimizers.
It is sufficient that one re-implements the base estimator.
"""
import numbers
import warnings
try:
from collections.abc import Iterable
except ImportError:
from collections.abc import Iterable
from ..callbacks import VerboseCallback, check_callback
from ..utils import eval_callbacks
from .optimizer import Optimizer
def base_minimize(
func,
dimensions,
base_estimator,
n_calls=100,
n_random_starts=None,
n_initial_points=10,
initial_point_generator="random",
acq_func="EI",
acq_optimizer="lbfgs",
x0=None,
y0=None,
random_state=None,
verbose=False,
callback=None,
n_points=10000,
n_restarts_optimizer=5,
xi=0.01,
kappa=1.96,
n_jobs=1,
model_queue_size=None,
space_constraint=None,
):
"""Base optimizer class.
Parameters
----------
func : callable
Function to minimize. Should take a single list of parameters
and return the objective value.
If you have a search-space where all dimensions have names,
then you can use :func:`skopt.utils.use_named_args` as a decorator
on your objective function, in order to call it directly
with the named arguments. See `use_named_args` for an example.
dimensions : list, shape (n_dims,)
List of search space dimensions.
Each search dimension can be defined either as
- a `(lower_bound, upper_bound)` tuple (for `Real` or `Integer`
dimensions),
- a `(lower_bound, upper_bound, "prior")` tuple (for `Real`
dimensions),
- as a list of categories (for `Categorical` dimensions), or
- an instance of a `Dimension` object (`Real`, `Integer` or
`Categorical`).
.. note:: The upper and lower bounds are inclusive for `Integer`
dimensions.
base_estimator : sklearn regressor
Should inherit from `sklearn.base.RegressorMixin`.
In addition, should have an optional `return_std` argument,
which returns `std(Y | x)` along with `E[Y | x]`.
n_calls : int, default: 100
Maximum number of calls to `func`. An objective function will
always be evaluated this number of times; Various options to
supply initialization points do not affect this value.
n_random_starts : int, default: None
Number of evaluations of `func` with random points before
approximating it with `base_estimator`.
.. deprecated:: 0.8
use `n_initial_points` instead.
n_initial_points : int, default: 10
Number of evaluations of `func` with initialization points
before approximating it with `base_estimator`. Initial point
generator can be changed by setting `initial_point_generator`.
initial_point_generator : str, InitialPointGenerator instance, \
default: `"random"`
Sets a initial points generator. Can be either
- `"random"` for uniform random numbers,
- `"sobol"` for a Sobol' sequence,
- `"halton"` for a Halton sequence,
- `"hammersly"` for a Hammersly sequence,
- `"lhs"` for a latin hypercube sequence,
- `"grid"` for a uniform grid sequence
acq_func : string, default: `"EI"`
Function to minimize over the posterior distribution. Can be either
- `"LCB"` for lower confidence bound,
- `"EI"` for negative expected improvement,
- `"PI"` for negative probability of improvement.
- `"EIps"` for negated expected improvement per second to take into
account the function compute time. Then, the objective function is
assumed to return two values, the first being the objective value and
the second being the time taken in seconds.
- `"PIps"` for negated probability of improvement per second. The
return type of the objective function is assumed to be similar to
that of `"EIps"`
acq_optimizer : string, `"sampling"` or `"lbfgs"`, default: `"lbfgs"`
Method to minimize the acquisition function. The fit model
is updated with the optimal value obtained by optimizing `acq_func`
with `acq_optimizer`.
- If set to `"sampling"`, then `acq_func` is optimized by computing
`acq_func` at `n_points` randomly sampled points and the smallest
value found is used.
- If set to `"lbfgs"`, then
- The `n_restarts_optimizer` no. of points which the acquisition
function is least are taken as start points.
- `"lbfgs"` is run for 20 iterations with these points as initial
points to find local minima.
- The optimal of these local minima is used to update the prior.
x0 : list, list of lists or `None`
Initial input points.
- If it is a list of lists, use it as a list of input points. If no
corresponding outputs `y0` are supplied, then len(x0) of total
calls to the objective function will be spent evaluating the points
in `x0`. If the corresponding outputs are provided, then they will
be used together with evaluated points during a run of the algorithm
to construct a surrogate.
- If it is a list, use it as a single initial input point. The
algorithm will spend 1 call to evaluate the initial point, if the
outputs are not provided.
- If it is `None`, no initial input points are used.
y0 : list, scalar or `None`
Objective values at initial input points.
- If it is a list, then it corresponds to evaluations of the function
at each element of `x0` : the i-th element of `y0` corresponds
to the function evaluated at the i-th element of `x0`.
- If it is a scalar, then it corresponds to the evaluation of the
function at `x0`.
- If it is None and `x0` is provided, then the function is evaluated
at each element of `x0`.
random_state : int, RandomState instance, or None (default)
Set random state to something other than None for reproducible
results.
verbose : boolean, default: False
Control the verbosity. It is advised to set the verbosity to True
for long optimization runs.
callback : callable, list of callables, optional
If callable then `callback(res)` is called after each call to `func`.
If list of callables, then each callable in the list is called.
n_points : int, default: 10000
If `acq_optimizer` is set to `"sampling"`, then `acq_func` is
optimized by computing `acq_func` at `n_points` randomly sampled
points.
n_restarts_optimizer : int, default: 5
The number of restarts of the optimizer when `acq_optimizer`
is `"lbfgs"`.
xi : float, default: 0.01
Controls how much improvement one wants over the previous best
values. Used when the acquisition is either `"EI"` or `"PI"`.
kappa : float, default: 1.96
Controls how much of the variance in the predicted values should be
taken into account. If set to be very high, then we are favouring
exploration over exploitation and vice versa.
Used when the acquisition is `"LCB"`.
n_jobs : int, default: 1
Number of cores to run in parallel while running the lbfgs
optimizations over the acquisition function and given to
the base_estimator. Valid only when
`acq_optimizer` is set to "lbfgs". or when the base_estimator
supports n_jobs as parameter and was given as string.
Defaults to 1 core. If `n_jobs=-1`, then number of jobs is set
to number of cores.
model_queue_size : int or None, default: None
Keeps list of models only as long as the argument given. In the
case of None, the list has no capped length.
space_constraint : callable or None, default: None
Constraint function. Should take a single list of parameters
(i.e. a point in space) and return True if the point satisfies
the constraints.
If None, the space is not conditionally constrained.
Returns
-------
res : `OptimizeResult`, scipy object
The optimization result returned as a OptimizeResult object.
Important attributes are:
- `x` [list]: location of the minimum.
- `fun` [float]: function value at the minimum.
- `models`: surrogate models used for each iteration.
- `x_iters` [list of lists]: location of function evaluation for each
iteration.
- `func_vals` [array]: function value for each iteration.
- `space` [Space]: the optimization space.
- `specs` [dict]`: the call specifications.
- `rng` [RandomState instance]: State of the random state
at the end of minimization.
For more details related to the OptimizeResult object, refer
http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.OptimizeResult.html
"""
specs = {"args": locals(), "function": "base_minimize"}
acq_optimizer_kwargs = {
"n_points": n_points,
"n_restarts_optimizer": n_restarts_optimizer,
"n_jobs": n_jobs,
}
acq_func_kwargs = {"xi": xi, "kappa": kappa}
# Initialize optimization
# Suppose there are points provided (x0 and y0), record them
# check x0: list-like, requirement of minimal points
if x0 is None:
x0 = []
elif not isinstance(x0[0], (list, tuple)):
x0 = [x0]
if not isinstance(x0, list):
raise ValueError("`x0` should be a list, but got %s" % type(x0))
# Check `n_random_starts` deprecation first
if n_random_starts is not None:
warnings.warn(
(
"n_random_starts will be removed in favour of "
"n_initial_points. It overwrites n_initial_points."
),
DeprecationWarning,
)
n_initial_points = n_random_starts
if n_initial_points <= 0 and not x0:
raise ValueError("Either set `n_initial_points` > 0," " or provide `x0`")
# check y0: list-like, requirement of maximal calls
if isinstance(y0, Iterable):
y0 = list(y0)
elif isinstance(y0, numbers.Number):
y0 = [y0]
required_calls = n_initial_points + (len(x0) if not y0 else 0)
if n_calls < required_calls:
raise ValueError("Expected `n_calls` >= %d, got %d" % (required_calls, n_calls))
# calculate the total number of initial points
n_random = n_initial_points
n_initial_points = n_initial_points + len(x0)
# Build optimizer
# create optimizer class
optimizer = Optimizer(
dimensions,
base_estimator,
n_initial_points=n_initial_points,
initial_point_generator=initial_point_generator,
n_jobs=n_jobs,
acq_func=acq_func,
acq_optimizer=acq_optimizer,
random_state=random_state,
model_queue_size=model_queue_size,
space_constraint=space_constraint,
acq_optimizer_kwargs=acq_optimizer_kwargs,
acq_func_kwargs=acq_func_kwargs,
)
# check x0: element-wise data type, dimensionality
assert all(isinstance(p, Iterable) for p in x0)
if not all(len(p) == optimizer.space.n_dims for p in x0):
raise RuntimeError(
"Optimization space (%s) and initial points in x0 "
"use inconsistent dimensions." % optimizer.space
)
# check callback
callbacks = check_callback(callback)
if verbose:
callbacks.append(
VerboseCallback(
n_init=len(x0) if not y0 else 0,
n_random=n_random,
n_total=n_calls,
)
)
# Record provided points
# create return object
result = None
# evaluate y0 if only x0 is provided
if x0 and y0 is None:
y0 = list(map(func, x0))
n_calls -= len(y0)
# record through tell function
if x0:
if not (isinstance(y0, Iterable) or isinstance(y0, numbers.Number)):
raise ValueError(
"`y0` should be an iterable or a scalar, got %s" % type(y0)
)
if len(x0) != len(y0):
raise ValueError("`x0` and `y0` should have the same length")
result = optimizer.tell(x0, y0)
result.specs = specs
if eval_callbacks(callbacks, result):
return result
# Optimize
for _ in range(n_calls):
next_x = optimizer.ask()
next_y = func(next_x)
result = optimizer.tell(next_x, next_y)
result.specs = specs
if eval_callbacks(callbacks, result):
break
return result
|