1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
|
# This mixin provides shared behavior for experiments. Includers must implement
# `enabled?` and `publish(result)`.
#
# Override Scientist::Experiment.new to set your own class which includes and
# implements Scientist::Experiment's interface.
module Scientist::Experiment
# Whether to raise when the control and candidate mismatch.
# If this is nil, raise_on_mismatches class attribute is used instead.
attr_accessor :raise_on_mismatches
def self.included(base)
self.set_default(base) if base.instance_of?(Class)
base.extend RaiseOnMismatch
end
# Instantiate a new experiment (using the class given to the .set_default method).
def self.new(name)
(@experiment_klass || Scientist::Default).new(name)
end
# Configure Scientist to use the given class for all future experiments
# (must implement the Scientist::Experiment interface).
#
# Called automatically when new experiments are defined.
def self.set_default(klass)
@experiment_klass = klass
end
# A mismatch, raised when raise_on_mismatches is enabled.
class MismatchError < Exception
attr_reader :name, :result
def initialize(name, result)
@name = name
@result = result
super "experiment '#{name}' observations mismatched"
end
# The default formatting is nearly unreadable, so make it useful.
#
# The assumption here is that errors raised in a test environment are
# printed out as strings, rather than using #inspect.
def to_s
super + ":\n" +
format_observation(result.control) + "\n" +
result.candidates.map { |candidate| format_observation(candidate) }.join("\n") +
"\n"
end
def format_observation(observation)
observation.name + ":\n" +
if observation.raised?
lines = observation.exception.backtrace.map { |line| " #{line}" }.join("\n")
" #{observation.exception.inspect}" + "\n" + lines
else
" #{observation.cleaned_value.inspect}"
end
end
end
module RaiseOnMismatch
# Set this flag to raise on experiment mismatches.
#
# This causes all science mismatches to raise a MismatchError. This is
# intended for test environments and should not be enabled in a production
# environment.
#
# bool - true/false - whether to raise when the control and candidate mismatch.
def raise_on_mismatches=(bool)
@raise_on_mismatches = bool
end
# Whether or not to raise a mismatch error when a mismatch occurs.
def raise_on_mismatches?
@raise_on_mismatches
end
end
# Define a block of code to run before an experiment begins, if the experiment
# is enabled.
#
# The block takes no arguments.
#
# Returns the configured block.
def before_run(&block)
@_scientist_before_run = block
end
# Define a block of code to run after an experiment completes, if the experiment
# is enabled.
#
# The block takes one argument, the Scientist::Result containing experiment results.
#
# Returns the configured block.
def after_run(&block)
@_scientist_after_run = block
end
# A Hash of behavior blocks, keyed by String name. Register behavior blocks
# with the `try` and `use` methods.
def behaviors
@_scientist_behaviors ||= {}
end
# A block to clean an observed value for publishing or storing.
#
# The block takes one argument, the observed value which will be cleaned.
#
# Returns the configured block.
def clean(&block)
@_scientist_cleaner = block
end
# Accessor for the clean block, if one is available.
#
# Returns the configured block, or nil.
def cleaner
@_scientist_cleaner
end
# Internal: Clean a value with the configured clean block, or return the value
# if no clean block is configured.
#
# Rescues and reports exceptions in the clean block if they occur.
def clean_value(value)
if @_scientist_cleaner
@_scientist_cleaner.call value
else
value
end
rescue StandardError => ex
raised :clean, ex
value
end
# A block which compares two experimental values.
#
# The block must take two arguments, the control value and a candidate value,
# and return true or false.
#
# Returns the block.
def compare(*args, &block)
@_scientist_comparator = block
end
# A block which compares two experimental errors.
#
# The block must take two arguments, the control Error and a candidate Error,
# and return true or false.
#
# Returns the block.
def compare_errors(*args, &block)
@_scientist_error_comparator = block
end
# A Symbol-keyed Hash of extra experiment data.
def context(context = nil)
@_scientist_context ||= {}
@_scientist_context.merge!(context) unless context.nil?
@_scientist_context
end
# Configure this experiment to ignore an observation with the given block.
#
# The block takes two arguments, the control observation and the candidate
# observation which didn't match the control. If the block returns true, the
# mismatch is disregarded.
#
# This can be called more than once with different blocks to use.
def ignore(&block)
@_scientist_ignores ||= []
@_scientist_ignores << block
end
# Internal: ignore a mismatched observation?
#
# Iterates through the configured ignore blocks and calls each of them with
# the given control and mismatched candidate observations.
#
# Returns true or false.
def ignore_mismatched_observation?(control, candidate)
return false unless @_scientist_ignores
@_scientist_ignores.any? do |ignore|
begin
ignore.call control.value, candidate.value
rescue StandardError => ex
raised :ignore, ex
false
end
end
end
# The String name of this experiment. Default is "experiment". See
# Scientist::Default for an example of how to override this default.
def name
"experiment"
end
# Internal: compare two observations, using the configured compare and compare_errors lambdas if present.
def observations_are_equivalent?(a, b)
a.equivalent_to? b, @_scientist_comparator, @_scientist_error_comparator
rescue StandardError => ex
raised :compare, ex
false
end
def raise_with(exception)
@_scientist_custom_mismatch_error = exception
end
# Called when an exception is raised while running an internal operation,
# like :publish. Override this method to track these exceptions. The
# default implementation re-raises the exception.
def raised(operation, error)
raise error
end
# Internal: Run all the behaviors for this experiment, observing each and
# publishing the results. Return the result of the named behavior, default
# "control".
def run(name = nil)
behaviors.freeze
context.freeze
name = (name || "control").to_s
block = behaviors[name]
if block.nil?
raise Scientist::BehaviorMissing.new(self, name)
end
unless should_experiment_run?
return block.call
end
if @_scientist_before_run
@_scientist_before_run.call
end
result = generate_result(name)
if @_scientist_after_run
@_scientist_after_run.call(result)
end
begin
publish(result)
rescue StandardError => ex
raised :publish, ex
end
if raise_on_mismatches? && result.mismatched?
if @_scientist_custom_mismatch_error
raise @_scientist_custom_mismatch_error.new(self.name, result)
else
raise MismatchError.new(self.name, result)
end
end
control = result.control
raise control.exception if control.raised?
control.value
end
# Define a block that determines whether or not the experiment should run.
def run_if(&block)
@_scientist_run_if_block = block
end
# Internal: does a run_if block allow the experiment to run?
#
# Rescues and reports exceptions in a run_if block if they occur.
def run_if_block_allows?
(@_scientist_run_if_block ? @_scientist_run_if_block.call : true)
rescue StandardError => ex
raised :run_if, ex
return false
end
# Internal: determine whether or not an experiment should run.
#
# Rescues and reports exceptions in the enabled method if they occur.
def should_experiment_run?
behaviors.size > 1 && enabled? && run_if_block_allows?
rescue StandardError => ex
raised :enabled, ex
return false
end
# Register a named behavior for this experiment, default "candidate".
def try(name = nil, &block)
name = (name || "candidate").to_s
if behaviors.include?(name)
raise Scientist::BehaviorNotUnique.new(self, name)
end
behaviors[name] = block
end
# Register the control behavior for this experiment.
def use(&block)
try "control", &block
end
# Whether or not to raise a mismatch error when a mismatch occurs.
def raise_on_mismatches?
if raise_on_mismatches.nil?
self.class.raise_on_mismatches?
else
!!raise_on_mismatches
end
end
# Provide predefined durations to use instead of actual timing data.
# This is here solely as a convenience for developers of libraries that extend Scientist.
def fabricate_durations_for_testing_purposes(fabricated_durations = {})
@_scientist_fabricated_durations = fabricated_durations
end
# Internal: Generate the observations and create the result from those and the control.
def generate_result(name)
observations = []
behaviors.keys.shuffle.each do |key|
block = behaviors[key]
fabricated_duration = @_scientist_fabricated_durations && @_scientist_fabricated_durations[key]
observations << Scientist::Observation.new(key, self, fabricated_duration: fabricated_duration, &block)
end
control = observations.detect { |o| o.name == name }
Scientist::Result.new(self, observations, control)
end
private
# In order to support marshaling, we have to make the procs marshalable. Some
# CI providers attempt to marshal Scientist mismatch errors so that they can
# be sent out to different places (logs, etc.) The mismatch errors contain
# code from the experiment. This code contains procs. These procs prevent the
# error from being marshaled. To fix this, we simple exclude the procs from
# the data that we marshal.
def marshal_dump
[@name, @result, @raise_on_mismatches]
end
def marshal_load(array)
@name, @result, @raise_on_mismatches = array
end
end
|