1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
|
#ifndef MAPNIK_BENCH_FRAMEWORK_HPP
#define MAPNIK_BENCH_FRAMEWORK_HPP
// mapnik
#include <mapnik/debug.hpp>
#include <mapnik/params.hpp>
#include <mapnik/value_types.hpp>
#include <mapnik/safe_cast.hpp>
#include "../test/cleanup.hpp"
// stl
#include <chrono>
#include <cmath> // log10, round
#include <cstdio> // snprintf
#include <iostream>
#include <set>
#include <sstream>
#include <thread>
#include <vector>
namespace benchmark {
template <typename T>
using milliseconds = std::chrono::duration<T, std::milli>;
template <typename T>
using seconds = std::chrono::duration<T>;
class test_case
{
protected:
mapnik::parameters params_;
std::size_t threads_;
std::size_t iterations_;
public:
test_case(mapnik::parameters const& params)
: params_(params),
threads_(mapnik::safe_cast<std::size_t>(*params.get<mapnik::value_integer>("threads",0))),
iterations_(mapnik::safe_cast<std::size_t>(*params.get<mapnik::value_integer>("iterations",0)))
{}
std::size_t threads() const
{
return threads_;
}
std::size_t iterations() const
{
return iterations_;
}
mapnik::parameters const& params() const
{
return params_;
}
virtual bool validate() const = 0;
virtual bool operator()() const = 0;
};
// gathers --long-option values in 'params';
// returns the index of the first non-option argument,
// or negated index of an ill-formed option argument
inline int parse_args(int argc, char** argv, mapnik::parameters & params)
{
for (int i = 1; i < argc; ++i) {
const char* opt = argv[i];
if (opt[0] != '-') {
// non-option argument, return its index
return i;
}
if (opt[1] != '-') {
// we only accept --long-options, but instead of throwing,
// just issue a warning and let the caller decide what to do
std::clog << argv[0] << ": invalid option '" << opt << "'\n";
return -i; // negative means ill-formed option #i
}
if (opt[2] == '\0') {
// option-list terminator '--'
return i + 1;
}
// take option name without the leading '--'
std::string key(opt + 2);
size_t eq = key.find('=');
if (eq != std::string::npos) {
// one-argument form '--foo=bar'
params[key.substr(0, eq)] = key.substr(eq + 1);
}
else if (i + 1 < argc) {
// two-argument form '--foo' 'bar'
params[key] = std::string(argv[++i]);
}
else {
// missing second argument
std::clog << argv[0] << ": missing option '" << opt << "' value\n";
return -i; // negative means ill-formed option #i
}
}
return argc; // there were no non-option arguments
}
inline void handle_common_args(mapnik::parameters const& params)
{
if (auto severity = params.get<std::string>("log")) {
if (*severity == "debug")
mapnik::logger::set_severity(mapnik::logger::debug);
else if (*severity == "warn")
mapnik::logger::set_severity(mapnik::logger::warn);
else if (*severity == "error")
mapnik::logger::set_severity(mapnik::logger::error);
else if (*severity == "none")
mapnik::logger::set_severity(mapnik::logger::none);
else
std::clog << "ignoring option --log='" << *severity
<< "' (allowed values are: debug, warn, error, none)\n";
}
}
inline int handle_args(int argc, char** argv, mapnik::parameters & params)
{
int res = parse_args(argc, argv, params);
handle_common_args(params);
return res;
}
#define BENCHMARK(test_class,name) \
int main(int argc, char** argv) \
{ \
try \
{ \
mapnik::parameters params; \
benchmark::handle_args(argc,argv,params); \
test_class test_runner(params); \
auto result = run(test_runner,name); \
testing::run_cleanup(); \
return result; \
} \
catch (std::exception const& ex) \
{ \
std::clog << ex.what() << "\n"; \
testing::run_cleanup(); \
return -1; \
} \
} \
struct big_number_fmt
{
int w;
double v;
const char* u;
big_number_fmt(int width, double value, int base = 1000)
: w(width), v(value), u("")
{
static const char* suffixes = "\0\0k\0M\0G\0T\0P\0E\0Z\0Y\0\0";
u = suffixes;
while (v > 1 && std::log10(std::round(v)) >= width && u[2])
{
v /= base;
u += 2;
}
// adjust width for proper alignment without suffix
w += (u == suffixes);
}
};
template <typename T>
int run(T const& test_runner, std::string const& name)
{
try
{
if (!test_runner.validate())
{
std::clog << "test did not validate: " << name << "\n";
return 1;
}
// run test once before timing
// if it returns false then we'll abort timing
if (!test_runner())
{
return 2;
}
std::chrono::high_resolution_clock::time_point start;
std::chrono::high_resolution_clock::duration elapsed;
auto opt_min_duration = test_runner.params().template get<double>("min-duration", 0.0);
std::chrono::duration<double> min_seconds(*opt_min_duration);
auto min_duration = std::chrono::duration_cast<decltype(elapsed)>(min_seconds);
auto num_iters = test_runner.iterations();
auto num_threads = test_runner.threads();
auto total_iters = 0;
if (num_threads > 0)
{
std::mutex mtx_ready;
std::unique_lock<std::mutex> lock_ready(mtx_ready);
auto stub = [&](T const& test_copy)
{
// workers will wait on this mutex until the main thread
// constructs all of them and starts measuring time
std::unique_lock<std::mutex> my_lock(mtx_ready);
my_lock.unlock();
test_copy();
};
std::vector<std::thread> tg;
tg.reserve(num_threads);
for (auto i = num_threads; i-- > 0; )
{
tg.emplace_back(stub, test_runner);
}
start = std::chrono::high_resolution_clock::now();
lock_ready.unlock();
// wait for all workers to finish
for (auto & t : tg)
{
if (t.joinable())
t.join();
}
elapsed = std::chrono::high_resolution_clock::now() - start;
// this is actually per-thread count, not total, but I think
// reporting average 'iters/thread/second' is more useful
// than 'iters/second' multiplied by the number of threads
total_iters += num_iters;
}
else
{
start = std::chrono::high_resolution_clock::now();
do {
test_runner();
elapsed = std::chrono::high_resolution_clock::now() - start;
total_iters += num_iters;
} while (elapsed < min_duration);
}
char msg[200];
double dur_total = milliseconds<double>(elapsed).count();
auto elapsed_nonzero = std::max(elapsed, decltype(elapsed){1});
big_number_fmt itersf(4, total_iters);
big_number_fmt ips(5, total_iters / seconds<double>(elapsed_nonzero).count());
std::snprintf(msg, sizeof(msg),
"%-43s %3zu threads %*.0f%s iters %6.0f milliseconds %*.0f%s i/s\n",
name.c_str(),
num_threads,
itersf.w, itersf.v, itersf.u,
dur_total,
ips.w, ips.v, ips.u
);
std::clog << msg;
return 0;
}
catch (std::exception const& ex)
{
std::clog << "test runner did not complete: " << ex.what() << "\n";
return 4;
}
}
struct sequencer
{
sequencer(int argc, char** argv)
: exit_code_(0)
{
benchmark::handle_args(argc, argv, params_);
}
int done() const
{
return exit_code_;
}
template <typename Test, typename... Args>
sequencer & run(std::string const& name, Args && ...args)
{
// Test instance lifetime is confined to this function
Test test_runner(params_, std::forward<Args>(args)...);
// any failing test run will make exit code non-zero
exit_code_ |= benchmark::run(test_runner, name);
return *this; // allow chaining calls
}
protected:
mapnik::parameters params_;
int exit_code_;
};
}
#endif // MAPNIK_BENCH_FRAMEWORK_HPP
|