1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
|
require_relative 'load_tasks'
namespace :perf do
desc "runs the performance test against two most recent commits of the current app"
task :app do
ENV["DERAILED_PATH_TO_LIBRARY"] = '.'
Rake::Task["perf:library"].invoke
end
desc "runs the same test against two different branches for statistical comparison"
task :library do
begin
DERAILED_SCRIPT_COUNT = (ENV["DERAILED_SCRIPT_COUNT"] ||= "200").to_i
ENV["TEST_COUNT"] ||= "200"
raise "test count must be at least 2, is set to #{DERAILED_SCRIPT_COUNT}" if DERAILED_SCRIPT_COUNT < 2
script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"
if ENV["DERAILED_PATH_TO_LIBRARY"]
library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
else
library_dir = DerailedBenchmarks.rails_path_on_disk
end
raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))
# Use either the explicit SHAs when present or grab last two SHAs from commit history
# if only one SHA is given, then use it and the last SHA from commit history
branch_names = []
branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
if branch_names.length < 2
Dir.chdir(library_dir) do
run!("git checkout '#{branch_names.first}'") unless branch_names.empty?
branches = run!('git log --format="%H" -n 2').chomp.split($/)
if branch_names.empty?
branch_names = branches
else
branches.shift
branch_names << branches.shift
end
end
end
current_library_branch = ""
Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
out_dir.mkpath
branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
branch_info = {}
branch_to_sha = {}
branches_to_test.each do |branch, file|
Dir.chdir(library_dir) do
run!("git checkout '#{branch}'")
description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip
time_stamp = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390
short_sha = run!("git rev-parse --short HEAD").strip
branch_to_sha[branch] = short_sha
branch_info[short_sha] = { desc: description, time: DateTime.parse(time_stamp), file: file }
end
run!("#{script}")
end
puts
puts
branches_to_test.each.with_index do |(branch, _), i|
short_sha = branch_to_sha[branch]
desc = branch_info[short_sha][:desc]
puts "Testing #{i + 1}: #{short_sha}: #{desc}"
end
puts
puts
raise "SHAs to test must be different" if branch_info.length == 1
stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
puts "Env var no longer has any affect DERAILED_STOP_VALID_COUNT" if ENV["DERAILED_STOP_VALID_COUNT"]
DERAILED_SCRIPT_COUNT.times do |i|
puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
branches_to_test.each do |branch, file|
Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
end
if (i % 50).zero?
puts "Intermediate result"
stats.call
stats.banner
puts "Continuing execution"
end
end
ensure
if library_dir && current_library_branch
puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}"
Dir.chdir(library_dir) do
run!("git checkout '#{current_library_branch}'")
end
end
if stats
stats.call
stats.banner
result_file = out_dir + "results.txt"
File.open(result_file, "w") do |f|
stats.banner(f)
end
puts "Output: #{result_file.to_s}"
end
end
end
desc "hits the url TEST_COUNT times"
task :test => [:setup] do
require 'benchmark'
Benchmark.bm { |x|
x.report("#{TEST_COUNT} derailed requests") {
TEST_COUNT.times {
call_app
}
}
}
end
desc "stackprof"
task :stackprof => [:setup] do
# [:wall, :cpu, :object]
begin
require 'stackprof'
rescue LoadError
raise "Add stackprof to your gemfile to continue `gem 'stackprof', group: :development`"
end
TEST_COUNT = (ENV["TEST_COUNT"] ||= "100").to_i
file = "tmp/#{Time.now.iso8601}-stackprof-cpu-myapp.dump"
StackProf.run(mode: :cpu, out: file) do
Rake::Task["perf:test"].invoke
end
cmd = "stackprof #{file}"
puts "Running `#{cmd}`. Execute `stackprof --help` for more info"
puts `#{cmd}`
end
task :kernel_require_patch do
require 'derailed_benchmarks/core_ext/kernel_require.rb'
end
desc "show memory usage caused by invoking require per gem"
task :mem => [:kernel_require_patch, :setup] do
puts "## Impact of `require <file>` on RAM"
puts
puts "Showing all `require <file>` calls that consume #{ENV['CUT_OFF']} MiB or more of RSS"
puts "Configure with `CUT_OFF=0` for all entries or `CUT_OFF=5` for few entries"
puts "Note: Files only count against RAM on their first load."
puts " If multiple libraries require the same file, then"
puts " the 'cost' only shows up under the first library"
puts
call_app
TOP_REQUIRE.print_sorted_children
end
desc "outputs memory usage over time"
task :mem_over_time => [:setup] do
require 'get_process_mem'
puts "PID: #{Process.pid}"
ram = GetProcessMem.new
@keep_going = true
begin
unless ENV["SKIP_FILE_WRITE"]
ruby = `ruby -v`.chomp
FileUtils.mkdir_p("tmp")
file = File.open("tmp/#{Time.now.iso8601}-#{ruby}-memory-#{TEST_COUNT}-times.txt", 'w')
file.sync = true
end
ram_thread = Thread.new do
while @keep_going
mb = ram.mb
STDOUT.puts mb
file.puts mb unless ENV["SKIP_FILE_WRITE"]
sleep 5
end
end
TEST_COUNT.times {
call_app
}
ensure
@keep_going = false
ram_thread.join
file.close unless ENV["SKIP_FILE_WRITE"]
end
end
task :ram_over_time do
raise "Use mem_over_time"
end
desc "iterations per second"
task :ips => [:setup] do
require 'benchmark/ips'
Benchmark.ips do |x|
x.warmup = Float(ENV["IPS_WARMUP"] || 2)
x.time = Float(ENV["IPS_TIME"] || 5)
x.suite = ENV["IPS_SUITE"] if ENV["IPS_SUITE"]
x.iterations = Integer(ENV["IPS_ITERATIONS"] || 1)
x.report("ips") { call_app }
end
end
desc "outputs GC::Profiler.report data while app is called TEST_COUNT times"
task :gc => [:setup] do
GC::Profiler.enable
TEST_COUNT.times { call_app }
GC::Profiler.report
GC::Profiler.disable
end
desc "outputs allocated object diff after app is called TEST_COUNT times"
task :allocated_objects => [:setup] do
call_app
GC.start
GC.disable
start = ObjectSpace.count_objects
TEST_COUNT.times { call_app }
finish = ObjectSpace.count_objects
GC.enable
finish.each do |k,v|
puts k => (v - start[k]) / TEST_COUNT.to_f
end
end
desc "profiles ruby allocation"
task :objects => [:setup] do
require 'memory_profiler'
call_app
GC.start
num = Integer(ENV["TEST_COUNT"] || 1)
opts = {}
opts[:ignore_files] = /#{ENV['IGNORE_FILES_REGEXP']}/ if ENV['IGNORE_FILES_REGEXP']
opts[:allow_files] = "#{ENV['ALLOW_FILES']}" if ENV['ALLOW_FILES']
puts "Running #{num} times"
report = MemoryProfiler.report(opts) do
num.times { call_app }
end
report.pretty_print
end
desc "heap analyzer"
task :heap => [:setup] do
require 'objspace'
file_name = "tmp/#{Time.now.iso8601}-heap.dump"
FileUtils.mkdir_p("tmp")
ObjectSpace.trace_object_allocations_start
puts "Running #{ TEST_COUNT } times"
TEST_COUNT.times {
call_app
}
GC.start
puts "Heap file generated: #{ file_name.inspect }"
ObjectSpace.dump_all(output: File.open(file_name, 'w'))
require 'heapy'
Heapy::Analyzer.new(file_name).analyze
puts ""
puts "Run `$ heapy --help` for more options"
puts ""
puts "Also try uploading #{file_name.inspect} to http://tenderlove.github.io/heap-analyzer/"
end
def run!(cmd)
out = `#{cmd}`
raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
out
end
end
|