File: tasks.rb

package info (click to toggle)
ruby-derailed-benchmarks 1.7.0-2
  • links: PTS, VCS
  • area: main
  • in suites: bookworm, bullseye
  • size: 760 kB
  • sloc: ruby: 1,317; makefile: 4
file content (294 lines) | stat: -rw-r--r-- 8,838 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
require_relative 'load_tasks'

namespace :perf do
  desc "runs the performance test against two most recent commits of the current app"
  task :app do
    ENV["DERAILED_PATH_TO_LIBRARY"] = '.'
    Rake::Task["perf:library"].invoke
  end

  desc "runs the same test against two different branches for statistical comparison"
  task :library do
    begin
      DERAILED_SCRIPT_COUNT = (ENV["DERAILED_SCRIPT_COUNT"] ||= "200").to_i
      ENV["TEST_COUNT"] ||= "200"

      raise "test count must be at least 2, is set to #{DERAILED_SCRIPT_COUNT}" if DERAILED_SCRIPT_COUNT < 2
      script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"

      if ENV["DERAILED_PATH_TO_LIBRARY"]
        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
      else
        library_dir = DerailedBenchmarks.rails_path_on_disk
      end

      raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))

      # Use either the explicit SHAs when present or grab last two SHAs from commit history
      # if only one SHA is given, then use it and the last SHA from commit history
      branch_names = []
      branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
      if branch_names.length < 2
        Dir.chdir(library_dir) do
          run!("git checkout '#{branch_names.first}'") unless branch_names.empty?

          branches = run!('git log --format="%H" -n 2').chomp.split($/)
          if branch_names.empty?
            branch_names = branches
          else
            branches.shift
            branch_names << branches.shift
          end
        end
      end

      current_library_branch = ""
      Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }

      out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
      out_dir.mkpath

      branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
      branch_info = {}
      branch_to_sha = {}

      branches_to_test.each do |branch, file|
        Dir.chdir(library_dir) do
          run!("git checkout '#{branch}'")
          description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip
          time_stamp  = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390
          short_sha   = run!("git rev-parse --short HEAD").strip
          branch_to_sha[branch] = short_sha

          branch_info[short_sha] = { desc: description, time: DateTime.parse(time_stamp), file: file }
        end
        run!("#{script}")
      end

      puts
      puts
      branches_to_test.each.with_index do |(branch, _), i|
        short_sha = branch_to_sha[branch]
        desc      = branch_info[short_sha][:desc]
        puts "Testing #{i + 1}: #{short_sha}: #{desc}"
      end
      puts
      puts

      raise "SHAs to test must be different" if branch_info.length == 1
      stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
      puts "Env var no longer has any affect DERAILED_STOP_VALID_COUNT" if ENV["DERAILED_STOP_VALID_COUNT"]

      DERAILED_SCRIPT_COUNT.times do |i|
        puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
        branches_to_test.each do |branch, file|
          Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
          run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
        end

        if (i % 50).zero?
          puts "Intermediate result"
          stats.call
          stats.banner
          puts "Continuing execution"
        end
      end

    ensure
      if library_dir && current_library_branch
        puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}"
        Dir.chdir(library_dir) do
          run!("git checkout '#{current_library_branch}'")
        end
      end

      if stats
        stats.call
        stats.banner

        result_file = out_dir + "results.txt"
        File.open(result_file, "w") do |f|
          stats.banner(f)
        end

        puts "Output: #{result_file.to_s}"
      end
    end
  end

  desc "hits the url TEST_COUNT times"
  task :test => [:setup] do
    require 'benchmark'

    Benchmark.bm { |x|
      x.report("#{TEST_COUNT} derailed requests") {
        TEST_COUNT.times {
          call_app
        }
      }
    }
  end

  desc "stackprof"
  task :stackprof => [:setup] do
    # [:wall, :cpu, :object]
    begin
      require 'stackprof'
    rescue LoadError
      raise "Add stackprof to your gemfile to continue `gem 'stackprof', group: :development`"
    end
    TEST_COUNT = (ENV["TEST_COUNT"] ||= "100").to_i
    file = "tmp/#{Time.now.iso8601}-stackprof-cpu-myapp.dump"
    StackProf.run(mode: :cpu, out: file) do
      Rake::Task["perf:test"].invoke
    end
    cmd = "stackprof #{file}"
    puts "Running `#{cmd}`. Execute `stackprof --help` for more info"
    puts `#{cmd}`
  end

  task :kernel_require_patch do
    require 'derailed_benchmarks/core_ext/kernel_require.rb'
  end

  desc "show memory usage caused by invoking require per gem"
  task :mem => [:kernel_require_patch, :setup] do
    puts "## Impact of `require <file>` on RAM"
    puts
    puts "Showing all `require <file>` calls that consume #{ENV['CUT_OFF']} MiB or more of RSS"
    puts "Configure with `CUT_OFF=0` for all entries or `CUT_OFF=5` for few entries"

    puts "Note: Files only count against RAM on their first load."
    puts "      If multiple libraries require the same file, then"
    puts "       the 'cost' only shows up under the first library"
    puts

    call_app

    TOP_REQUIRE.print_sorted_children
  end

  desc "outputs memory usage over time"
  task :mem_over_time => [:setup] do
    require 'get_process_mem'
    puts "PID: #{Process.pid}"
    ram = GetProcessMem.new
    @keep_going = true
    begin
      unless ENV["SKIP_FILE_WRITE"]
        ruby = `ruby -v`.chomp
        FileUtils.mkdir_p("tmp")
        file = File.open("tmp/#{Time.now.iso8601}-#{ruby}-memory-#{TEST_COUNT}-times.txt", 'w')
        file.sync = true
      end

      ram_thread = Thread.new do
        while @keep_going
          mb = ram.mb
          STDOUT.puts mb
          file.puts mb unless ENV["SKIP_FILE_WRITE"]
          sleep 5
        end
      end

      TEST_COUNT.times {
        call_app
      }
    ensure
      @keep_going = false
      ram_thread.join
      file.close unless ENV["SKIP_FILE_WRITE"]
    end
  end

  task :ram_over_time do
    raise "Use mem_over_time"
  end

  desc "iterations per second"
  task :ips => [:setup] do
    require 'benchmark/ips'

    Benchmark.ips do |x|
      x.warmup = Float(ENV["IPS_WARMUP"] || 2)
      x.time = Float(ENV["IPS_TIME"] || 5)
      x.suite = ENV["IPS_SUITE"] if ENV["IPS_SUITE"]
      x.iterations = Integer(ENV["IPS_ITERATIONS"] || 1)

      x.report("ips") { call_app }
    end
  end

  desc "outputs GC::Profiler.report data while app is called TEST_COUNT times"
  task :gc => [:setup] do
    GC::Profiler.enable
    TEST_COUNT.times { call_app }
    GC::Profiler.report
    GC::Profiler.disable
  end

  desc "outputs allocated object diff after app is called TEST_COUNT times"
  task :allocated_objects => [:setup] do
    call_app
    GC.start
    GC.disable
    start = ObjectSpace.count_objects
    TEST_COUNT.times { call_app }
    finish = ObjectSpace.count_objects
    GC.enable
    finish.each do |k,v|
      puts k => (v - start[k]) / TEST_COUNT.to_f
    end
  end


  desc "profiles ruby allocation"
  task :objects => [:setup] do
    require 'memory_profiler'
    call_app
    GC.start

    num = Integer(ENV["TEST_COUNT"] || 1)
    opts = {}
    opts[:ignore_files] = /#{ENV['IGNORE_FILES_REGEXP']}/ if ENV['IGNORE_FILES_REGEXP']
    opts[:allow_files]  = "#{ENV['ALLOW_FILES']}"         if ENV['ALLOW_FILES']

    puts "Running #{num} times"
    report = MemoryProfiler.report(opts) do
      num.times { call_app }
    end
    report.pretty_print
  end

  desc "heap analyzer"
  task :heap => [:setup] do
    require 'objspace'

    file_name = "tmp/#{Time.now.iso8601}-heap.dump"
    FileUtils.mkdir_p("tmp")
    ObjectSpace.trace_object_allocations_start
    puts "Running #{ TEST_COUNT } times"
    TEST_COUNT.times {
      call_app
    }
    GC.start

    puts "Heap file generated: #{ file_name.inspect }"
    ObjectSpace.dump_all(output: File.open(file_name, 'w'))

    require 'heapy'

    Heapy::Analyzer.new(file_name).analyze

    puts ""
    puts "Run `$ heapy --help` for more options"
    puts ""
    puts "Also try uploading #{file_name.inspect} to http://tenderlove.github.io/heap-analyzer/"
  end

  def run!(cmd)
    out = `#{cmd}`
    raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
    out
  end
end