File: maxitest_spec.rb

package info (click to toggle)
ruby-maxitest 7.1.1-2
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 444 kB
  • sloc: ruby: 1,339; makefile: 7
file content (345 lines) | stat: -rw-r--r-- 11,858 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
# frozen_string_literal: true
require "spec_helper"
require 'open3'

describe Maxitest do
  it "has a VERSION" do
    Maxitest::VERSION.should =~ /^[.\da-z]+$/
  end

  xit "does not add extra output" do
    result = run_cmd("ruby spec/cases/plain.rb")
    result.sub!(/seed \d+/, 'seed X')
    result.gsub!(/\d+\.\d+/, 'X')
    result.should == "Run options: --seed X\n\n# Running:\n\n..\n\nFinished in Xs, X runs/s, X assertions/s.\n\n2 runs, 2 assertions, 0 failures, 0 errors, 0 skips"
  end

  it "runs via ruby" do
    run_cmd("ruby spec/cases/plain.rb").should include "\n2 runs, 2 assertions"
  end

  it "supports context" do
    run_cmd("ruby spec/cases/context.rb").should include "\n2 runs, 2 assertions"
  end

  it "supports let!" do
    run_cmd("ruby spec/cases/let_bang.rb").should include "\n1 runs, 1 assertions"
  end

  it "supports let_all" do
    run_cmd("ruby spec/cases/let_all.rb")
  end

  it "can use static_class_order" do
    run_cmd("ruby spec/cases/static_class_order.rb").should include("\n0\n.0n\n.1\n.1n\n.2\n.2n\n.3\n.3n\n.4\n.4n\n.\n")
  end

  it "supports order_dependent" do
    run_cmd("ruby spec/cases/order_dependent.rb").should include "5 runs, 1 assertions, 0 failures, 0 errors, 0 skips"
  end

  it "has pending" do
    run_cmd("ruby spec/cases/pending.rb")
  end

  it "does not call xit specs" do
    result = run_cmd("ruby spec/cases/xit.rb -v")
    result.should include "(no tests defined)"
    result.should include "4 runs, 3 assertions, 0 failures, 0 errors, 2 skips"
  end

  it "shows short backtraces" do
    out = run_cmd("ruby spec/cases/raise.rb", fail: true)

    # Ruby 3.2 has a different backtrace it add 2 lines between the lib/maxitest/timeout.rb
    # and the spec/cases/raise.rb
    out.gsub!(/\n.*previous definition of Timeout.*/, "")

    # unify backtraces between ruby versions
    output_in = out.gsub!(/:in .*/, "")

    output_in.should include "TypeError: Timeout is not a module"
    output_in.should include 'spec/cases/raise.rb:12'

    # Minitest 5.21.0+ backtrace is more verbose and the short backtrace feature seems to be gone
    # re-test by running spec/cases/raise.rb and only loading minitest/autorun and not maxitest
    if Minitest::VERSION <= "5.21.0"
      output_in.should_not include 'lib/maxitest'
    end
  end

  it "has helpers" do
    run_cmd("ruby spec/cases/helpers.rb")
  end

  it "does not use old MiniTest constant" do
    Dir["lib/**/*.rb"].each do |p|
      File.read(p).should_not include "MiniTest"
    end
  end

  describe "before/after/around" do
    it "works" do
      out = run_cmd("ruby spec/cases/hook_all.rb")
      out.should include "Running:\n\nALL\nT1\n.T2\n.ALL-SUB\nTS1\n.TS2\n.\n\nFinished"
    end

    it "fails when using unsupported type" do
      with_env HOOK_TYPE: "foo" do
        out = run_cmd("ruby spec/cases/hook_all.rb", fail: true)
        out.should include "only :each and :all are supported (ArgumentError)"
      end
    end

    it "informs user about missing after :all" do
      with_env HOOK_METHOD: "after" do
        out = run_cmd("ruby spec/cases/hook_all.rb --seed 123", fail: true)
        out.should include ":all is not supported in after (ArgumentError)"
      end
    end

    it "supports around" do
      run_cmd("ruby spec/cases/around.rb").should include "\n2 runs, 3 assertions"
    end

    it "informs user about missing around :all" do
      with_env HOOK_TYPE: "all" do
        out = run_cmd("ruby spec/cases/around.rb", fail: true)
        out.should include "only :each"
      end
    end
  end

  describe "color" do
    it "is color-less without tty" do
      run_cmd("ruby spec/cases/plain.rb").should include "\n2 runs, 2 assertions"
    end

    it "is colorful on tty" do
      simulate_tty do
        run_cmd("ruby spec/cases/plain.rb").should include "\n\e[32m2 runs, 2 assertions"
      end
    end

    it "is colorful without tty but --rg" do
      run_cmd("ruby spec/cases/plain.rb --rg").should include "\n\e[32m2 runs, 2 assertions"
    end

    it "is color-less with --no-rg and tty" do
      simulate_tty do
        run_cmd("ruby spec/cases/plain.rb --no-rg").should include "\n2 runs, 2 assertions"
      end
    end
  end

  describe "timeout" do
    it "times out long running tests" do
      result = run_cmd("ruby spec/cases/timeout.rb -v", fail: true)

      # 1 test takes too long and fails with a nice error message
      result.should include "Maxitest::Timeout::TestCaseTimeout: Test took too long to finish, aborting"

      # results look normal
      result.should include ", 2 errors,"
    end

    it "times out after custom interval" do
      result = run_cmd("CUSTOM=1 ruby spec/cases/timeout.rb -v", fail: true)

      # 1 test takes too long and fails with a nice error message
      result.should include "Maxitest::Timeout::TestCaseTimeout: Test took too long to finish, aborting"

      # results look normal
      result.should include ", 2 errors,"
    end

    it "does not time out when disabled" do
      result = run_cmd("DISABLE=1 ruby spec/cases/timeout.rb -v")

      # 1 test takes too long and fails with a nice error message
      result.should include "DID NOT TIME OUT"

      # results look normal
      result.should include "4 runs"
    end
  end

  describe "extra threads" do
    it "fails on extra and passes on regular" do
      result = run_cmd("ruby spec/cases/threads.rb -v", fail: true)
      result.gsub(/\d\.\d+/, "0.0").should include <<-OUT.gsub(/^\s+/, "")
        threads#test_0001_is fine without extra threads = 0.0 s = .
        threads#test_0002_fails on extra threads = 0.0 s = F
        threads#test_0003_can kill extra threads = 0.0 s = .
        threads#test_0004_can wait for extra threads = 0.0 s = .
      OUT
    end
  end

  describe "global_must" do
    xit "does not complain when used and loaded" do
      with_env GLOBAL_MUST: 'true' do
        run_cmd("ruby spec/cases/global_must.rb")
      end
    end

    xit "complains when used and not loaded" do
      out = run_cmd("ruby spec/cases/global_must.rb", fail: true)
      out.gsub!('`', "'") # for ruby <3.4 had
      out.should include "undefined method 'must_equal'"
    end

    xit "does not obfuscate test origin" do
      with_env GLOBAL_MUST: 'true', FAIL: 'true' do
        out = run_cmd("ruby spec/cases/global_must.rb", fail: true)
        out.should include "_fails [spec/cases/global_must.rb:18]:"
      end
    end
  end

  describe "Interrupts" do
    it "stops on ctrl+c and prints errors" do
      t = Thread.new { run_cmd("ruby spec/cases/cltr_c.rb", fail: true) }
      sleep 1 # let thread start
      kill_process_with_name("spec/cases/cltr_c.rb")
      output = t.value
      output.should include "4 runs, 1 assertions, 1 failures, 1 errors, 2 skips" # failed, error from interrupt (so you see a backtrace), rest skipped
      output.gsub(/:\d+:in/, ":D:in").should match(/cltr_c\.rb:D:in (`|'Kernel#)sleep'/) # let you know where it happened
      output.should include "Interrupt:" # let you know what happened
      output.should include "Expected: true\n  Actual: false" # not hide other errors
      output.scan('BEFORE').size.should == 2 # before calls avoided when skipping
      output.scan('AFTER').size.should == 2 # after calls avoided when skipping
    end

    it "allows Interrupts to be caught normally" do
      output = run_cmd("ruby spec/cases/catch_interrupt.rb")
      output.should include "1 runs, 1 assertions, 0 failures, 0 errors, 0 skips"
    end

    it "catches directly raised Interrupt" do
      output = run_cmd("ruby spec/cases/raise_interrupt.rb", fail: true)
      output.should include "runs, "
      output.should include "Interrupt: Interrupt"
    end

    it "shows backtraces when in verbose mode" do
      t = Thread.new { run_cmd("ruby spec/cases/cltr_c.rb -v", fail: true) }
      sleep 1 # let thread start
      kill_process_with_name("spec/cases/cltr_c.rb")
      output = t.value
      output.gsub(/:\d+:in/, ":D:in").should match(/cltr_c.rb:D:in (`|'Kernel#)sleep'/) # let you know where it happened
    end
  end

  describe "minitest executable" do
    xit "shows version" do
      run_cmd("minitest --version", fail: true).should include "minitest"
    end

    xit "shows help" do
      run_cmd("minitest -h", fail: true).should include "Usage:"
      run_cmd("minitest --help", fail: true).should include "Usage:"
    end

    xit "runs a single file" do
      run_cmd("minitest spec/cases/plain.rb").should include "2 runs, 2 assertions, 0 failures, 0 errors, 0 skips"
    end

    xit "runs a single line" do
      run_cmd("minitest spec/cases/plain.rb:6").should include "1 runs, 1 assertions, 0 failures, 0 errors, 0 skips"
    end

    xit "runs a folder" do
      run_cmd("minitest spec/cases/mtest").should include "2 runs, 2 assertions, 0 failures, 0 errors, 0 skips"
    end

    xit "runs multiple files" do
      run_cmd("minitest spec/cases/mtest/a_test.rb spec/cases/mtest/c.rb").should include "2 runs, 2 assertions, 0 failures, 0 errors, 0 skips"
    end
  end

  describe "line" do
    let(:separator) { "Focus on failing tests:\n" }

    xit "prints rerun commands" do
      result = run_cmd("ruby spec/cases/line.rb", fail: true)
      result.should include "4 runs, 4 assertions"
      foucs = result.split(separator, 2)[1]
      foucs.should == "minitest spec/cases/line.rb:9"
    end

    xit "does not print rerun commands when already filtered" do
      result = run_cmd("minitest spec/cases/line.rb:9", fail: true)
      result.should include "1 runs, 1 assertions"
      result.should_not include separator
    end
  end

  describe "backtraces" do
    xit "shows no backtrace without verbose" do
      result = run_cmd("ruby spec/cases/error_and_failure.rb", fail: true)
      result.should include "error_and_failure.rb:6"
      result.should include "error_and_failure.rb:10"
      result.should_not include "minitest.rb"
    end

    it "shows backtrace for errors with verbose" do
      result = run_cmd("ruby spec/cases/error_and_failure.rb -n '/errors/' -v", fail: true)
      result.should include "1 run"
      result.should include "error_and_failure.rb:6"
      result.should include "minitest.rb"
    end

    it "shows backtrace for failures with verbose" do
      result = run_cmd("ruby spec/cases/error_and_failure.rb -n '/fails/' -v", fail: true)
      result.should include "1 run"
      result.should include "error_and_failure.rb:10"
      result.should include "minitest.rb"
    end
  end

  describe "parallel" do
    it "can run in parallel" do
      result = run_cmd("ruby spec/cases/parallel.rb -v")
      result.should include "\n3 runs"
      result.should include "Finished in 0.1"
    end
  end

  private

  def simulate_tty(&block)
    with_env SIMULATE_TTY: 'true', &block
  end

  def with_env(h)
    old = ENV.to_h
    h.each { |k, v| ENV[k.to_s] = v }
    yield
  ensure
    ENV.replace old
  end

  def run_cmd(command, deprecated: :fail, keep_output: false, fail: false)
    stdout, stderr, status = Open3.capture3(command)

    stderr.should_not include("DEPRECATED") unless deprecated == :ignore

    stdout += "\n#{stderr}" unless keep_output

    if status.success? == fail
      raise "#{fail ? "SUCCESS" : "FAIL"} #{command}\n#{stdout}"
    end

    stdout.strip
  end

  # copied from https://github.com/grosser/parallel/blob/master/spec/parallel_spec.rb#L10-L15
  def kill_process_with_name(file, signal = 'INT')
    running_processes = `ps -f`.split("\n").map { |line| line.split(/\s+/) }
    pid_index = running_processes.detect { |p| p.include?("UID") }.index("UID") + 1
    parent = running_processes.detect { |p| p.include?(file) and !p.include?("sh") }
    raise "Unable to find parent in #{running_processes} with #{file}" unless parent
    `kill -s #{signal} #{parent.fetch(pid_index)}`
  end
end