File: maxitest_spec.rb

package info (click to toggle)
ruby-maxitest 6.0.1-1
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 504 kB
  • sloc: ruby: 1,583; makefile: 7
file content (367 lines) | stat: -rw-r--r-- 12,434 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
require "spec_helper"
require 'open3'

describe Maxitest do
  it "has a VERSION" do
    Maxitest::VERSION.should =~ /^[\.\da-z]+$/
  end

  xit "does not add extra output" do
    result = with_global_must do
      run_cmd("ruby spec/cases/plain.rb")
    end
    result.sub!(/seed \d+/, 'seed X')
    result.gsub!(/\d+\.\d+/, 'X')
    result.should == "Run options: --seed X\n\n# Running:\n\n..\n\nFinished in Xs, X runs/s, X assertions/s.\n\n2 runs, 2 assertions, 0 failures, 0 errors, 0 skips"
  end

  it "runs via ruby" do
    run_cmd("ruby spec/cases/plain.rb").should include "\n2 runs, 2 assertions"
  end

  it "supports context" do
    run_cmd("ruby spec/cases/context.rb").should include "\n2 runs, 2 assertions"
  end

  it "supports let!" do
    run_cmd("ruby spec/cases/let_bang.rb").should include "\n1 runs, 1 assertions"
  end

  it "supports let_all" do
    run_cmd("ruby spec/cases/let_all.rb")
  end

  it "can use static_class_order" do
    run_cmd("ruby spec/cases/static_class_order.rb").should include("\n0\n.0n\n.1\n.1n\n.2\n.2n\n.3\n.3n\n.4\n.4n\n.\n")
  end

  it "supports order_dependent" do
    run_cmd("ruby spec/cases/order_dependent.rb").should include "5 runs, 1 assertions, 0 failures, 0 errors, 0 skips"
  end

  it "has pending" do
    run_cmd("ruby spec/cases/pending.rb")
  end

  it "does not call xit specs" do
    result = run_cmd("ruby spec/cases/xit.rb -v")
    result.should include "(no tests defined)"
    result.should include "4 runs, 3 assertions, 0 failures, 0 errors, 2 skips"
  end

  it "shows short backtraces" do
    out = run_cmd("ruby spec/cases/raise.rb", fail: true)

    # Ruby 3.2 has a different backtrace it add 2 lines between the lib/maxitest/timeout.rb
    # and the spec/cases/raise.rb
    out.gsub!(/\n.*previous definition of Timeout.*/, "")

    # unify backtraces between ruby versions
    output_in = out.gsub!(/:in .*/, "")

    output_in.should include "TypeError: Timeout is not a module"
    output_in.should include 'spec/cases/raise.rb:11'

    # Minitest 5.21.0+ backtrace is more verbose and the short backtrace feature seems to be gone
    # re-test by running spec/cases/raise.rb and only loading minitest/autorun and not maxitest
    if Minitest::VERSION <= "5.21.0"
      output_in.should_not include 'lib/maxitest'
    end
  end

  it "has helpers" do
    run_cmd("ruby spec/cases/helpers.rb")
  end

  it "does not use old MiniTest constant" do
    Dir["lib/**/*.rb"].each do |p|
      File.read(p).should_not include "MiniTest"
    end
  end

  describe "before/after/around" do
    it "works" do
      out = run_cmd("ruby spec/cases/hook_all.rb")
      out.should include "Running:\n\nALL\nT1\n.T2\n.ALL-SUB\nTS1\n.TS2\n.\n\nFinished"
    end

    it "fails when using unsupported type" do
      with_env HOOK_TYPE: "foo" do
        out = run_cmd("ruby spec/cases/hook_all.rb", fail: true)
        out.should include "only :each and :all are supported (ArgumentError)"
      end
    end

    it "informs user about missing after :all" do
      with_env HOOK_METHOD: "after" do
        out = run_cmd("ruby spec/cases/hook_all.rb --seed 123", fail: true)
        out.should include ":all is not supported in after (ArgumentError)"
      end
    end

    it "supports around" do
      run_cmd("ruby spec/cases/around.rb").should include "\n2 runs, 3 assertions"
    end

    it "informs user about missing around :all" do
      with_env HOOK_TYPE: "all" do
        out = run_cmd("ruby spec/cases/around.rb", fail: true)
        out.should include "only :each or no argument is supported (ArgumentError)"
      end
    end
  end

  describe "color" do
    it "is color-less without tty" do
      run_cmd("ruby spec/cases/plain.rb").should include "\n2 runs, 2 assertions"
    end

    it "is colorful on tty" do
      simulate_tty do
        run_cmd("ruby spec/cases/plain.rb").should include "\n\e[32m2 runs, 2 assertions"
      end
    end

    it "is colorful without tty but --rg" do
      run_cmd("ruby spec/cases/plain.rb --rg").should include "\n\e[32m2 runs, 2 assertions"
    end

    it "is color-less with --no-rg and tty" do
      simulate_tty do
        run_cmd("ruby spec/cases/plain.rb --no-rg").should include "\n2 runs, 2 assertions"
      end
    end
  end

  describe "timeout" do
    it "times out long running tests" do
      result = run_cmd("ruby spec/cases/timeout.rb -v", fail: true)

      # 1 test takes too long and fails with a nice error message
      result.should include "Maxitest::Timeout::TestCaseTimeout: Test took too long to finish, aborting"

      # results look normal
      result.should include ", 2 errors,"
    end

    it "times out after custom interval" do
      result = run_cmd("CUSTOM=1 ruby spec/cases/timeout.rb -v", fail: true)

      # 1 test takes too long and fails with a nice error message
      result.should include "Maxitest::Timeout::TestCaseTimeout: Test took too long to finish, aborting"

      # results look normal
      result.should include ", 2 errors,"
    end

    it "does not time out when disabled" do
      result = run_cmd("DISABLE=1 ruby spec/cases/timeout.rb -v")

      # 1 test takes too long and fails with a nice error message
      result.should include "DID NOT TIME OUT"

      # results look normal
      result.should include "4 runs"
    end
  end

  describe "global_must" do
    let(:deprecated) { "DEPRECATED" }

    it "complain when used and not loaded" do
      with_env USE_GLOBAL_MUST: 'true' do
        run_cmd("ruby spec/cases/plain.rb", deprecated: :ignore).should include deprecated
      end
    end

    it "does not complain when used and loaded" do
      with_global_must do
        with_env USE_GLOBAL_MUST: 'true' do
          run_cmd("ruby spec/cases/plain.rb").should_not include deprecated
        end
      end
    end

    it "fails when used in threads" do
        with_global_must do
          run_cmd("ruby spec/cases/global_must.rb")
        end
      end
  end

  describe "extra threads" do
    it "fails on extra and passes on regular" do
      result = with_global_must do
        run_cmd("ruby spec/cases/threads.rb -v", fail: true)
      end
      result.gsub(/\d\.\d+/, "0.0").should include <<-OUT.gsub(/^\s+/, "")
        threads#test_0001_is fine without extra threads = 0.0 s = .
        threads#test_0002_fails on extra threads = 0.0 s = F
        threads#test_0003_can kill extra threads = 0.0 s = .
        threads#test_0004_can wait for extra threads = 0.0 s = .
      OUT
    end
  end

  describe "line" do
    let(:focus) { "Focus on failing tests:" }
    let(:expected_command) { "mtest spec/cases/line.rb:8" }

    it "prints line numbers on failed" do
      run_cmd("ruby spec/cases/line.rb", fail: true).should include "#{focus}\n#{expected_command}"
    end

    xit "can run with line numbers" do
      result = run_cmd(expected_command, fail: true)
      result.should include("1 runs, 1 assertions, 1 failures, 0 errors, 0 skips")
      result.should_not include(focus) # ran 1 line, no need to reprint
    end

    xit "can describe with line numbers" do
      result = run_cmd("mtest spec/cases/line.rb:12")
      result.should include("2 runs, 2 assertions, 0 failures, 0 errors, 0 skips")
    end

    it "can run with -l line numbers" do
      result = run_cmd("ruby spec/cases/line.rb -l 8", fail: true)
      result.should include("1 runs, 1 assertions, 1 failures, 0 errors, 0 skips")
      result.should_not include(focus) # ran 1 line, no need to reprint
    end

    it "uses colors on tty" do
      simulate_tty do
        run_cmd("ruby spec/cases/line.rb", fail: true).should include "\e[31m#{expected_command}\e[0m"
      end
    end
  end

  describe "Interrupts" do
    it "stops on ctrl+c and prints errors" do
      t = Thread.new { run_cmd("ruby spec/cases/cltr_c.rb", fail: true) }
      sleep 1 # let thread start
      kill_process_with_name("spec/cases/cltr_c.rb")
      output = t.value
      output.should include "4 runs, 1 assertions, 1 failures, 1 errors, 2 skips" # failed, error from interrupt (so you see a backtrace), rest skipped
      output.gsub(/:\d+:in/, ":D:in").should match /cltr_c\.rb:D:in (`|'Kernel#)sleep'/ # let you know where it happened
      output.should include "Interrupt:" # let you know what happened
      output.should include "Expected: true\n  Actual: false" # not hide other errors
      output.scan(/BEFORE/).size.should == 2 # before calls avoided when skipping
      output.scan(/AFTER/).size.should == 2 # after calls avoided when skipping
    end

    it "allows Interrupts to be caught normally" do
      output = run_cmd("ruby spec/cases/catch_interrupt.rb")
      output.should include "1 runs, 1 assertions, 0 failures, 0 errors, 0 skips"
    end

    it "catches directly raised Interrupt" do
      output = run_cmd("ruby spec/cases/raise_interrupt.rb", fail: true)
      output.should include "runs, "
      output.should include "Interrupt: Interrupt"
    end

    it "shows backtraces when in verbose mode" do
      t = Thread.new { run_cmd("ruby spec/cases/cltr_c.rb -v", fail: true) }
      sleep 1 # let thread start
      kill_process_with_name("spec/cases/cltr_c.rb")
      output = t.value
      output.gsub(/:\d+:in/, ":D:in").should match /cltr_c.rb:D:in (`|'Kernel#)sleep'/ # let you know where it happened
    end
  end

  describe "mtest" do
    xit "shows version" do
      run_cmd("mtest -v").should == Maxitest::VERSION
      run_cmd("mtest --version").should == Maxitest::VERSION
    end

    xit "shows help" do
      run_cmd("mtest -h").should include "Usage:"
      run_cmd("mtest --help").should include "Usage:"
    end

    xit "runs a single file" do
      run_cmd("mtest spec/cases/mtest/a_test.rb").should include "1 runs, 1 assertions, 0 failures, 0 errors, 0 skips"
    end

    xit "runs a folder" do
      run_cmd("mtest spec/cases/mtest").should include "2 runs, 2 assertions, 0 failures, 0 errors, 0 skips"
    end

    xit "runs multiple files" do
      run_cmd("mtest spec/cases/mtest/a_test.rb spec/cases/mtest/c.rb").should include "2 runs, 2 assertions, 0 failures, 0 errors, 0 skips"
    end
  end

  describe "backtraces" do
    it "shows no backtrace without verbose" do
      result = run_cmd("ruby spec/cases/error_and_failure.rb", fail: true)
      result.should include "error_and_failure.rb:5"
      result.should include "error_and_failure.rb:9"
      result.should_not include "minitest.rb"
    end

    it "shows backtrace for errors with verbose" do
      result = run_cmd("ruby spec/cases/error_and_failure.rb -n '/errors/' -v", fail: true)
      result.should include "1 run"
      result.should include "error_and_failure.rb:5"
      result.should include "minitest.rb"
    end

    it "shows backtrace for failures with verbose" do
      result = run_cmd("ruby spec/cases/error_and_failure.rb -n '/fails/' -v", fail: true)
      result.should include "1 run"
      result.should include "error_and_failure.rb:9"
      result.should include "minitest.rb"
    end
  end

  describe "parallel" do
    it "can run in parallel" do
      result = run_cmd("ruby spec/cases/parallel.rb -v")
      result.should include "\n3 runs"
      result.should include "Finished in 0.1"
    end
  end

  private

  def simulate_tty(&block)
    with_env SIMULATE_TTY: 'true', &block
  end

  def with_global_must(&block)
    with_env GLOBAL_MUST: 'true', &block
  end

  def with_env(h)
    old = ENV.to_h
    h.each { |k, v| ENV[k.to_s] = v}
    yield
  ensure
    ENV.replace old
  end

  def run_cmd(command, deprecated: :fail, keep_output: false, fail: false)
    stdout, stderr, status = Open3.capture3(command)

    stderr.should_not include("DEPRECATED") unless deprecated == :ignore

    stdout += "\n" + stderr unless keep_output

    if status.success? == fail
      raise "#{fail ? "SUCCESS" : "FAIL"} #{command}\n#{stdout}"
    end

    stdout.strip
  end

  # copied from https://github.com/grosser/parallel/blob/master/spec/parallel_spec.rb#L10-L15
  def kill_process_with_name(file, signal='INT')
    running_processes = `ps -f`.split("\n").map{ |line| line.split(/\s+/) }
    pid_index = running_processes.detect { |p| p.include?("UID") }.index("UID") + 1
    parent = running_processes.detect { |p| p.include?(file) and not p.include?("sh") }
    raise "Unable to find parent in #{running_processes} with #{file}" unless parent
    `kill -s #{signal} #{parent.fetch(pid_index)}`
  end
end