File: runner.rb

package info (click to toggle)
ruby-parallel-tests 4.1.0-1
  • links: PTS, VCS
  • area: main
  • in suites: bookworm, forky, sid, trixie
  • size: 1,280 kB
  • sloc: ruby: 5,381; javascript: 30; makefile: 4
file content (299 lines) | stat: -rw-r--r-- 9,860 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
# frozen_string_literal: true
require 'shellwords'
require 'parallel_tests'

module ParallelTests
  module Test
    class Runner
      RuntimeLogTooSmallError = Class.new(StandardError)

      class << self
        # --- usually overwritten by other runners

        def runtime_log
          'tmp/parallel_runtime_test.log'
        end

        def test_suffix
          /_(test|spec).rb$/
        end

        def default_test_folder
          "test"
        end

        def test_file_name
          "test"
        end

        def run_tests(test_files, process_number, num_processes, options)
          require_list = test_files.map { |file| file.gsub(" ", "\\ ") }.join(" ")
          cmd = [
            *executable,
            '-Itest',
            '-e',
            "%w[#{require_list}].each { |f| require %{./\#{f}} }",
            '--',
            *options[:test_options]
          ]
          execute_command(cmd, process_number, num_processes, options)
        end

        # ignores other commands runner noise
        def line_is_result?(line)
          line =~ /\d+ failure(?!:)/
        end

        # --- usually used by other runners

        # finds all tests and partitions them into groups
        def tests_in_groups(tests, num_groups, options = {})
          tests = tests_with_size(tests, options)
          Grouper.in_even_groups_by_size(tests, num_groups, options)
        end

        def tests_with_size(tests, options)
          tests = find_tests(tests, options)

          case options[:group_by]
          when :found
            tests.map! { |t| [t, 1] }
          when :filesize
            sort_by_filesize(tests)
          when :runtime
            sort_by_runtime(
              tests, runtimes(tests, options),
              options.merge(allowed_missing: (options[:allowed_missing_percent] || 50) / 100.0)
            )
          when nil
            # use recorded test runtime if we got enough data
            runtimes = begin
              runtimes(tests, options)
            rescue StandardError
              []
            end
            if runtimes.size * 1.5 > tests.size
              puts "Using recorded test runtime" unless options[:quiet]
              sort_by_runtime(tests, runtimes)
            else
              sort_by_filesize(tests)
            end
          else
            raise ArgumentError, "Unsupported option #{options[:group_by]}"
          end

          tests
        end

        def execute_command(cmd, process_number, num_processes, options)
          number = test_env_number(process_number, options).to_s
          env = (options[:env] || {}).merge(
            "TEST_ENV_NUMBER" => number,
            "PARALLEL_TEST_GROUPS" => num_processes.to_s,
            "PARALLEL_PID_FILE" => ParallelTests.pid_file_path
          )
          cmd = ["nice", *cmd] if options[:nice]

          # being able to run with for example `-output foo-$TEST_ENV_NUMBER` worked originally and is convenient
          cmd.map! { |c| c.gsub("$TEST_ENV_NUMBER", number).gsub("${TEST_ENV_NUMBER}", number) }

          print_command(cmd, env) if report_process_command?(options) && !options[:serialize_stdout]

          execute_command_and_capture_output(env, cmd, options)
        end

        def print_command(command, env)
          env_str = ['TEST_ENV_NUMBER', 'PARALLEL_TEST_GROUPS'].map { |e| "#{e}=#{env[e]}" }.join(' ')
          puts [env_str, Shellwords.shelljoin(command)].compact.join(' ')
        end

        def execute_command_and_capture_output(env, cmd, options)
          pid = nil

          popen_options = { pgroup: true }
          popen_options[:err] = [:child, :out] if options[:combine_stderr]

          output = IO.popen(env, cmd, popen_options) do |io|
            pid = io.pid
            ParallelTests.pids.add(pid)
            capture_output(io, env, options)
          end
          ParallelTests.pids.delete(pid) if pid
          exitstatus = $?.exitstatus
          seed = output[/seed (\d+)/, 1]

          output = "#{Shellwords.shelljoin(cmd)}\n#{output}" if report_process_command?(options) && options[:serialize_stdout]

          { env: env, stdout: output, exit_status: exitstatus, command: cmd, seed: seed }
        end

        def find_results(test_output)
          test_output.lines.map do |line|
            line.chomp!
            line.gsub!(/\e\[\d+m/, '') # remove color coding
            next unless line_is_result?(line)
            line
          end.compact
        end

        def test_env_number(process_number, options = {})
          if process_number == 0 && !options[:first_is_1]
            ''
          else
            process_number + 1
          end
        end

        def summarize_results(results)
          sums = sum_up_results(results)
          sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ')
        end

        # remove old seed and add new seed
        def command_with_seed(cmd, seed)
          clean = remove_command_arguments(cmd, '--seed')
          [*clean, '--seed', seed]
        end

        protected

        def executable
          if (executable = ENV['PARALLEL_TESTS_EXECUTABLE'])
            [executable]
          else
            determine_executable
          end
        end

        def determine_executable
          ["ruby"]
        end

        def sum_up_results(results)
          results = results.join(' ').gsub(/s\b/, '') # combine and singularize results
          counts = results.scan(/(\d+) (\w+)/)
          counts.each_with_object(Hash.new(0)) do |(number, word), sum|
            sum[word] += number.to_i
          end
        end

        # read output of the process and print it in chunks
        def capture_output(out, env, options = {})
          result = +""
          begin
            loop do
              read = out.readpartial(1000000) # read whatever chunk we can get
              if Encoding.default_internal
                read = read.force_encoding(Encoding.default_internal)
              end
              result << read
              unless options[:serialize_stdout]
                message = read
                message = "[TEST GROUP #{env['TEST_ENV_NUMBER']}] #{message}" if options[:prefix_output_with_test_env_number]
                $stdout.print message
                $stdout.flush
              end
            end
          rescue EOFError
            nil
          end
          result
        end

        def sort_by_runtime(tests, runtimes, options = {})
          allowed_missing = options[:allowed_missing] || 1.0
          allowed_missing = tests.size * allowed_missing

          # set know runtime for each test
          tests.sort!
          tests.map! do |test|
            allowed_missing -= 1 unless time = runtimes[test]
            if allowed_missing < 0
              log = options[:runtime_log] || runtime_log
              raise RuntimeLogTooSmallError, "Runtime log file '#{log}' does not contain sufficient data to sort #{tests.size} test files, please update or remove it."
            end
            [test, time]
          end

          puts "Runtime found for #{tests.count(&:last)} of #{tests.size} tests" if options[:verbose]

          set_unknown_runtime tests, options
        end

        def runtimes(tests, options)
          log = options[:runtime_log] || runtime_log
          lines = File.read(log).split("\n")
          lines.each_with_object({}) do |line, times|
            test, _, time = line.rpartition(':')
            next unless test && time
            times[test] = time.to_f if tests.include?(test)
          end
        end

        def sort_by_filesize(tests)
          tests.sort!
          tests.map! { |test| [test, File.stat(test).size] }
        end

        def find_tests(tests, options = {})
          suffix_pattern = options[:suffix] || test_suffix
          include_pattern = options[:pattern] || //
          exclude_pattern = options[:exclude_pattern]

          (tests || []).flat_map do |file_or_folder|
            if File.directory?(file_or_folder)
              files = files_in_folder(file_or_folder, options)
              files = files.grep(suffix_pattern).grep(include_pattern)
              files -= files.grep(exclude_pattern) if exclude_pattern
              files
            else
              file_or_folder
            end
          end.uniq
        end

        def files_in_folder(folder, options = {})
          pattern = if options[:symlinks] == false # not nil or true
            "**/*"
          else
            # follow one symlink and direct children
            # http://stackoverflow.com/questions/357754/can-i-traverse-symlinked-directories-in-ruby-with-a-glob
            "**{,/*/**}/*"
          end
          Dir[File.join(folder, pattern)].uniq.sort
        end

        def remove_command_arguments(command, *args)
          remove_next = false
          command.select do |arg|
            if remove_next
              remove_next = false
              false
            elsif args.include?(arg)
              remove_next = true
              false
            else
              true
            end
          end
        end

        private

        # fill gaps with unknown-runtime if given, average otherwise
        # NOTE: an optimization could be doing runtime by average runtime per file size, but would need file checks
        def set_unknown_runtime(tests, options)
          known, unknown = tests.partition(&:last)
          return if unknown.empty?
          unknown_runtime = options[:unknown_runtime] ||
            (known.empty? ? 1 : known.map!(&:last).sum / known.size) # average
          unknown.each { |set| set[1] = unknown_runtime }
        end

        def report_process_command?(options)
          options[:verbose] || options[:verbose_command]
        end
      end
    end
  end
end