1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
|
# frozen_string_literal: true
require 'shellwords'
require 'parallel_tests'
module ParallelTests
module Test
class Runner
RuntimeLogTooSmallError = Class.new(StandardError)
class << self
# --- usually overwritten by other runners
def runtime_log
'tmp/parallel_runtime_test.log'
end
def test_suffix
/_(test|spec).rb$/
end
def default_test_folder
"test"
end
def test_file_name
"test"
end
def run_tests(test_files, process_number, num_processes, options)
require_list = test_files.map { |file| file.gsub(" ", "\\ ") }.join(" ")
cmd = [
*executable,
'-Itest',
'-e',
"%w[#{require_list}].each { |f| require %{./\#{f}} }",
'--',
*options[:test_options]
]
execute_command(cmd, process_number, num_processes, options)
end
# ignores other commands runner noise
def line_is_result?(line)
line =~ /\d+ failure(?!:)/
end
# --- usually used by other runners
# finds all tests and partitions them into groups
def tests_in_groups(tests, num_groups, options = {})
tests = tests_with_size(tests, options)
Grouper.in_even_groups_by_size(tests, num_groups, options)
end
def tests_with_size(tests, options)
tests = find_tests(tests, options)
case options[:group_by]
when :found
tests.map! { |t| [t, 1] }
when :filesize
sort_by_filesize(tests)
when :runtime
sort_by_runtime(
tests, runtimes(tests, options),
options.merge(allowed_missing: (options[:allowed_missing_percent] || 50) / 100.0)
)
when nil
# use recorded test runtime if we got enough data
runtimes = begin
runtimes(tests, options)
rescue StandardError
[]
end
if runtimes.size * 1.5 > tests.size
puts "Using recorded test runtime" unless options[:quiet]
sort_by_runtime(tests, runtimes)
else
sort_by_filesize(tests)
end
else
raise ArgumentError, "Unsupported option #{options[:group_by]}"
end
tests
end
def execute_command(cmd, process_number, num_processes, options)
number = test_env_number(process_number, options).to_s
env = (options[:env] || {}).merge(
"TEST_ENV_NUMBER" => number,
"PARALLEL_TEST_GROUPS" => num_processes.to_s,
"PARALLEL_PID_FILE" => ParallelTests.pid_file_path
)
cmd = ["nice", *cmd] if options[:nice]
# being able to run with for example `-output foo-$TEST_ENV_NUMBER` worked originally and is convenient
cmd.map! { |c| c.gsub("$TEST_ENV_NUMBER", number).gsub("${TEST_ENV_NUMBER}", number) }
print_command(cmd, env) if report_process_command?(options) && !options[:serialize_stdout]
execute_command_and_capture_output(env, cmd, options)
end
def print_command(command, env)
env_str = ['TEST_ENV_NUMBER', 'PARALLEL_TEST_GROUPS'].map { |e| "#{e}=#{env[e]}" }.join(' ')
puts [env_str, Shellwords.shelljoin(command)].compact.join(' ')
end
def execute_command_and_capture_output(env, cmd, options)
pid = nil
popen_options = { pgroup: true }
popen_options[:err] = [:child, :out] if options[:combine_stderr]
output = IO.popen(env, cmd, popen_options) do |io|
pid = io.pid
ParallelTests.pids.add(pid)
capture_output(io, env, options)
end
ParallelTests.pids.delete(pid) if pid
exitstatus = $?.exitstatus
seed = output[/seed (\d+)/, 1]
output = "#{Shellwords.shelljoin(cmd)}\n#{output}" if report_process_command?(options) && options[:serialize_stdout]
{ env: env, stdout: output, exit_status: exitstatus, command: cmd, seed: seed }
end
def find_results(test_output)
test_output.lines.map do |line|
line.chomp!
line.gsub!(/\e\[\d+m/, '') # remove color coding
next unless line_is_result?(line)
line
end.compact
end
def test_env_number(process_number, options = {})
if process_number == 0 && !options[:first_is_1]
''
else
process_number + 1
end
end
def summarize_results(results)
sums = sum_up_results(results)
sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ')
end
# remove old seed and add new seed
def command_with_seed(cmd, seed)
clean = remove_command_arguments(cmd, '--seed')
[*clean, '--seed', seed]
end
protected
def executable
if (executable = ENV['PARALLEL_TESTS_EXECUTABLE'])
[executable]
else
determine_executable
end
end
def determine_executable
["ruby"]
end
def sum_up_results(results)
results = results.join(' ').gsub(/s\b/, '') # combine and singularize results
counts = results.scan(/(\d+) (\w+)/)
counts.each_with_object(Hash.new(0)) do |(number, word), sum|
sum[word] += number.to_i
end
end
# read output of the process and print it in chunks
def capture_output(out, env, options = {})
result = +""
begin
loop do
read = out.readpartial(1000000) # read whatever chunk we can get
if Encoding.default_internal
read = read.force_encoding(Encoding.default_internal)
end
result << read
unless options[:serialize_stdout]
message = read
message = "[TEST GROUP #{env['TEST_ENV_NUMBER']}] #{message}" if options[:prefix_output_with_test_env_number]
$stdout.print message
$stdout.flush
end
end
rescue EOFError
nil
end
result
end
def sort_by_runtime(tests, runtimes, options = {})
allowed_missing = options[:allowed_missing] || 1.0
allowed_missing = tests.size * allowed_missing
# set know runtime for each test
tests.sort!
tests.map! do |test|
allowed_missing -= 1 unless time = runtimes[test]
if allowed_missing < 0
log = options[:runtime_log] || runtime_log
raise RuntimeLogTooSmallError, "Runtime log file '#{log}' does not contain sufficient data to sort #{tests.size} test files, please update or remove it."
end
[test, time]
end
puts "Runtime found for #{tests.count(&:last)} of #{tests.size} tests" if options[:verbose]
set_unknown_runtime tests, options
end
def runtimes(tests, options)
log = options[:runtime_log] || runtime_log
lines = File.read(log).split("\n")
lines.each_with_object({}) do |line, times|
test, _, time = line.rpartition(':')
next unless test && time
times[test] = time.to_f if tests.include?(test)
end
end
def sort_by_filesize(tests)
tests.sort!
tests.map! { |test| [test, File.stat(test).size] }
end
def find_tests(tests, options = {})
suffix_pattern = options[:suffix] || test_suffix
include_pattern = options[:pattern] || //
exclude_pattern = options[:exclude_pattern]
(tests || []).flat_map do |file_or_folder|
if File.directory?(file_or_folder)
files = files_in_folder(file_or_folder, options)
files = files.grep(suffix_pattern).grep(include_pattern)
files -= files.grep(exclude_pattern) if exclude_pattern
files
else
file_or_folder
end
end.uniq
end
def files_in_folder(folder, options = {})
pattern = if options[:symlinks] == false # not nil or true
"**/*"
else
# follow one symlink and direct children
# http://stackoverflow.com/questions/357754/can-i-traverse-symlinked-directories-in-ruby-with-a-glob
"**{,/*/**}/*"
end
Dir[File.join(folder, pattern)].uniq.sort
end
def remove_command_arguments(command, *args)
remove_next = false
command.select do |arg|
if remove_next
remove_next = false
false
elsif args.include?(arg)
remove_next = true
false
else
true
end
end
end
private
# fill gaps with unknown-runtime if given, average otherwise
# NOTE: an optimization could be doing runtime by average runtime per file size, but would need file checks
def set_unknown_runtime(tests, options)
known, unknown = tests.partition(&:last)
return if unknown.empty?
unknown_runtime = options[:unknown_runtime] ||
(known.empty? ? 1 : known.map!(&:last).sum / known.size) # average
unknown.each { |set| set[1] = unknown_runtime }
end
def report_process_command?(options)
options[:verbose] || options[:verbose_command]
end
end
end
end
end
|