File: tasks.rb

package info (click to toggle)
ruby-parallel-tests 5.4.0-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 760 kB
  • sloc: ruby: 5,446; javascript: 16; makefile: 4
file content (309 lines) | stat: -rw-r--r-- 10,926 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
# frozen_string_literal: true
require 'rake'
require 'shellwords'

module ParallelTests
  module Tasks
    class << self
      def rails_env
        ENV['PARALLEL_RAILS_ENV'] || 'test'
      end

      def load_lib
        $LOAD_PATH << File.expand_path('..', __dir__)
        require "parallel_tests"
      end

      def purge_before_load
        if ActiveRecord.version > Gem::Version.new('4.2.0')
          Rake::Task.task_defined?('db:purge') ? 'db:purge' : 'app:db:purge'
        end
      end

      def run_in_parallel(cmd, options = {})
        load_lib

        # Using the relative path to find the binary allow to run a specific version of it
        executable = File.expand_path('../../bin/parallel_test', __dir__)
        command = ParallelTests.with_ruby_binary(executable)
        command += ['--exec', Shellwords.join(cmd)]
        command += ['-n', options[:count]] unless options[:count].to_s.empty?
        command << '--non-parallel' if options[:non_parallel]

        abort unless system(*command)
      end

      # this is a crazy-complex solution for a very simple problem:
      # removing certain lines from the output without changing the exit-status
      # normally I'd not do this, but it has been lots of fun and a great learning experience :)
      #
      # - sed does not support | without -r
      # - grep changes 0 exitstatus to 1 if nothing matches
      # - sed changes 1 exitstatus to 0
      # - pipefail makes pipe fail with exitstatus of first failed command
      # - pipefail is not supported in (zsh)
      # - defining a new rake task like silence_schema would force users to load parallel_tests in test env
      # - simple system "set -o pipefail" returns nil even though set -o pipefail exists with 0
      def suppress_output(command, ignore_regex)
        activate_pipefail = "set -o pipefail"
        remove_ignored_lines = %{(grep -v #{Shellwords.escape(ignore_regex)} || true)}

        # remove nil values (ex: #purge_before_load returns nil)
        command.compact!

        if system('/bin/bash', '-c', "#{activate_pipefail} 2>/dev/null")
          shell_command = "#{activate_pipefail} && (#{Shellwords.shelljoin(command)}) | #{remove_ignored_lines}"
          ['/bin/bash', '-c', shell_command]
        else
          command
        end
      end

      def suppress_schema_load_output(command)
        ParallelTests::Tasks.suppress_output(command, "^   ->\\|^-- ")
      end

      def check_for_pending_migrations
        ["db:abort_if_pending_migrations", "app:db:abort_if_pending_migrations"].each do |abort_migrations|
          if Rake::Task.task_defined?(abort_migrations)
            Rake::Task[abort_migrations].invoke
            break
          end
        end
      end

      # parallel:spec[:count, :pattern, :options, :pass_through]
      def parse_args(args)
        # order as given by user
        args = [args[:count], args[:pattern], args[:options], args[:pass_through]]

        # count given or empty ?
        # parallel:spec[2,models,options]
        # parallel:spec[,models,options]
        count = args.shift if args.first.to_s =~ /^\d*$/
        num_processes = (count.to_s.empty? ? nil : Integer(count))
        pattern = args.shift
        options = args.shift
        pass_through = args.shift

        [num_processes, pattern, options, pass_through]
      end

      def schema_format_based_on_rails_version
        if active_record_7_or_greater?
          ActiveRecord.schema_format
        else
          ActiveRecord::Base.schema_format
        end
      end

      def schema_type_based_on_rails_version
        if active_record_61_or_greater? || schema_format_based_on_rails_version == :ruby
          "schema"
        else
          "structure"
        end
      end

      def build_run_command(type, args)
        count, pattern, options, pass_through = ParallelTests::Tasks.parse_args(args)
        test_framework = {
          'spec' => 'rspec',
          'test' => 'test',
          'features' => 'cucumber',
          'features-spinach' => 'spinach'
        }.fetch(type)

        type = 'features' if test_framework == 'spinach'

        # Using the relative path to find the binary allow to run a specific version of it
        executable = File.expand_path('../../bin/parallel_test', __dir__)
        executable = ParallelTests.with_ruby_binary(executable)

        command = [*executable, type, '--type', test_framework]
        command += ['-n', count.to_s] if count
        command += ['--pattern', pattern] if pattern
        command += ['--test-options', options] if options
        command += Shellwords.shellsplit pass_through if pass_through
        command
      end

      def configured_databases
        return [] unless defined?(ActiveRecord) && active_record_61_or_greater?

        @@configured_databases ||= ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
      end

      def for_each_database(&block)
        # Use nil to represent all databases
        block&.call(nil)

        # skip if not rails or old rails version
        return if !defined?(ActiveRecord::Tasks::DatabaseTasks) || !ActiveRecord::Tasks::DatabaseTasks.respond_to?(:for_each)

        ActiveRecord::Tasks::DatabaseTasks.for_each(configured_databases) do |name|
          block&.call(name)
        end
      end

      private

      def active_record_7_or_greater?
        ActiveRecord.version >= Gem::Version.new('7.0')
      end

      def active_record_61_or_greater?
        ActiveRecord.version >= Gem::Version.new('6.1.0')
      end
    end
  end
end

namespace :parallel do
  desc "Setup test databases via db:setup --> parallel:setup[num_cpus]"
  task :setup, :count do |_, args|
    command = [$0, "db:setup", "RAILS_ENV=#{ParallelTests::Tasks.rails_env}"]
    ParallelTests::Tasks.run_in_parallel(ParallelTests::Tasks.suppress_schema_load_output(command), args)
  end

  ParallelTests::Tasks.for_each_database do |name|
    task_name = 'create'
    task_name += ":#{name}" if name
    desc "Create test#{" #{name}" if name} database via db:#{task_name} --> parallel:#{task_name}[num_cpus]"
    task task_name.to_sym, :count do |_, args|
      ParallelTests::Tasks.run_in_parallel(
        [$0, "db:#{task_name}", "RAILS_ENV=#{ParallelTests::Tasks.rails_env}"],
        args
      )
    end
  end

  ParallelTests::Tasks.for_each_database do |name|
    task_name = 'drop'
    task_name += ":#{name}" if name
    desc "Drop test#{" #{name}" if name} database via db:#{task_name} --> parallel:#{task_name}[num_cpus]"
    task task_name.to_sym, :count do |_, args|
      ParallelTests::Tasks.run_in_parallel(
        [
          $0,
          "db:#{task_name}",
          "RAILS_ENV=#{ParallelTests::Tasks.rails_env}",
          "DISABLE_DATABASE_ENVIRONMENT_CHECK=1"
        ],
        args
      )
    end
  end

  desc "Update test databases by dumping and loading --> parallel:prepare[num_cpus]"
  task(:prepare, [:count]) do |_, args|
    ParallelTests::Tasks.check_for_pending_migrations

    if defined?(ActiveRecord) && [:ruby, :sql].include?(ParallelTests::Tasks.schema_format_based_on_rails_version)
      # fast: dump once, load in parallel
      type = ParallelTests::Tasks.schema_type_based_on_rails_version

      Rake::Task["db:#{type}:dump"].invoke

      # remove database connection to prevent "database is being accessed by other users"
      ActiveRecord::Base.remove_connection if ActiveRecord::Base.configurations.any?

      Rake::Task["parallel:load_#{type}"].invoke(args[:count])
    else
      # slow: dump and load in in serial
      args = args.to_hash.merge(non_parallel: true) # normal merge returns nil
      task_name = Rake::Task.task_defined?('db:test:prepare') ? 'db:test:prepare' : 'app:db:test:prepare'
      ParallelTests::Tasks.run_in_parallel([$0, task_name], args)
      next
    end
  end

  # when dumping/resetting takes too long
  ParallelTests::Tasks.for_each_database do |name|
    task_name = 'migrate'
    task_name += ":#{name}" if name
    desc "Update test#{" #{name}" if name} database via db:#{task_name} --> parallel:#{task_name}[num_cpus]"
    task task_name.to_sym, :count do |_, args|
      ParallelTests::Tasks.run_in_parallel(
        [$0, "db:#{task_name}", "RAILS_ENV=#{ParallelTests::Tasks.rails_env}"],
        args
      )
    end
  end

  desc "Rollback test databases via db:rollback --> parallel:rollback[num_cpus]"
  task :rollback, :count do |_, args|
    ParallelTests::Tasks.run_in_parallel(
      [$0, "db:rollback", "RAILS_ENV=#{ParallelTests::Tasks.rails_env}"],
      args
    )
  end

  # just load the schema (good for integration server <-> no development db)
  ParallelTests::Tasks.for_each_database do |name|
    rails_task = 'db:schema:load'
    rails_task += ":#{name}" if name

    task_name = 'load_schema'
    task_name += ":#{name}" if name

    desc "Load dumped schema for test#{" #{name}" if name} database via #{rails_task} --> parallel:#{task_name}[num_cpus]"
    task task_name.to_sym, :count do |_, args|
      command = [
        $0,
        ParallelTests::Tasks.purge_before_load,
        rails_task,
        "RAILS_ENV=#{ParallelTests::Tasks.rails_env}",
        "DISABLE_DATABASE_ENVIRONMENT_CHECK=1"
      ]
      ParallelTests::Tasks.run_in_parallel(ParallelTests::Tasks.suppress_schema_load_output(command), args)
    end
  end

  # load the structure from the structure.sql file
  # (faster for rails < 6.1, deprecated after and only configured by `ActiveRecord::Base.schema_format`)
  desc "Load structure for test databases via db:schema:load --> parallel:load_structure[num_cpus]"
  task :load_structure, :count do |_, args|
    ParallelTests::Tasks.run_in_parallel(
      [
        $0,
        ParallelTests::Tasks.purge_before_load,
        "db:structure:load",
        "RAILS_ENV=#{ParallelTests::Tasks.rails_env}",
        "DISABLE_DATABASE_ENVIRONMENT_CHECK=1"
      ],
      args
    )
  end

  desc "Load the seed data from db/seeds.rb via db:seed --> parallel:seed[num_cpus]"
  task :seed, :count do |_, args|
    ParallelTests::Tasks.run_in_parallel(
      [
        $0,
        "db:seed",
        "RAILS_ENV=#{ParallelTests::Tasks.rails_env}"
      ],
      args
    )
  end

  desc "Launch given rake command in parallel"
  task :rake, :command, :count do |_, args|
    ParallelTests::Tasks.run_in_parallel(
      [$0, args.command, "RAILS_ENV=#{ParallelTests::Tasks.rails_env}"],
      args
    )
  end

  ['test', 'spec', 'features', 'features-spinach'].each do |type|
    desc "Run #{type} in parallel with parallel:#{type}[num_cpus]"
    task type, [:count, :pattern, :options, :pass_through] do |_t, args|
      ParallelTests::Tasks.check_for_pending_migrations
      ParallelTests::Tasks.load_lib
      command = ParallelTests::Tasks.build_run_command(type, args)

      abort unless system(*command) # allow to chain tasks e.g. rake parallel:spec parallel:features
    end
  end
end