Add test parallelization to Rails

Provides both a forked process and threaded parallelization options. To
use add `parallelize` to your test suite.

Takes a `workers` argument that controls how many times the process
is forked. For each process a new database will be created suffixed
with the worker number; test-database-0 and test-database-1
respectively.

If `ENV["PARALLEL_WORKERS"]` is set the workers argument will be ignored
and the environment variable will be used instead. This is useful for CI
environments, or other environments where you may need more workers than
you do for local testing.

If the number of workers is set to `1` or fewer, the tests will not be
parallelized.

The default parallelization method is to fork processes. If you'd like to
use threads instead you can pass `with: :threads` to the `parallelize`
method. Note the threaded parallelization does not create multiple
database and will not work with system tests at this time.

parallelize(workers: 2, with: :threads)

The threaded parallelization uses Minitest's parallel exector directly.
The processes paralleliztion uses a Ruby Drb server.

For parallelization via threads a setup hook and cleanup hook are
provided.

```
class ActiveSupport::TestCase
  parallelize_setup do |worker|
    # setup databases
  end

  parallelize_teardown do |worker|
    # cleanup database
  end

  parallelize(workers: 2)
end
```

[Eileen M. Uchitelle, Aaron Patterson]

activerecord/lib/active_record.rb

@@ -163,6 +163,7 @@ module ActiveRecord
       "active_record/tasks/postgresql_database_tasks"
   end
 
+  autoload :TestDatabases, "active_record/test_databases"
   autoload :TestFixtures, "active_record/fixtures"
 
   def self.eager_load!

activerecord/lib/active_record/fixtures.rb

@@ -874,6 +874,7 @@ module ActiveRecord
       class_attribute :use_instantiated_fixtures, default: false # true, false, or :no_instances
       class_attribute :pre_loaded_fixtures, default: false
       class_attribute :config, default: ActiveRecord::Base
+      class_attribute :lock_threads, default: true
     end
 
     module ClassMethods
@@ -973,7 +974,7 @@ module ActiveRecord
         @fixture_connections = enlist_fixture_connections
         @fixture_connections.each do |connection|
           connection.begin_transaction joinable: false
-          connection.pool.lock_thread = true
+          connection.pool.lock_thread = true if lock_threads
         end
 
         # When connections are established in the future, begin a transaction too
@@ -989,7 +990,7 @@ module ActiveRecord
 
             if connection && !@fixture_connections.include?(connection)
               connection.begin_transaction joinable: false
-              connection.pool.lock_thread = true
+              connection.pool.lock_thread = true if lock_threads
               @fixture_connections << connection
             end
           end

activerecord/lib/active_record/test_databases.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "active_support/testing/parallelization"
+
+module ActiveRecord
+  module TestDatabases # :nodoc:
+    ActiveSupport::Testing::Parallelization.after_fork_hook do |i|
+      create_and_migrate(i, spec_name: Rails.env)
+    end
+
+    ActiveSupport::Testing::Parallelization.run_cleanup_hook do |i|
+      drop(i, spec_name: Rails.env)
+    end
+
+    def self.create_and_migrate(i, spec_name:)
+      old, ENV["VERBOSE"] = ENV["VERBOSE"], "false"
+
+      connection_spec = ActiveRecord::Base.configurations[spec_name]
+
+      connection_spec["database"] += "-#{i}"
+      ActiveRecord::Tasks::DatabaseTasks.create(connection_spec)
+      ActiveRecord::Base.establish_connection(connection_spec)
+      ActiveRecord::Tasks::DatabaseTasks.migrate
+    ensure
+      ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[Rails.env])
+      ENV["VERBOSE"] = old
+    end
+
+    def self.drop(i, spec_name:)
+      old, ENV["VERBOSE"] = ENV["VERBOSE"], "false"
+      connection_spec = ActiveRecord::Base.configurations[spec_name]
+
+      ActiveRecord::Tasks::DatabaseTasks.drop(connection_spec)
+    ensure
+      ENV["VERBOSE"] = old
+    end
+  end
+end

activesupport/CHANGELOG.md

@@ -1,2 +1,7 @@
+* Adds parallel testing to Rails
+
+  Parallelize your test suite with forked processes or threads.
+
+  *Eileen M. Uchitelle*, *Aaron Patterson*
 
 Please check [5-2-stable](https://github.com/rails/rails/blob/5-2-stable/activesupport/CHANGELOG.md) for previous changes.

activesupport/lib/active_support/test_case.rb

@@ -11,6 +11,7 @@ require "active_support/testing/isolation"
 require "active_support/testing/constant_lookup"
 require "active_support/testing/time_helpers"
 require "active_support/testing/file_fixtures"
+require "active_support/testing/parallelization"
 
 module ActiveSupport
   class TestCase < ::Minitest::Test
@@ -39,6 +40,91 @@ module ActiveSupport
       def test_order
         ActiveSupport.test_order ||= :random
       end
+
+      # Parallelizes the test suite.
+      #
+      # Takes a `workers` argument that controls how many times the process
+      # is forked. For each process a new database will be created suffixed
+      # with the worker number.
+      #
+      #   test-database-0
+      #   test-database-1
+      #
+      # If `ENV["PARALLEL_WORKERS"]` is set the workers argument will be ignored
+      # and the environment variable will be used instead. This is useful for CI
+      # environments, or other environments where you may need more workers than
+      # you do for local testing.
+      #
+      # If the number of workers is set to `1` or fewer, the tests will not be
+      # parallelized.
+      #
+      # The default parallelization method is to fork processes. If you'd like to
+      # use threads instead you can pass `with: :threads` to the `parallelize`
+      # method. Note the threaded parallelization does not create multiple
+      # database and will not work with system tests at this time.
+      #
+      #   parallelize(workers: 2, with: :threads)
+      #
+      # The threaded parallelization uses Minitest's parallel exector directly.
+      # The processes paralleliztion uses a Ruby Drb server.
+      def parallelize(workers: 2, with: :processes)
+        workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"]
+
+        return if workers <= 1
+
+        executor = case with
+                   when :processes
+                     Testing::Parallelization.new(workers)
+                   when :threads
+                     Minitest::Parallel::Executor.new(workers)
+                   else
+                     raise ArgumentError, "#{with} is not a supported parallelization exectutor."
+        end
+
+        self.lock_threads = false if defined?(self.lock_threads) && with == :threads
+
+        Minitest.parallel_executor = executor
+
+        parallelize_me!
+      end
+
+      # Set up hook for parallel testing. This can be used if you have multiple
+      # databases or any behavior that needs to be run after the process is forked
+      # but before the tests run.
+      #
+      # Note: this feature is not available with the threaded parallelization.
+      #
+      # In your +test_helper.rb+ add the following:
+      #
+      #   class ActiveSupport::TestCase
+      #     parallelize_setup do
+      #       # create databases
+      #     end
+      #   end
+      def parallelize_setup(&block)
+        ActiveSupport::Testing::Parallelization.after_fork_hook do |worker|
+          yield worker
+        end
+      end
+
+      # Clean up hook for parallel testing. This can be used to drop databases
+      # if your app uses multiple write/read databases or other clean up before
+      # the tests finish. This runs before the forked process is closed.
+      #
+      # Note: this feature is not available with the threaded parallelization.
+      #
+      # In your +test_helper.rb+ add the following:
+      #
+      #   class ActiveSupport::TestCase
+      #     parallelize_teardown do
+      #       # drop databases
+      #     end
+      #   end
+      def parallelize_teardown(&block)
+        ActiveSupport::Testing::Parallelization.run_cleanup_hook do |worker|
+          yield worker
+        end
+      end
     end
 
     alias_method :method_name, :name

activesupport/lib/active_support/testing/parallelization.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "drb"
+require "drb/unix"
+
+module ActiveSupport
+  module Testing
+    class Parallelization # :nodoc:
+      class Server
+        include DRb::DRbUndumped
+
+        def initialize
+          @queue = Queue.new
+        end
+
+        def record(reporter, result)
+          reporter.synchronize do
+            reporter.record(result)
+          end
+        end
+
+        def <<(o)
+          @queue << o
+        end
+
+        def pop; @queue.pop; end
+      end
+
+      @after_fork_hooks = []
+
+      def self.after_fork_hook(&blk)
+        @after_fork_hooks << blk
+      end
+
+      def self.after_fork_hooks
+        @after_fork_hooks
+      end
+
+      @run_cleanup_hooks = []
+
+      def self.run_cleanup_hook(&blk)
+        @run_cleanup_hooks << blk
+      end
+
+      def self.run_cleanup_hooks
+        @run_cleanup_hooks
+      end
+
+      def initialize(queue_size)
+        @queue_size = queue_size
+        @queue      = Server.new
+        @pool       = []
+
+        @url = DRb.start_service("drbunix:", @queue).uri
+      end
+
+      def after_fork(worker)
+        self.class.after_fork_hooks.each do |cb|
+          cb.call(worker)
+        end
+      end
+
+      def run_cleanup(worker)
+        self.class.run_cleanup_hooks.each do |cb|
+          cb.call(worker)
+        end
+      end
+
+      def start
+        @pool = @queue_size.times.map do |worker|
+          fork do
+            DRb.stop_service
+
+            after_fork(worker)
+
+            queue = DRbObject.new_with_uri(@url)
+
+            while job = queue.pop
+              klass    = job[0]
+              method   = job[1]
+              reporter = job[2]
+              result   = Minitest.run_one_method(klass, method)
+
+              queue.record(reporter, result)
+            end
+
+            run_cleanup(worker)
+          end
+        end
+      end
+
+      def <<(work)
+        @queue << work
+      end
+
+      def shutdown
+        @queue_size.times { @queue << nil }
+        @pool.each { |pid| Process.waitpid pid }
+      end
+    end
+  end
+end

guides/source/testing.md

@@ -462,6 +462,89 @@ Rails options:
     -c, --[no-]color                 Enable color in the output
 ```
 
+Parallel Testing
+----------------
+
+Parallel testing allows you to parallelize your test suite. While forking processes is the
+default method, threading is supported as well. Running tests in parallel reduces the time it
+takes your entire test suite to run.
+
+## Parallel testing with processes
+
+The default parallelization method is to fork processes using Ruby's DRb system. The processes
+are forked based on the number of workers provided. The default is 2, but can be changed by the
+number passed to the parallelize method. Active Record automatically handles creating and
+migrating a new database for each worker to use.
+
+To enable parallelization add the following to your `test_helper.rb`:
+
+```
+class ActiveSupport::TestCase
+  parallelize(workers: 2)
+end
+```
+
+The number of workers passed is the number of times the process will be forked. You may want to
+parallelize your local test suite differently from your CI, so an environment variable is provided
+to be able to easily change the number of workers a test run should use:
+
+```
+PARALLEL_WORKERS=15 bin/rails test
+```
+
+When parallelizing tests, Active Record automatically handles creating and migrating a database for each
+process. The databases will be suffixed with the number corresponding to the worker. For example, if you
+have 2 workers the tests will create `test-database-0` and `test-database-1` respectively.
+
+If the number of workers passed is 1 or fewer the processes will not be forked and the tests will not
+be parallelized and the tests will use the original `test-database` database.
+
+Two hooks are provided, one runs when the process is forked, and one runs before the processes are closed.
+These can be useful if your app uses multiple databases or perform other tasks that depend on the number of
+workers.
+
+The `parallelize_setup` method is called right after the processes are forked. The `parallelize_teardown` metod
+is called right before the processes are closed.
+
+```
+class ActiveSupport::TestCase
+  parallelize_setup do |worker|
+    # setup databases
+  end
+
+  parallelize_teardown do |worker|
+    # cleanup database
+  end
+
+  parallelize(workers: 2)
+end
+```
+
+These methods are not needed or available when using parallel testing with threads.
+
+## Parallel testing with threads
+
+If you prefer using threads or are using JRuby, a threaded parallelization option is provided. The threaded
+parallelizer is backed by Minitest's `Parallel::Executor`.
+
+To change the parallelization method to use threads over forks put the following in your `test_helper.rb`
+
+```
+class ActiveSupport::TestCase
+  parallelize(workers: 2, with: :threads)
+end
+```
+
+Rails applications generated from JRuby will automatically include the `with: :threads` option.
+
+The number of workers passed to `parallelize` determines the number of threads the tests will use. You may
+want to parallelize your local test suite differently from your CI, so an environment variable is provided
+to be able to easily change the number of workers a test run should use:
+
+```
+PARALLEL_WORKERS=15 bin/rails test
+```
+
 The Test Database
 -----------------
 

railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt

@@ -3,6 +3,13 @@ require_relative '../config/environment'
 require 'rails/test_help'
 
 class ActiveSupport::TestCase
+  # Run tests in parallel with specified workers
+<% if defined?(JRUBY_VERSION) -%>
+  parallelize(workers: 2, with: :threads)
+<%- else -%>
+  parallelize(workers: 2)
+<% end -%>
+
 <% unless options[:skip_active_record] -%>
   # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.
   fixtures :all

railties/lib/rails/test_help.rb

@@ -22,6 +22,7 @@ if defined?(ActiveRecord::Base)
 
   module ActiveSupport
     class TestCase
+      include ActiveRecord::TestDatabases
       include ActiveRecord::TestFixtures
       self.fixture_path = "#{Rails.root}/test/fixtures/"
       self.file_fixture_path = fixture_path + "files"

railties/test/application/test_runner_test.rb

@@ -502,10 +502,10 @@ module ApplicationTests
     end
 
     def test_output_inline_by_default
-      create_test_file :models, "post", pass: false
+      create_test_file :models, "post", pass: false, print: false
 
       output = run_test_command("test/models/post_test.rb")
-      expect = %r{Running:\n\nPostTest\nF\n\nFailure:\nPostTest#test_truth \[[^\]]+test/models/post_test.rb:6\]:\nwups!\n\nbin/rails test test/models/post_test.rb:4\n\n\n\n}
+      expect = %r{Running:\n\nF\n\nFailure:\nPostTest#test_truth \[[^\]]+test/models/post_test.rb:6\]:\nwups!\n\nbin/rails test test/models/post_test.rb:4\n\n\n\n}
       assert_match expect, output
     end
 
@@ -523,6 +523,29 @@ module ApplicationTests
         capture(:stderr) { run_test_command("test/models/post_test.rb --fail-fast", stderr: true) })
     end
 
+    def test_run_in_parallel_with_processes
+      file_name = create_parallel_processes_test_file
+
+      output = run_test_command(file_name)
+
+      assert_match %r{Finished in.*\n2 runs, 2 assertions}, output
+    end
+
+    def test_run_in_parallel_with_threads
+      app_path("/test/test_helper.rb") do |file_name|
+        file = File.read(file_name)
+        file.sub!(/parallelize\(([^\)]*)\)/, "parallelize(\\1, with: :threads)")
+        puts file
+        File.write(file_name, file)
+      end
+
+      file_name = create_parallel_threads_test_file
+
+      output = run_test_command(file_name)
+
+      assert_match %r{Finished in.*\n2 runs, 2 assertions}, output
+    end
+
     def test_raise_error_when_specified_file_does_not_exist
       error = capture(:stderr) { run_test_command("test/not_exists.rb", stderr: true) }
       assert_match(%r{cannot load such file.+test/not_exists\.rb}, error)
@@ -800,19 +823,70 @@ module ApplicationTests
         RUBY
       end
 
-      def create_test_file(path = :unit, name = "test", pass: true)
+      def create_test_file(path = :unit, name = "test", pass: true, print: true)
         app_file "test/#{path}/#{name}_test.rb", <<-RUBY
           require 'test_helper'
 
           class #{name.camelize}Test < ActiveSupport::TestCase
             def test_truth
-              puts "#{name.camelize}Test"
+              puts "#{name.camelize}Test" if #{print}
               assert #{pass}, 'wups!'
             end
           end
         RUBY
       end
 
+      def create_parallel_processes_test_file
+        app_file "test/models/parallel_test.rb", <<-RUBY
+          require 'test_helper'
+
+          class ParallelTest < ActiveSupport::TestCase
+            RD1, WR1 = IO.pipe
+            RD2, WR2 = IO.pipe
+
+            test "one" do
+              WR1.close
+              assert_equal "x", RD1.read(1) # blocks until two runs
+
+              RD2.close
+              WR2.write "y" # Allow two to run
+              WR2.close
+            end
+
+            test "two" do
+              RD1.close
+              WR1.write "x" # Allow one to run
+              WR1.close
+
+              WR2.close
+              assert_equal "y", RD2.read(1) # blocks until one runs
+            end
+          end
+        RUBY
+      end
+
+      def create_parallel_threads_test_file
+        app_file "test/models/parallel_test.rb", <<-RUBY
+          require 'test_helper'
+
+          class ParallelTest < ActiveSupport::TestCase
+            Q1 = Queue.new
+            Q2 = Queue.new
+            test "one" do
+              assert_equal "x", Q1.pop # blocks until two runs
+
+              Q2 << "y"
+            end
+
+            test "two" do
+              Q1 << "x"
+
+              assert_equal "y", Q2.pop # blocks until one runs
+            end
+          end
+        RUBY
+      end
+
       def create_env_test
         app_file "test/unit/env_test.rb", <<-RUBY
           require 'test_helper'

railties/test/application/test_test.rb

@@ -7,10 +7,15 @@ module ApplicationTests
     include ActiveSupport::Testing::Isolation
 
     def setup
+      @old = ENV["PARALLEL_WORKERS"]
+      ENV["PARALLEL_WORKERS"] = "0"
+
       build_app
     end
 
     def teardown
+      ENV["PARALLEL_WORKERS"] = @old
+
       teardown_app
     end
 

railties/test/isolation/abstract_unit.rb

@@ -38,7 +38,12 @@ module TestHelpers
     end
 
     def app_path(*args)
-      tmp_path(*%w[app] + args)
+      path = tmp_path(*%w[app] + args)
+      if block_given?
+        yield path
+      else
+        path
+      end
     end
 
     def framework_path