From e9f6ce617be2f67b098e6fb689592f70c0009eb2 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Tue, 18 Dec 2018 20:25:35 +0200 Subject: [PATCH] Add option to set parallel test worker count to the physical core count of the machine (#34735) * Add option to set parallel test worker count to the physical core count of the machine Also, use the physical core count of the machine as the default number of workers, and generate the `test_helper.rb` file with `parallelize(workers: :number_of_processors)` Closes #34734 * Ensure that we always test parallel testing Since #34734 we decided to use the physical core count of the machine as the default number of workers in the parallel testing, we need to ensure that some tests use at least 2 workers because we could run those tests on VM that has only 1 physical core. It also fixes tests failures on the CI since Travis server we are using has only one physical core. See https://travis-ci.org/rails/rails/jobs/469281088#L2352 --- activesupport/lib/active_support/test_case.rb | 9 +++++++-- guides/source/testing.md | 8 ++++---- .../app/templates/test/test_helper.rb.tt | 4 ++-- railties/test/application/test_runner_test.rb | 19 ++++++++++++++----- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/activesupport/lib/active_support/test_case.rb b/activesupport/lib/active_support/test_case.rb index ef12c6b9b0..7be4108ed7 100644 --- a/activesupport/lib/active_support/test_case.rb +++ b/activesupport/lib/active_support/test_case.rb @@ -12,6 +12,7 @@ require "active_support/testing/constant_lookup" require "active_support/testing/time_helpers" require "active_support/testing/file_fixtures" require "active_support/testing/parallelization" +require "concurrent/utility/processor_counter" module ActiveSupport class TestCase < ::Minitest::Test @@ -58,16 +59,20 @@ module ActiveSupport # If the number of workers is set to +1+ or fewer, the tests will not be # parallelized. # + # If +workers+ is set to +:number_of_processors+, the number of workers will be + # set to the actual core count on the machine you are on. + # # The default parallelization method is to fork processes. If you'd like to # use threads instead you can pass with: :threads to the +parallelize+ # method. Note the threaded parallelization does not create multiple # database and will not work with system tests at this time. # - # parallelize(workers: 2, with: :threads) + # parallelize(workers: :number_of_processors, with: :threads) # # The threaded parallelization uses minitest's parallel executor directly. # The processes parallelization uses a Ruby DRb server. - def parallelize(workers: 2, with: :processes) + def parallelize(workers: :number_of_processors, with: :processes) + workers = Concurrent.physical_processor_count if workers == :number_of_processors workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"] return if workers <= 1 diff --git a/guides/source/testing.md b/guides/source/testing.md index 9541598b26..f34f9d95f4 100644 --- a/guides/source/testing.md +++ b/guides/source/testing.md @@ -473,8 +473,8 @@ takes your entire test suite to run. ### Parallel testing with processes The default parallelization method is to fork processes using Ruby's DRb system. The processes -are forked based on the number of workers provided. The default is 2, but can be changed by the -number passed to the parallelize method. +are forked based on the number of workers provided. The default number is the actual core count +on the machine you are on, but can be changed by the number passed to the parallelize method. To enable parallelization add the following to your `test_helper.rb`: @@ -516,7 +516,7 @@ class ActiveSupport::TestCase # cleanup databases end - parallelize(workers: 2) + parallelize(workers: :number_of_processors) end ``` @@ -531,7 +531,7 @@ To change the parallelization method to use threads over forks put the following ```ruby class ActiveSupport::TestCase - parallelize(workers: 2, with: :threads) + parallelize(workers: :number_of_processors, with: :threads) end ``` diff --git a/railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt b/railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt index c06cd525d7..47b4cf745c 100644 --- a/railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt +++ b/railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt @@ -5,9 +5,9 @@ require 'rails/test_help' class ActiveSupport::TestCase # Run tests in parallel with specified workers <% if defined?(JRUBY_VERSION) || Gem.win_platform? -%> - parallelize(workers: 2, with: :threads) + parallelize(workers: :number_of_processors, with: :threads) <%- else -%> - parallelize(workers: 2) + parallelize(workers: :number_of_processors) <% end -%> <% unless options[:skip_active_record] -%> diff --git a/railties/test/application/test_runner_test.rb b/railties/test/application/test_runner_test.rb index 140703e118..55cfda82ea 100644 --- a/railties/test/application/test_runner_test.rb +++ b/railties/test/application/test_runner_test.rb @@ -523,6 +523,8 @@ module ApplicationTests end def test_run_in_parallel_with_processes + substitute_arguments_of_parallelize_method("workers: 2, with: :processes") + file_name = create_parallel_processes_test_file app_file "db/schema.rb", <<-RUBY @@ -540,11 +542,7 @@ module ApplicationTests end def test_run_in_parallel_with_threads - app_path("/test/test_helper.rb") do |file_name| - file = File.read(file_name) - file.sub!(/parallelize\(([^\)]*)\)/, "parallelize(\\1, with: :threads)") - File.write(file_name, file) - end + substitute_arguments_of_parallelize_method("workers: 2, with: :threads") file_name = create_parallel_threads_test_file @@ -563,6 +561,8 @@ module ApplicationTests end def test_run_in_parallel_with_unmarshable_exception + substitute_arguments_of_parallelize_method("workers: 2, with: :processes") + file = app_file "test/fail_test.rb", <<-RUBY require "test_helper" class FailTest < ActiveSupport::TestCase @@ -587,6 +587,7 @@ module ApplicationTests end def test_run_in_parallel_with_unknown_object + substitute_arguments_of_parallelize_method("workers: 2, with: :processes") create_scaffold app_file "config/environments/test.rb", <<-RUBY Rails.application.configure do @@ -966,6 +967,14 @@ module ApplicationTests RUBY end + def substitute_arguments_of_parallelize_method(arguments) + app_path("test/test_helper.rb") do |file_name| + file = File.read(file_name) + file.sub!(/parallelize\(([^\)]*)\)/, "parallelize(#{arguments})") + File.write(file_name, file) + end + end + def create_env_test app_file "test/unit/env_test.rb", <<-RUBY require 'test_helper'