Load framework test files in deterministic order

`Dir.glob` doesn't guarantee the order of its results:

https://ruby-doc.org/core-2.6.5/Dir.html#method-c-glob

> Case sensitivity depends on your system (File::FNM_CASEFOLD is
> ignored), as does the order in which the results are returned.

Minitest stores a list of all test cases in the order that they were
defined; it shuffles them before they're run, but doesn't sort them:

https://github.com/seattlerb/minitest/blob/v5.13.0/lib/minitest.rb#L1048
https://github.com/seattlerb/minitest/blob/v5.13.0/lib/minitest.rb#L156

This means that the order in which framework tests run is platform
dependent, and running a test command that failed in CI locally won't
necessarily reproduce the error, even when the same seed is provided.

`Rake::FileList` resolves glob patterns to a sorted list of files:

https://github.com/ruby/rake/blob/v13.0.1/lib/rake/file_list.rb#L408

By using `Rake::FileList` instead of `Dir.glob`, framework tests will
always run in the same order when given the same seed, and reproducing
order dependent CI failures will be easier.
This commit is contained in:
Eugene Kenny 2019-12-16 16:55:06 +00:00
parent 3578f6929b
commit ee525ff663
5 changed files with 8 additions and 8 deletions

View File

@ -12,7 +12,7 @@ task :package
Rake::TestTask.new do |t|
t.libs << "test"
t.test_files = Dir.glob("#{__dir__}/test/**/*_test.rb")
t.test_files = FileList["#{__dir__}/test/**/*_test.rb"]
t.warning = true
t.verbose = true
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)

View File

@ -2,7 +2,7 @@
require "rake/testtask"
test_files = Dir.glob("test/**/*_test.rb")
test_files = FileList["test/**/*_test.rb"]
desc "Default Task"
task default: :test

View File

@ -23,7 +23,7 @@ namespace :test do
Rake::TestTask.new(:template) do |t|
t.libs << "test"
t.test_files = Dir.glob("test/template/**/*_test.rb")
t.test_files = FileList["test/template/**/*_test.rb"]
t.warning = true
t.verbose = true
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
@ -81,7 +81,7 @@ namespace :test do
# Active Record Integration Tests
Rake::TestTask.new(:active_record) do |t|
t.libs << "test"
t.test_files = Dir.glob("test/activerecord/*_test.rb")
t.test_files = FileList["test/activerecord/*_test.rb"]
t.warning = true
t.verbose = true
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
@ -90,7 +90,7 @@ namespace :test do
# Action Pack Integration Tests
Rake::TestTask.new(:action_pack) do |t|
t.libs << "test"
t.test_files = Dir.glob("test/actionpack/**/*_test.rb")
t.test_files = FileList["test/actionpack/**/*_test.rb"]
t.warning = true
t.verbose = true
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)

View File

@ -8,7 +8,7 @@ task :package
Rake::TestTask.new do |t|
t.libs << "test"
t.test_files = Dir.glob("#{__dir__}/test/cases/**/*_test.rb")
t.test_files = FileList["#{__dir__}/test/cases/**/*_test.rb"]
t.warning = true
t.verbose = true
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)

View File

@ -52,9 +52,9 @@ end
Rake::TestTask.new(adapter => "#{adapter}:env") { |t|
adapter_short = adapter == "db2" ? adapter : adapter[/^[a-z0-9]+/]
t.libs << "test"
t.test_files = (Dir.glob("test/cases/**/*_test.rb").reject {
t.test_files = (FileList["test/cases/**/*_test.rb"].reject {
|x| x.include?("/adapters/")
} + Dir.glob("test/cases/adapters/#{adapter_short}/**/*_test.rb"))
} + FileList["test/cases/adapters/#{adapter_short}/**/*_test.rb"])
t.warning = true
t.verbose = true