From ee525ff66353552c47fcfbcc9f2e835f0b9f8608 Mon Sep 17 00:00:00 2001
From: Eugene Kenny <elkenny@gmail.com>
Date: Mon, 16 Dec 2019 16:55:06 +0000
Subject: [PATCH] Load framework test files in deterministic order

`Dir.glob` doesn't guarantee the order of its results:

https://ruby-doc.org/core-2.6.5/Dir.html#method-c-glob

> Case sensitivity depends on your system (File::FNM_CASEFOLD is
> ignored), as does the order in which the results are returned.

Minitest stores a list of all test cases in the order that they were
defined; it shuffles them before they're run, but doesn't sort them:

https://github.com/seattlerb/minitest/blob/v5.13.0/lib/minitest.rb#L1048
https://github.com/seattlerb/minitest/blob/v5.13.0/lib/minitest.rb#L156

This means that the order in which framework tests run is platform
dependent, and running a test command that failed in CI locally won't
necessarily reproduce the error, even when the same seed is provided.

`Rake::FileList` resolves glob patterns to a sorted list of files:

https://github.com/ruby/rake/blob/v13.0.1/lib/rake/file_list.rb#L408

By using `Rake::FileList` instead of `Dir.glob`, framework tests will
always run in the same order when given the same seed, and reproducing
order dependent CI failures will be easier.
---
 actioncable/Rakefile  | 2 +-
 actionpack/Rakefile   | 2 +-
 actionview/Rakefile   | 6 +++---
 activemodel/Rakefile  | 2 +-
 activerecord/Rakefile | 4 ++--
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/actioncable/Rakefile b/actioncable/Rakefile
index 34b8ee5f80..d61d38d8e7 100644
--- a/actioncable/Rakefile
+++ b/actioncable/Rakefile
@@ -12,7 +12,7 @@ task :package
 
 Rake::TestTask.new do |t|
   t.libs << "test"
-  t.test_files = Dir.glob("#{__dir__}/test/**/*_test.rb")
+  t.test_files = FileList["#{__dir__}/test/**/*_test.rb"]
   t.warning = true
   t.verbose = true
   t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
diff --git a/actionpack/Rakefile b/actionpack/Rakefile
index e99eb1723a..cddc536d3b 100644
--- a/actionpack/Rakefile
+++ b/actionpack/Rakefile
@@ -2,7 +2,7 @@
 
 require "rake/testtask"
 
-test_files = Dir.glob("test/**/*_test.rb")
+test_files = FileList["test/**/*_test.rb"]
 
 desc "Default Task"
 task default: :test
diff --git a/actionview/Rakefile b/actionview/Rakefile
index 237e458b6f..73be000231 100644
--- a/actionview/Rakefile
+++ b/actionview/Rakefile
@@ -23,7 +23,7 @@ namespace :test do
 
   Rake::TestTask.new(:template) do |t|
     t.libs << "test"
-    t.test_files = Dir.glob("test/template/**/*_test.rb")
+    t.test_files = FileList["test/template/**/*_test.rb"]
     t.warning = true
     t.verbose = true
     t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
@@ -81,7 +81,7 @@ namespace :test do
     # Active Record Integration Tests
     Rake::TestTask.new(:active_record) do |t|
       t.libs << "test"
-      t.test_files = Dir.glob("test/activerecord/*_test.rb")
+      t.test_files = FileList["test/activerecord/*_test.rb"]
       t.warning = true
       t.verbose = true
       t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
@@ -90,7 +90,7 @@ namespace :test do
     # Action Pack Integration Tests
     Rake::TestTask.new(:action_pack) do |t|
       t.libs << "test"
-      t.test_files = Dir.glob("test/actionpack/**/*_test.rb")
+      t.test_files = FileList["test/actionpack/**/*_test.rb"]
       t.warning = true
       t.verbose = true
       t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
diff --git a/activemodel/Rakefile b/activemodel/Rakefile
index d39f50a962..c9a9d197c1 100644
--- a/activemodel/Rakefile
+++ b/activemodel/Rakefile
@@ -8,7 +8,7 @@ task :package
 
 Rake::TestTask.new do |t|
   t.libs << "test"
-  t.test_files = Dir.glob("#{__dir__}/test/cases/**/*_test.rb")
+  t.test_files = FileList["#{__dir__}/test/cases/**/*_test.rb"]
   t.warning = true
   t.verbose = true
   t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
diff --git a/activerecord/Rakefile b/activerecord/Rakefile
index f259ae7e12..06138680ee 100644
--- a/activerecord/Rakefile
+++ b/activerecord/Rakefile
@@ -52,9 +52,9 @@ end
     Rake::TestTask.new(adapter => "#{adapter}:env") { |t|
       adapter_short = adapter == "db2" ? adapter : adapter[/^[a-z0-9]+/]
       t.libs << "test"
-      t.test_files = (Dir.glob("test/cases/**/*_test.rb").reject {
+      t.test_files = (FileList["test/cases/**/*_test.rb"].reject {
         |x| x.include?("/adapters/")
-      } + Dir.glob("test/cases/adapters/#{adapter_short}/**/*_test.rb"))
+      } + FileList["test/cases/adapters/#{adapter_short}/**/*_test.rb"])
 
       t.warning = true
       t.verbose = true