Refactor ordering of tests

* Split the sorting types into classes.
* Apply the same sorting to method sorting under the parallel
  test.

tool/lib/test/unit.rb

@@ -57,6 +57,84 @@ module Test
 
     class PendedError < AssertionFailedError; end
 
+    module Order
+      class NoSort
+        def initialize(seed)
+        end
+
+        def sort_by_name(list)
+          list
+        end
+
+        alias sort_by_string sort_by_name
+
+        def group(list)
+          # JIT first
+          jit, others = list.partition {|e| /test_jit/ =~ e}
+          jit + others
+        end
+      end
+
+      class Alpha < NoSort
+        def sort_by_name(list)
+          list.sort_by(&:name)
+        end
+
+        def sort_by_string(list)
+          list.sort
+        end
+
+      end
+
+      # shuffle test suites based on CRC32 of their names
+      Shuffle = Struct.new(:seed, :salt) do
+        def initialize(seed)
+          self.class::CRC_TBL ||= (0..255).map {|i|
+            (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) }
+          }.freeze
+
+          salt = [seed].pack("V").unpack1("H*")
+          super(seed, "\n#{salt}".freeze).freeze
+        end
+
+        def sort_by_name(list)
+          list.sort_by {|e| randomize_key(e.name)}
+        end
+
+        def sort_by_string(list)
+          list.sort_by {|e| randomize_key(e)}
+        end
+
+        def group(list)
+          list
+        end
+
+        private
+
+        def crc32(str, crc32 = 0xffffffff)
+          crc_tbl = self.class::CRC_TBL
+          str.each_byte do |data|
+            crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8)
+          end
+          crc32
+        end
+
+        def randomize_key(name)
+          crc32(salt, crc32(name)) ^ 0xffffffff
+        end
+      end
+
+      Types = {
+        random: Shuffle,
+        alpha: Alpha,
+        sorted: Alpha,
+        nosort: NoSort,
+      }
+      Types.default_proc = proc {|_, order|
+        raise "Unknown test_order: #{order.inspect}"
+      }
+    end
+
     module RunCount # :nodoc: all
       @@run_count = 0
 
@@ -103,13 +181,13 @@ module Test
         order = options[:test_order]
         if seed = options[:seed]
           order ||= :random
-          srand(seed)
+        elsif order == :random
-        else
+          seed = options[:seed] = rand(0x10000)
-          seed = options[:seed] = srand % 100_000
-          srand(seed)
           orig_args.unshift "--seed=#{seed}"
         end
         Test::Unit::TestCase.test_order = order if order
+        order = Test::Unit::TestCase.test_order
+        @order = Test::Unit::Order::Types[order].new(seed)
 
         @help = "\n" + orig_args.map { |s|
           "  " + (s =~ /[\s|&<>$()]/ ? s.inspect : s)
@@ -139,7 +217,8 @@ module Test
           (options[:filter] ||= []) << a
         end
 
-        opts.on '--test-order=random|alpha|sorted|nosort', [:random, :alpha, :sorted, :nosort] do |a|
+        orders = Test::Unit::Order::Types.keys
+        opts.on "--test-order=#{orders.join('|')}", orders do |a|
           options[:test_order] = a
         end
       end
@@ -545,16 +624,7 @@ module Test
 
         # Require needed thing for parallel running
         require 'timeout'
-        @tasks = @files.dup # Array of filenames.
+        @tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames.
-
-        case Test::Unit::TestCase.test_order
-        when :random
-          @tasks.shuffle!
-        else
-          # JIT first
-          ts = @tasks.group_by{|e| /test_jit/ =~ e ? 0 : 1}
-          @tasks = ts[0] + ts[1] if ts.size == 2
-        end
 
         @need_quit = false
         @dead_workers = []  # Array of dead workers.
@@ -1302,6 +1372,8 @@ module Test
         suites = Test::Unit::TestCase.send "#{type}_suites"
         return if suites.empty?
 
+        suites = @order.sort_by_name(suites)
+
         puts
         puts "# Running #{type}s:"
         puts
@@ -1356,6 +1428,12 @@ module Test
         filter = options[:filter]
 
         all_test_methods = suite.send "#{type}_methods"
+        if filter
+          all_test_methods.select! {|method|
+            filter === method || filter === "#{suite}##{method}"
+          }
+        end
+        all_test_methods = @order.sort_by_name(all_test_methods)
 
         leakchecker = LeakChecker.new
         if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"]
@@ -1363,10 +1441,7 @@ module Test
           trace = true
         end
 
-        assertions = all_test_methods.filter_map { |method|
+        assertions = all_test_methods.map { |method|
-          if filter
-            next unless filter === method || filter === "#{suite}##{method}"
-          end
 
           inst = suite.new method
           inst._assertions = 0

tool/lib/test/unit/testcase.rb

@@ -159,7 +159,6 @@ module Test
         start_time = Time.now
 
         result = ""
-        srand(runner.options[:seed])
 
         begin
           @passed = nil
@@ -267,46 +266,11 @@ module Test
       end
 
       def self.test_suites # :nodoc:
-        suites = @@test_suites.keys
+        @@test_suites.keys
-
-        case self.test_order
-        when :random
-          # shuffle test suites based on CRC32 of their names
-          salt = "\n" + rand(1 << 32).to_s
-          crc_tbl = (0..255).map do |i|
-            (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) }
-          end
-          suites = suites.sort_by do |suite|
-            crc32 = 0xffffffff
-            "#{suite.name}#{salt}".each_byte do |data|
-              crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8)
-            end
-            crc32 ^ 0xffffffff
-          end
-        when :nosort
-          suites
-        else
-          suites.sort_by { |ts| ts.name.to_s }
-        end
       end
 
       def self.test_methods # :nodoc:
-        methods = public_instance_methods(true).grep(/^test/).map { |m| m.to_s }
+        public_instance_methods(true).grep(/^test/)
-
-        case self.test_order
-        when :parallel
-          max = methods.size
-          ParallelEach.new methods.sort.sort_by { rand max }
-        when :random then
-          max = methods.size
-          methods.sort.sort_by { rand max }
-        when :alpha, :sorted then
-          methods.sort
-        when :nosort
-          methods
-        else
-          raise "Unknown test_order: #{self.test_order.inspect}"
-        end
       end
 
       ##

tool/test/testunit/test_minitest_unit.rb

@@ -238,7 +238,7 @@ class TestMiniTestRunner < MetaMetaMetaTestCase
     tc = Class.new(Test::Unit::TestCase)
 
     assert_equal 2, Test::Unit::TestCase.test_suites.size
-    assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites
+    assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites.sort_by {|ts| ts.name.to_s}
   end
 
   def assert_filtering name, expected, a = false
@@ -1331,34 +1331,17 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase
     end
   end
 
-  def test_test_methods_random
+  def test_test_methods
     @assertion_count = 0
 
     sample_test_case = Class.new Test::Unit::TestCase do
-      def self.test_order; :random; end
       def test_test1; assert "does not matter" end
       def test_test2; assert "does not matter" end
       def test_test3; assert "does not matter" end
-      @test_order = [1, 0, 2]
-      def self.rand(n) @test_order.shift; end
     end
 
-    expected = %w(test_test2 test_test1 test_test3)
+    expected = %i(test_test1 test_test2 test_test3)
-    assert_equal expected, sample_test_case.test_methods
+    assert_equal expected, sample_test_case.test_methods.sort
-  end
-
-  def test_test_methods_sorted
-    @assertion_count = 0
-
-    sample_test_case = Class.new Test::Unit::TestCase do
-      def self.test_order; :sorted end
-      def test_test3; assert "does not matter" end
-      def test_test2; assert "does not matter" end
-      def test_test1; assert "does not matter" end
-    end
-
-    expected = %w(test_test1 test_test2 test_test3)
-    assert_equal expected, sample_test_case.test_methods
   end
 
   def assert_triggered expected, klass = Test::Unit::AssertionFailedError

tool/test/testunit/test_sorting.rb

@@ -15,4 +15,61 @@ class TestTestUnitSorting < Test::Unit::TestCase
       f.read
     }
   end
+
+  Item = Struct.new(:name)
+  SEED = 0x50975eed
+
+  def make_test_list
+    (1..16).map {"test_%.3x" % rand(0x1000)}.freeze
+  end
+
+  def test_sort_alpha
+    sorter = Test::Unit::Order::Types[:alpha].new(SEED)
+    assert_kind_of(Test::Unit::Order::Types[:sorted], sorter)
+
+    list = make_test_list
+    sorted = list.sort
+    16.times do
+      assert_equal(sorted, sorter.sort_by_string(list))
+    end
+
+    list = list.map {|s| Item.new(s)}.freeze
+    sorted = list.sort_by(&:name)
+    16.times do
+      assert_equal(sorted, sorter.sort_by_name(list))
+    end
+  end
+
+  def test_sort_nosort
+    sorter = Test::Unit::Order::Types[:nosort].new(SEED)
+
+    list = make_test_list
+    16.times do
+      assert_equal(list, sorter.sort_by_string(list))
+    end
+
+    list = list.map {|s| Item.new(s)}.freeze
+    16.times do
+      assert_equal(list, sorter.sort_by_name(list))
+    end
+  end
+
+  def test_sort_random
+    type = Test::Unit::Order::Types[:random]
+    sorter = type.new(SEED)
+
+    list = make_test_list
+    sorted = type.new(SEED).sort_by_string(list).freeze
+    16.times do
+      assert_equal(sorted, sorter.sort_by_string(list))
+    end
+    assert_not_equal(sorted, type.new(SEED+1).sort_by_string(list))
+
+    list = list.map {|s| Item.new(s)}.freeze
+    sorted = sorted.map {|s| Item.new(s)}.freeze
+    16.times do
+      assert_equal(sorted, sorter.sort_by_name(list))
+    end
+    assert_not_equal(sorted, type.new(SEED+1).sort_by_name(list))
+  end
 end