From c4570acc86837fefa542a678dfdaba73cdd1fd03 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sat, 18 Sep 2021 16:05:26 +0900 Subject: [PATCH] Refactor ordering of tests * Split the sorting types into classes. * Apply the same sorting to method sorting under the parallel test. --- tool/lib/test/unit.rb | 113 +++++++++++++++++++---- tool/lib/test/unit/testcase.rb | 40 +------- tool/test/testunit/test_minitest_unit.rb | 25 +---- tool/test/testunit/test_sorting.rb | 57 ++++++++++++ 4 files changed, 157 insertions(+), 78 deletions(-) diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index bfa1964963..5fbb8db1a9 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -57,6 +57,84 @@ module Test class PendedError < AssertionFailedError; end + module Order + class NoSort + def initialize(seed) + end + + def sort_by_name(list) + list + end + + alias sort_by_string sort_by_name + + def group(list) + # JIT first + jit, others = list.partition {|e| /test_jit/ =~ e} + jit + others + end + end + + class Alpha < NoSort + def sort_by_name(list) + list.sort_by(&:name) + end + + def sort_by_string(list) + list.sort + end + + end + + # shuffle test suites based on CRC32 of their names + Shuffle = Struct.new(:seed, :salt) do + def initialize(seed) + self.class::CRC_TBL ||= (0..255).map {|i| + (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } + }.freeze + + salt = [seed].pack("V").unpack1("H*") + super(seed, "\n#{salt}".freeze).freeze + end + + def sort_by_name(list) + list.sort_by {|e| randomize_key(e.name)} + end + + def sort_by_string(list) + list.sort_by {|e| randomize_key(e)} + end + + def group(list) + list + end + + private + + def crc32(str, crc32 = 0xffffffff) + crc_tbl = self.class::CRC_TBL + str.each_byte do |data| + crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) + end + crc32 + end + + def randomize_key(name) + crc32(salt, crc32(name)) ^ 0xffffffff + end + end + + Types = { + random: Shuffle, + alpha: Alpha, + sorted: Alpha, + nosort: NoSort, + } + Types.default_proc = proc {|_, order| + raise "Unknown test_order: #{order.inspect}" + } + end + module RunCount # :nodoc: all @@run_count = 0 @@ -103,13 +181,13 @@ module Test order = options[:test_order] if seed = options[:seed] order ||= :random - srand(seed) - else - seed = options[:seed] = srand % 100_000 - srand(seed) + elsif order == :random + seed = options[:seed] = rand(0x10000) orig_args.unshift "--seed=#{seed}" end Test::Unit::TestCase.test_order = order if order + order = Test::Unit::TestCase.test_order + @order = Test::Unit::Order::Types[order].new(seed) @help = "\n" + orig_args.map { |s| " " + (s =~ /[\s|&<>$()]/ ? s.inspect : s) @@ -139,7 +217,8 @@ module Test (options[:filter] ||= []) << a end - opts.on '--test-order=random|alpha|sorted|nosort', [:random, :alpha, :sorted, :nosort] do |a| + orders = Test::Unit::Order::Types.keys + opts.on "--test-order=#{orders.join('|')}", orders do |a| options[:test_order] = a end end @@ -545,16 +624,7 @@ module Test # Require needed thing for parallel running require 'timeout' - @tasks = @files.dup # Array of filenames. - - case Test::Unit::TestCase.test_order - when :random - @tasks.shuffle! - else - # JIT first - ts = @tasks.group_by{|e| /test_jit/ =~ e ? 0 : 1} - @tasks = ts[0] + ts[1] if ts.size == 2 - end + @tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames. @need_quit = false @dead_workers = [] # Array of dead workers. @@ -1302,6 +1372,8 @@ module Test suites = Test::Unit::TestCase.send "#{type}_suites" return if suites.empty? + suites = @order.sort_by_name(suites) + puts puts "# Running #{type}s:" puts @@ -1356,6 +1428,12 @@ module Test filter = options[:filter] all_test_methods = suite.send "#{type}_methods" + if filter + all_test_methods.select! {|method| + filter === method || filter === "#{suite}##{method}" + } + end + all_test_methods = @order.sort_by_name(all_test_methods) leakchecker = LeakChecker.new if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] @@ -1363,10 +1441,7 @@ module Test trace = true end - assertions = all_test_methods.filter_map { |method| - if filter - next unless filter === method || filter === "#{suite}##{method}" - end + assertions = all_test_methods.map { |method| inst = suite.new method inst._assertions = 0 diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb index 241421d6d9..4cc1aae3e4 100644 --- a/tool/lib/test/unit/testcase.rb +++ b/tool/lib/test/unit/testcase.rb @@ -159,7 +159,6 @@ module Test start_time = Time.now result = "" - srand(runner.options[:seed]) begin @passed = nil @@ -267,46 +266,11 @@ module Test end def self.test_suites # :nodoc: - suites = @@test_suites.keys - - case self.test_order - when :random - # shuffle test suites based on CRC32 of their names - salt = "\n" + rand(1 << 32).to_s - crc_tbl = (0..255).map do |i| - (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } - end - suites = suites.sort_by do |suite| - crc32 = 0xffffffff - "#{suite.name}#{salt}".each_byte do |data| - crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) - end - crc32 ^ 0xffffffff - end - when :nosort - suites - else - suites.sort_by { |ts| ts.name.to_s } - end + @@test_suites.keys end def self.test_methods # :nodoc: - methods = public_instance_methods(true).grep(/^test/).map { |m| m.to_s } - - case self.test_order - when :parallel - max = methods.size - ParallelEach.new methods.sort.sort_by { rand max } - when :random then - max = methods.size - methods.sort.sort_by { rand max } - when :alpha, :sorted then - methods.sort - when :nosort - methods - else - raise "Unknown test_order: #{self.test_order.inspect}" - end + public_instance_methods(true).grep(/^test/) end ## diff --git a/tool/test/testunit/test_minitest_unit.rb b/tool/test/testunit/test_minitest_unit.rb index 68e88f574e..5941392fa0 100644 --- a/tool/test/testunit/test_minitest_unit.rb +++ b/tool/test/testunit/test_minitest_unit.rb @@ -238,7 +238,7 @@ class TestMiniTestRunner < MetaMetaMetaTestCase tc = Class.new(Test::Unit::TestCase) assert_equal 2, Test::Unit::TestCase.test_suites.size - assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites + assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites.sort_by {|ts| ts.name.to_s} end def assert_filtering name, expected, a = false @@ -1331,34 +1331,17 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase end end - def test_test_methods_random + def test_test_methods @assertion_count = 0 sample_test_case = Class.new Test::Unit::TestCase do - def self.test_order; :random; end def test_test1; assert "does not matter" end def test_test2; assert "does not matter" end def test_test3; assert "does not matter" end - @test_order = [1, 0, 2] - def self.rand(n) @test_order.shift; end end - expected = %w(test_test2 test_test1 test_test3) - assert_equal expected, sample_test_case.test_methods - end - - def test_test_methods_sorted - @assertion_count = 0 - - sample_test_case = Class.new Test::Unit::TestCase do - def self.test_order; :sorted end - def test_test3; assert "does not matter" end - def test_test2; assert "does not matter" end - def test_test1; assert "does not matter" end - end - - expected = %w(test_test1 test_test2 test_test3) - assert_equal expected, sample_test_case.test_methods + expected = %i(test_test1 test_test2 test_test3) + assert_equal expected, sample_test_case.test_methods.sort end def assert_triggered expected, klass = Test::Unit::AssertionFailedError diff --git a/tool/test/testunit/test_sorting.rb b/tool/test/testunit/test_sorting.rb index f9de3ec154..7678249ec2 100644 --- a/tool/test/testunit/test_sorting.rb +++ b/tool/test/testunit/test_sorting.rb @@ -15,4 +15,61 @@ class TestTestUnitSorting < Test::Unit::TestCase f.read } end + + Item = Struct.new(:name) + SEED = 0x50975eed + + def make_test_list + (1..16).map {"test_%.3x" % rand(0x1000)}.freeze + end + + def test_sort_alpha + sorter = Test::Unit::Order::Types[:alpha].new(SEED) + assert_kind_of(Test::Unit::Order::Types[:sorted], sorter) + + list = make_test_list + sorted = list.sort + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + sorted = list.sort_by(&:name) + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + end + + def test_sort_nosort + sorter = Test::Unit::Order::Types[:nosort].new(SEED) + + list = make_test_list + 16.times do + assert_equal(list, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(list, sorter.sort_by_name(list)) + end + end + + def test_sort_random + type = Test::Unit::Order::Types[:random] + sorter = type.new(SEED) + + list = make_test_list + sorted = type.new(SEED).sort_by_string(list).freeze + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_string(list)) + + list = list.map {|s| Item.new(s)}.freeze + sorted = sorted.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_name(list)) + end end