benchmark: introduce benchmark_driver.gem

Makefile.in: Clone benchmark-driver repository in benchmark/benchmark-driver
  `make update-benchmark-driver`, like simplecov.
win32/Makefile.sub: Roughly do the same thing.
.gitignore: Ignore the cloned repository.
common.mk: Trigger `make update-benchmark-driver` to run `make benchmark`
  and adjust arguments for benchmark_driver.gem.

benchmark/require.yml: renamed from benchmark/bm_require.rb, benchmark/prepare_require.rb
benchmark/require_thread.yml: renamed from benchmark/bm_require_thread.rb, benchmark/prepare_require_thread.rb
benchmark/so_count_words.yml: renamed from benchmark/bm_so_count_words.rb, benchmark/prepare_so_count_words.rb,
  benchmark/wc.input.base
benchmark/so_k_nucleotide.yml: renamed from benchmark/bm_so_k_nucleotide.rb, benchmark/prepare_so_k_nucleotide.rb,
  benchmark/make_fasta_output.rb
benchmark/so_reverse_complement.yml: renamed from benchmark/bm_so_reverse_complement.rb, benchmark/prepare_so_reverse_complement.rb,
  benchmark/make_fasta_output.rb

I'm sorry but I made some duplications between benchmark/require.yml and benchmark/require_thread.yml,
and between benchmark/so_k_nucleotide.yml and benchmark/so_reverse_complement.yml.

If you're not comfortable with it, please combine these YAMLs to share
the same prelude. One YAML file can have multiple benchmark definitions
sharing prelude.

benchmark/driver.rb: Replace its core feature with benchmark_driver.gem.
  Some old features are gone for now, but I'll add them again later.

[Misc #14902]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@63888 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

.gitignore

@@ -60,6 +60,7 @@ lcov*.info
 /archive
 /autom4te*.cache
 /automake
+/benchmark/benchmark-driver
 /beos
 /bmlog-*
 /breakpoints.gdb

Makefile.in

@@ -515,6 +515,11 @@ gcov:
 lcov:
 	$(Q) $(BASERUBY) $(srcdir)/tool/run-lcov.rb
 
+update-benchmark-driver:
+	$(Q) $(srcdir)/tool/git-refresh -C $(srcdir)/benchmark $(Q1:0=-q) \
+		--branch $(BENCHMARK_DRIVER_GIT_REF) \
+		$(BENCHMARK_DRIVER_GIT_URL) benchmark-driver $(GIT_OPTS)
+
 update-doclie:
 	$(Q) $(srcdir)/tool/git-refresh -C $(srcdir)/coverage $(Q1:0=-q) \
 		--branch $(DOCLIE_GIT_REF) \

benchmark/bm_require.rb

@@ -1,7 +0,0 @@
-$:.push File.join(File.dirname(__FILE__), "bm_require.data")
-
-1.upto(10000) do |i|
-  require "c#{i}"
-end
-
-$:.pop

benchmark/bm_require_thread.rb

@@ -1,15 +0,0 @@
-$:.push File.join(File.dirname(__FILE__), "bm_require.data")
-
-i=0
-t = Thread.new do
-  while true
-    i = i+1 # dummy loop
-  end
-end
-
-1.upto(100) do |i|
-  require "c#{i}"
-end
-
-$:.pop
-t.kill

benchmark/bm_so_count_words.rb

@@ -1,19 +0,0 @@
-#!/usr/bin/ruby
-# -*- mode: ruby -*-
-# $Id: wc-ruby.code,v 1.4 2004/11/13 07:43:32 bfulgham Exp $
-# http://www.bagley.org/~doug/shootout/
-# with help from Paul Brannan
-
-input = open(File.join(File.dirname($0), 'wc.input'), 'rb')
-
-nl = nw = nc = 0
-while true
-  tmp = input.read(4096) or break
-  data = tmp << (input.gets || "")
-  nc += data.length
-  nl += data.count("\n")
-  ((data.strip! || data).tr!("\n", " ") || data).squeeze!
-  nw += data.count(" ") + 1
-end
-# STDERR.puts "#{nl} #{nw} #{nc}"
-

benchmark/bm_so_k_nucleotide.rb

@@ -1,48 +0,0 @@
-# The Computer Language Shootout
-# http://shootout.alioth.debian.org
-#
-# contributed by jose fco. gonzalez
-# modified by Sokolov Yura
-
-seq = String.new
-
-def frecuency( seq,length )
-    n, table = seq.length - length + 1, Hash.new(0)
-    f, i = nil, nil
-    (0 ... length).each do |f|
-        (f ... n).step(length) do |i|
-            table[seq[i,length]] += 1
-        end
-    end
-    [n,table]
-
-end
-
-def sort_by_freq( seq,length )
-    n,table = frecuency( seq,length )
-    a, b, v = nil, nil, nil
-    table.sort{|a,b| b[1] <=> a[1]}.each do |v|
-        puts "%s %.3f" % [v[0].upcase,((v[1]*100).to_f/n)]
-    end
-    puts
-end
-
-def find_seq( seq,s )
-    n,table = frecuency( seq,s.length )
-    puts "#{table[s].to_s}\t#{s.upcase}"
-end
-
-input = open(File.join(File.dirname($0), 'fasta.output.100000'), 'rb')
-
-line = input.gets while line !~ /^>THREE/
-line = input.gets
-
-while (line !~ /^>/) & line do
-    seq << line.chomp
-    line = input.gets
-end
-
-[1,2].each {|i| sort_by_freq( seq,i ) }
-
-%w(ggt ggta ggtatt ggtattttaatt ggtattttaatttatagt).each{|s| find_seq( seq,s) }
-

benchmark/bm_so_reverse_complement.rb

@@ -1,30 +0,0 @@
-#!/usr/bin/ruby
-# The Great Computer Language Shootout
-# http://shootout.alioth.debian.org/
-#
-# Contributed by Peter Bjarke Olsen
-# Modified by Doug King
-
-seq=Array.new
-
-def revcomp(seq)
-  seq.reverse!.tr!('wsatugcyrkmbdhvnATUGCYRKMBDHVN','WSTAACGRYMKVHDBNTAACGRYMKVHDBN')
-  stringlen=seq.length
-  0.step(stringlen-1,60) {|x| print seq.slice(x,60) , "\n"}
-end
-
-input = open(File.join(File.dirname($0), 'fasta.output.2500000'), 'rb')
-
-while input.gets
-  if $_ =~ />/
-    if seq.length != 0
-      revcomp(seq.join)
-      seq=Array.new
-    end
-    puts $_
-  else
-    $_.sub(/\n/,'')
-    seq.push $_
-  end
-end
-revcomp(seq.join)

benchmark/driver.rb

@@ -10,256 +10,66 @@ rescue LoadError
   require 'optparse'
 end
 
-require 'benchmark'
+require 'shellwords'
-require 'pp'
 require 'tempfile'
 
 class BenchmarkDriver
-  def self.benchmark(opt)
+  # Run benchmark-driver prepared by `make update-benchmark-driver`
-    driver = self.new(opt[:execs], opt[:dir], opt)
+  def self.run(*args)
-    begin
+    benchmark_driver = File.expand_path('./benchmark-driver/exe/benchmark-driver', __dir__)
-      driver.run
+    command = [benchmark_driver, *args]
-    ensure
+    unless system(command.shelljoin)
-      driver.show_results
+      abort "Failed to execute: #{command.shelljoin}"
     end
   end
 
-  def self.load(input, type, opt)
+  def initialize(dir, opt = {})
-    case type
-    when 'yaml'
-      require 'yaml'
-      h = YAML.load(input)
-    when 'json'
-      require 'json'
-      h = JSON.load(input)
-    else
-      h = eval(input.read)
-    end
-    results = h[:results] || h["results"]
-    obj = allocate
-    obj.instance_variable_set("@execs", h[:executables] || h["executables"])
-    obj.instance_variable_set("@results", results)
-    obj.instance_variable_set("@opt", opt)
-    [1, 2].each do |i|
-      loop = results.assoc((n = "loop_whileloop#{i}").intern) || results.assoc(n)
-      obj.instance_variable_set("@loop_wl#{i}", loop ? loop[1].map {|t,*|t} : nil)
-    end
-    obj.instance_variable_set("@measure_target", opt[:measure_target] || opt["measure_target"])
-    obj
-  end
-
-  def output *args
-    puts(*args)
-    @output and @output.puts(*args)
-  end
-
-  def message *args
-    output(*args) if @verbose
-  end
-
-  def message_print *args
-    if @verbose
-      print(*args)
-      STDOUT.flush
-      @output and @output.print(*args)
-    end
-  end
-
-  def progress_message *args
-    unless STDOUT.tty?
-      STDERR.print(*args)
-      STDERR.flush
-    end
-  end
-
-  def initialize execs, dir, opt = {}
-    @execs = execs.map{|e|
-      e.strip!
-      next if e.empty?
-
-      if /(.+)::(.+)/ =~ e
-        # ex) ruby-a::/path/to/ruby-a
-        label = $1.strip
-        path = $2
-        version = `#{path} --version`.chomp
-      else
-        path = e
-        version = label = `#{path} --version`.chomp
-      end
-      [path, label, version]
-    }.compact
-
     @dir = dir
-    @repeat = opt[:repeat] || 1
-    @repeat = 1 if @repeat < 1
     @pattern = opt[:pattern] || nil
     @exclude = opt[:exclude] || nil
-    @verbose = opt[:quiet] ? false : (opt[:verbose] || false)
-    @output = opt[:output] ? open(opt[:output], 'w') : nil
-    @loop_wl1 = @loop_wl2 = nil
-    @ruby_arg = opt[:ruby_arg] || nil
-    @measure_target = opt[:measure_target]
-    @opt = opt
-
-    # [[name, [[r-1-1, r-1-2, ...], [r-2-1, r-2-2, ...]]], ...]
-    @results = []
-
-    if @verbose
-      @start_time = Time.now
-      message @start_time
-      @execs.each_with_index{|(path, label, version), i|
-        message "target #{i}: " + (label == version ? "#{label}" : "#{label} (#{version})") + " at \"#{path}\""
-      }
-      message "measure target: #{@measure_target}"
-    end
   end
 
-  def adjusted_results name, results
+  def with_yamls(&block)
-    s = nil
+    ios = files.map do |file|
-    results.each_with_index{|e, i|
+      Tempfile.open.tap do |io|
-      r = e.min
+        if file.end_with?('.yml')
-      case name
+          io.write(File.read(file))
-      when /^vm1_/
+        else
-        if @loop_wl1
+          io.write(build_yaml(file))
-          r -= @loop_wl1[i]
-          r = 0 if r < 0
-          s = '*'
-        end
-      when /^vm2_/
-        if @loop_wl2
-          r -= @loop_wl2[i]
-          r = 0 if r < 0
-          s = '*'
         end
+        io.close
       end
-      yield r
+    end
-    }
+    block.call(ios.map(&:path))
-    s
+  ensure
+    ios.each(&:close)
   end
 
-  def show_results
+  private
-    case @opt[:format]
+
-    when :tsv
+  def build_yaml(file)
-      strformat = "\t%1$s"
+    magic_comment = '# prelude' # bm_so_nsieve_bits hangs without magic comment
-      numformat = "\t%1$*2$.3f"
+    name = File.basename(file).sub(/\Abm_/, '').sub(/\.rb\z/, '')
-      minwidth = 0
+    script = File.read(file).sub(/^__END__\n(.+\n)*/m, '').sub(/\A(^#.+\n)+/m) do |comment|
-      name_width = 0
+      magic_comment = comment
-    when :markdown
+      ''
-      markdown = true
-      strformat = "|%1$-*2$s"
-      numformat = "|%1$*2$.3f"
-    when :plain
-      strformat = " %1$-*2$s"
-      numformat = " %1$*2$.3f"
     end
 
-    name_width ||= @results.map {|v, result|
+    <<-YAML
-      v.size + (case v; when /^vm1_/; @loop_wl1; when /^vm2_/; @loop_wl2; end ? 1 : 0)
+prelude: |
-    }.max
+#{magic_comment.gsub(/^/, '  ')}
-    minwidth ||= 7
+benchmark:
-    width = @execs.map{|(_, v)| [v.size, minwidth].max}
+  #{name}: |
-
+#{script.gsub(/^/, '    ')}
-    output
+loop_count: 1
-
+    YAML
-    if @verbose
-      message '-----------------------------------------------------------'
-      message 'raw data:'
-      message
-      message PP.pp(@results, "", 79)
-      message
-      message "Elapsed time: #{Time.now - @start_time} (sec)"
-    end
-
-    if rawdata_output = @opt[:rawdata_output]
-      h = {}
-      h[:cpuinfo] = File.read('/proc/cpuinfo') if File.exist?('/proc/cpuinfo')
-      h[:executables] = @execs
-      h[:results] = @results
-      if (type = File.extname(rawdata_output)).empty?
-        type = rawdata_output
-        rawdata_output = @output.path.sub(/\.[^.\/]+\z/, '') << '.' << rawdata_output
-      end
-      case type
-      when 'yaml'
-        require 'yaml'
-        h = YAML.dump(h)
-      when 'json'
-        require 'json'
-        h = JSON.pretty_generate(h)
-      else
-        require 'pp'
-        h = h.pretty_inspect
-      end
-      open(rawdata_output, 'w') {|f| f.puts h}
-    end
-
-    output '-----------------------------------------------------------'
-    output 'benchmark results:'
-
-    if @verbose and @repeat > 1
-      output "minimum results in each #{@repeat} measurements."
-    end
-
-    output({
-      real: "Execution time (sec)",
-      utime: "user CPU time",
-      stime: "system CPU time",
-      cutime: "user CPU time of children",
-      cstime: "system CPU time of children",
-      total: "all CPU time",
-      peak: "Memory usage (peak) (B)",
-      size: "Memory usage (last size) (B)",
-    }[@measure_target])
-    output if markdown
-    output ["name".ljust(name_width), @execs.map.with_index{|(_, v), i| sprintf(strformat, v, width[i])}].join("").rstrip
-    output ["-"*name_width, width.map{|n|":".rjust(n, "-")}].join("|") if markdown
-    @results.each{|v, result|
-      rets = []
-      s = adjusted_results(v, result){|r|
-        rets << sprintf(numformat, r, width[rets.size])
-      }
-      v += s if s
-      output [v.ljust(name_width), rets].join("")
-    }
-
-    if @execs.size > 1
-      output
-      output({
-        real: "Speedup ratio: compare with the result of `#{@execs[0][1]}' (greater is better)",
-        peak: "Memory consuming ratio (peak) with the result of `#{@execs[0][1]}' (greater is better)",
-        size: "Memory consuming ratio (size) with the result of `#{@execs[0][1]}' (greater is better)",
-      }[@measure_target])
-      output if markdown
-      output ["name".ljust(name_width), @execs[1..-1].map.with_index{|(_, v), i| sprintf(strformat, v, width[i])}].join("").rstrip
-      output ["-"*name_width, width[1..-1].map{|n|":".rjust(n, "-")}].join("|") if markdown
-      @results.each{|v, result|
-        rets = []
-        first_value = nil
-        s = adjusted_results(v, result){|r|
-          if first_value
-            if r == 0
-              rets << "Error"
-            else
-              rets << sprintf(numformat, first_value/Float(r), width[rets.size+1])
-            end
-          else
-            first_value = r
-          end
-        }
-        v += s if s
-        output [v.ljust(name_width), rets].join("")
-      }
-    end
-
-    if @opt[:output]
-      output
-      output "Log file: #{@opt[:output]}"
-    end
   end
 
   def files
     flag = {}
-    @files = Dir.glob(File.join(@dir, 'bm*.rb')).map{|file|
+    legacy_files = Dir.glob(File.join(@dir, 'bm*.rb'))
+    yaml_files = Dir.glob(File.join(@dir, '*.yml'))
+    files = (legacy_files + yaml_files).map{|file|
       next if @pattern && /#{@pattern}/ !~ File.basename(file)
       next if @exclude && /#{@exclude}/ =~ File.basename(file)
       case file
@@ -270,90 +80,13 @@ class BenchmarkDriver
     }.compact
 
     if flag['vm1'] && !flag['whileloop']
-      @files << File.join(@dir, 'bm_loop_whileloop.rb')
+      files << File.join(@dir, 'bm_loop_whileloop.rb')
     elsif flag['vm2'] && !flag['whileloop2']
-      @files << File.join(@dir, 'bm_loop_whileloop2.rb')
+      files << File.join(@dir, 'bm_loop_whileloop2.rb')
     end
 
-    @files.sort!
+    files.sort!
-    progress_message "total: #{@files.size * @repeat} trial(s) (#{@repeat} trial(s) for #{@files.size} benchmark(s))\n"
+    files
-    @files
-  end
-
-  def run
-    files.each_with_index{|file, i|
-      @i = i
-      r = measure_file(file)
-
-      if /bm_loop_whileloop.rb/ =~ file
-        @loop_wl1 = r[1].map{|e| e.min}
-      elsif /bm_loop_whileloop2.rb/ =~ file
-        @loop_wl2 = r[1].map{|e| e.min}
-      end
-    }
-  end
-
-  def measure_file file
-    name = File.basename(file, '.rb').sub(/^bm_/, '')
-    prepare_file = File.join(File.dirname(file), "prepare_#{name}.rb")
-    load prepare_file if FileTest.exist?(prepare_file)
-
-    if @verbose
-      output
-      output '-----------------------------------------------------------'
-      output name
-      output
-      output File.read(file)
-      output
-    end
-
-    result = [name]
-    result << @execs.map{|(e, v)|
-      (0...@repeat).map{
-        message_print "#{v}\t"
-        progress_message '.'
-
-        m = measure(e, file)
-        message "#{m}"
-        m
-      }
-    }
-    @results << result
-    result
-  end
-
-  unless defined?(File::NULL)
-    if File.exist?('/dev/null')
-      File::NULL = '/dev/null'
-    end
-  end
-
-  def measure executable, file
-    case @measure_target
-    when :real, :utime, :stime, :cutime, :cstime, :total
-      cmd = "#{executable} #{@ruby_arg} #{file}"
-      m = Benchmark.measure{
-        system(cmd, out: File::NULL)
-      }
-      result = m.__send__(@measure_target)
-    when :peak, :size
-      tmp = Tempfile.new("benchmark-memory-wrapper-data")
-      wrapper = "#{File.join(__dir__, 'memory_wrapper.rb')} #{tmp.path} #{@measure_target}"
-      cmd = "#{executable} #{@ruby_arg} #{wrapper} #{file}"
-      system(cmd, out: File::NULL)
-      result = tmp.read.to_i
-      tmp.close
-    else
-      raise "unknown measure target"
-    end
-
-    if $? != 0
-      raise $?.inspect if $? && $?.signaled?
-      output "\`#{cmd}\' exited with abnormal status (#{$?})"
-      0
-    else
-      result
-    end
   end
 end
 
@@ -362,15 +95,7 @@ if __FILE__ == $0
     :execs => [],
     :dir => File.dirname(__FILE__),
     :repeat => 1,
-    :measure_target => :real,
+    :verbose => 1,
-    :output => nil,
-    :raw_output => nil,
-    :format => :tsv,
-  }
-  formats = {
-    :tsv => ".tsv",
-    :markdown => ".md",
-    :plain => ".txt",
   }
 
   parser = OptionParser.new{|o|
@@ -397,44 +122,18 @@ if __FILE__ == $0
     o.on('-r', '--repeat-count [NUM]', "Repeat count"){|n|
       opt[:repeat] = n.to_i
     }
-    o.on('-o', '--output-file [FILE]', "Output file"){|f|
-      opt[:output] = f
-    }
-    o.on('--ruby-arg [ARG]', "Optional argument for ruby"){|a|
-      opt[:ruby_arg] = a
-    }
-    o.on('--measure-target [TARGET]',
-         'real (execution time), peak, size (memory), total'){|mt|
-      opt[:measure_target] = mt.to_sym
-    }
-    o.on('--rawdata-output [FILE]', 'output rawdata'){|r|
-      opt[:rawdata_output] = r
-    }
-    o.on('--load-rawdata=FILE', 'input rawdata'){|r|
-      opt[:rawdata_input] = r
-    }
-    o.on('-f', "--format=FORMAT", "output format (#{formats.keys.join(",")})", formats.keys){|r|
-      opt[:format] = r
-    }
     o.on('-v', '--verbose'){|v|
-      opt[:verbose] = v
+      opt[:verbose] = 2
     }
     o.on('-q', '--quiet', "Run without notify information except result table."){|q|
-      opt[:quiet] = q
+      opt[:verbose] = 0
-      opt[:verbose] = false
     }
   }
 
   parser.parse!(ARGV)
 
-  if input = opt[:rawdata_input]
+  execs = opt[:execs].map { |exec| ['--executables', exec.shellsplit.join(',')] }.flatten
-    b = open(input) {|f|
+  BenchmarkDriver.new(opt[:dir], opt).with_yamls do |yamls|
-      BenchmarkDriver.load(f, File.extname(input)[1..-1], opt)
+    BenchmarkDriver.run(*yamls, *execs, "--verbose=#{opt[:verbose]}", "--repeat-count=#{opt[:repeat]}")
-    }
-    b.show_results
-  else
-    opt[:output] ||= "bmlog-#{Time.now.strftime('%Y%m%d-%H%M%S')}.#{$$}#{formats[opt[:format]]}"
-    BenchmarkDriver.benchmark(opt)
   end
 end
-

benchmark/make_fasta_output.rb

@@ -1,19 +0,0 @@
-# prepare 'fasta.output'
-
-def prepare_fasta_output n
-  filebase = File.join(File.dirname($0), 'fasta.output')
-  script = File.join(File.dirname($0), 'bm_so_fasta.rb')
-  file = "#{filebase}.#{n}"
-
-  unless FileTest.exist?(file)
-    STDERR.puts "preparing #{file}"
-
-    open(file, 'w'){|f|
-      ARGV[0] = n
-      $stdout = f
-      load script
-      $stdout = STDOUT
-    }
-  end
-end
-

benchmark/prepare_require.rb

@@ -1,25 +0,0 @@
-require "fileutils"
-
-def prepare
-  num_files = 10000
-
-  basename = File.dirname($0)
-  data_dir = File.join(basename, "bm_require.data")
-
-  # skip if all of files exists
-  if File.exist?(File.join(data_dir, "c#{num_files}.rb"))
-    return
-  end
-
-  FileUtils.mkdir_p(data_dir)
-
-  1.upto(num_files) do |i|
-    f = File.open("#{data_dir}/c#{i}.rb", "w")
-    f.puts <<-END
-      class C#{i}
-      end
-    END
-  end
-end
-
-prepare

benchmark/prepare_require_thread.rb

@@ -1,2 +0,0 @@
-load File.join(File.dirname(__FILE__), "prepare_require.rb")
-

benchmark/prepare_so_count_words.rb

@@ -1,15 +0,0 @@
-# prepare 'wc.input'
-
-def prepare_wc_input
-  wcinput = File.join(File.dirname($0), 'wc.input')
-  wcbase  = File.join(File.dirname($0), 'wc.input.base')
-  unless FileTest.exist?(wcinput)
-    data = File.read(wcbase)
-    13.times{
-      data << data
-    }
-    open(wcinput, 'w'){|f| f.write data}
-  end
-end
-
-prepare_wc_input

benchmark/prepare_so_k_nucleotide.rb

@@ -1,2 +0,0 @@
-require_relative 'make_fasta_output'
-prepare_fasta_output(100_000)

benchmark/prepare_so_reverse_complement.rb

@@ -1,2 +0,0 @@
-require_relative 'make_fasta_output'
-prepare_fasta_output(2_500_000)

benchmark/require.yml

@@ -0,0 +1,36 @@
+prelude: |
+  require "fileutils"
+
+  def prepare
+    num_files = 10000
+
+    basename = File.dirname($0)
+    data_dir = File.join(basename, "bm_require.data")
+
+    # skip if all of files exists
+    if File.exist?(File.join(data_dir, "c#{num_files}.rb"))
+      return
+    end
+
+    FileUtils.mkdir_p(data_dir)
+
+    1.upto(num_files) do |i|
+      f = File.open("#{data_dir}/c#{i}.rb", "w")
+      f.puts <<-END
+        class C#{i}
+        end
+      END
+    end
+  end
+
+  prepare
+benchmark:
+  require: |
+    $:.push File.join(File.dirname(__FILE__), "bm_require.data")
+
+    1.upto(10000) do |i|
+      require "c#{i}"
+    end
+
+    $:.pop
+loop_count: 1

benchmark/require_thread.yml

@@ -0,0 +1,44 @@
+prelude: |
+  require "fileutils"
+
+  def prepare
+    num_files = 10000
+
+    basename = File.dirname($0)
+    data_dir = File.join(basename, "bm_require.data")
+
+    # skip if all of files exists
+    if File.exist?(File.join(data_dir, "c#{num_files}.rb"))
+      return
+    end
+
+    FileUtils.mkdir_p(data_dir)
+
+    1.upto(num_files) do |i|
+      f = File.open("#{data_dir}/c#{i}.rb", "w")
+      f.puts <<-END
+        class C#{i}
+        end
+      END
+    end
+  end
+
+  prepare
+benchmark:
+  require_thread: |
+    $:.push File.join(File.dirname(__FILE__), "bm_require.data")
+
+    i=0
+    t = Thread.new do
+      while true
+        i = i+1 # dummy loop
+      end
+    end
+
+    1.upto(100) do |i|
+      require "c#{i}"
+    end
+
+    $:.pop
+    t.kill
+loop_count: 1

benchmark/so_k_nucleotide.yml

@@ -0,0 +1,155 @@
+prelude: |
+  bm_so_fasta = <<'EOS'
+  # The Computer Language Shootout
+  # http://shootout.alioth.debian.org/
+  # Contributed by Sokolov Yura
+
+  $last = 42.0
+  def gen_random(max, im=139968, ia=3877, ic=29573)
+      (max * ($last = ($last * ia + ic) % im)) / im
+  end
+
+  alu =
+     "GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGG"+
+     "GAGGCCGAGGCGGGCGGATCACCTGAGGTCAGGAGTTCGAGA"+
+     "CCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACTAAAAAT"+
+     "ACAAAAATTAGCCGGGCGTGGTGGCGCGCGCCTGTAATCCCA"+
+     "GCTACTCGGGAGGCTGAGGCAGGAGAATCGCTTGAACCCGGG"+
+     "AGGCGGAGGTTGCAGTGAGCCGAGATCGCGCCACTGCACTCC"+
+     "AGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA"
+
+  iub = [
+      ["a", 0.27],
+      ["c", 0.12],
+      ["g", 0.12],
+      ["t", 0.27],
+
+      ["B", 0.02],
+      ["D", 0.02],
+      ["H", 0.02],
+      ["K", 0.02],
+      ["M", 0.02],
+      ["N", 0.02],
+      ["R", 0.02],
+      ["S", 0.02],
+      ["V", 0.02],
+      ["W", 0.02],
+      ["Y", 0.02],
+  ]
+  homosapiens = [
+      ["a", 0.3029549426680],
+      ["c", 0.1979883004921],
+      ["g", 0.1975473066391],
+      ["t", 0.3015094502008],
+  ]
+
+  def make_repeat_fasta(id, desc, src, n)
+      puts ">#{id} #{desc}"
+      v = nil
+      width = 60
+      l = src.length
+      s = src * ((n / l) + 1)
+      s.slice!(n, l)
+      puts(s.scan(/.{1,#{width}}/).join("\n"))
+  end
+
+  def make_random_fasta(id, desc, table, n)
+      puts ">#{id} #{desc}"
+      rand, v = nil,nil
+      width = 60
+      chunk = 1 * width
+      prob = 0.0
+      table.each{|v| v[1]= (prob += v[1])}
+      for i in 1..(n/width)
+          puts((1..width).collect{
+              rand = gen_random(1.0)
+              table.find{|v| v[1]>rand}[0]
+          }.join)
+      end
+      if n%width != 0
+          puts((1..(n%width)).collect{
+              rand = gen_random(1.0)
+              table.find{|v| v[1]>rand}[0]
+          }.join)
+      end
+  end
+
+
+  n = (ARGV[0] or 250_000).to_i
+
+  make_repeat_fasta('ONE', 'Homo sapiens alu', alu, n*2)
+  make_random_fasta('TWO', 'IUB ambiguity codes', iub, n*3)
+  make_random_fasta('THREE', 'Homo sapiens frequency', homosapiens, n*5)
+  EOS
+benchmark:
+  - name: so_k_nucleotide
+    prelude: |
+      script = File.join(File.dirname($0), 'bm_so_fasta.rb')
+      File.write(script, bm_so_fasta)
+
+      def prepare_fasta_output n
+        filebase = File.join(File.dirname($0), 'fasta.output')
+        script = File.join(File.dirname($0), 'bm_so_fasta.rb')
+        file = "#{filebase}.#{n}"
+
+        unless FileTest.exist?(file)
+          STDERR.puts "preparing #{file}"
+
+          open(file, 'w'){|f|
+            ARGV[0] = n
+            $stdout = f
+            load script
+            $stdout = STDOUT
+          }
+        end
+      end
+      prepare_fasta_output(100_000)
+    script: |
+      # The Computer Language Shootout
+      # http://shootout.alioth.debian.org
+      #
+      # contributed by jose fco. gonzalez
+      # modified by Sokolov Yura
+
+      seq = String.new
+
+      def frecuency( seq,length )
+          n, table = seq.length - length + 1, Hash.new(0)
+          f, i = nil, nil
+          (0 ... length).each do |f|
+              (f ... n).step(length) do |i|
+                  table[seq[i,length]] += 1
+              end
+          end
+          [n,table]
+
+      end
+
+      def sort_by_freq( seq,length )
+          n,table = frecuency( seq,length )
+          a, b, v = nil, nil, nil
+          table.sort{|a,b| b[1] <=> a[1]}.each do |v|
+              puts "%s %.3f" % [v[0].upcase,((v[1]*100).to_f/n)]
+          end
+          puts
+      end
+
+      def find_seq( seq,s )
+          n,table = frecuency( seq,s.length )
+          puts "#{table[s].to_s}\t#{s.upcase}"
+      end
+
+      input = open(File.join(File.dirname($0), 'fasta.output.100000'), 'rb')
+
+      line = input.gets while line !~ /^>THREE/
+      line = input.gets
+
+      while (line !~ /^>/) & line do
+          seq << line.chomp
+          line = input.gets
+      end
+
+      [1,2].each {|i| sort_by_freq( seq,i ) }
+
+      %w(ggt ggta ggtatt ggtattttaatt ggtattttaatttatagt).each{|s| find_seq( seq,s) }
+    loop_count: 1

benchmark/so_reverse_complement.yml

@@ -0,0 +1,137 @@
+prelude: |
+  bm_so_fasta = <<'EOS'
+  # The Computer Language Shootout
+  # http://shootout.alioth.debian.org/
+  # Contributed by Sokolov Yura
+
+  $last = 42.0
+  def gen_random(max, im=139968, ia=3877, ic=29573)
+      (max * ($last = ($last * ia + ic) % im)) / im
+  end
+
+  alu =
+     "GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGG"+
+     "GAGGCCGAGGCGGGCGGATCACCTGAGGTCAGGAGTTCGAGA"+
+     "CCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACTAAAAAT"+
+     "ACAAAAATTAGCCGGGCGTGGTGGCGCGCGCCTGTAATCCCA"+
+     "GCTACTCGGGAGGCTGAGGCAGGAGAATCGCTTGAACCCGGG"+
+     "AGGCGGAGGTTGCAGTGAGCCGAGATCGCGCCACTGCACTCC"+
+     "AGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA"
+
+  iub = [
+      ["a", 0.27],
+      ["c", 0.12],
+      ["g", 0.12],
+      ["t", 0.27],
+
+      ["B", 0.02],
+      ["D", 0.02],
+      ["H", 0.02],
+      ["K", 0.02],
+      ["M", 0.02],
+      ["N", 0.02],
+      ["R", 0.02],
+      ["S", 0.02],
+      ["V", 0.02],
+      ["W", 0.02],
+      ["Y", 0.02],
+  ]
+  homosapiens = [
+      ["a", 0.3029549426680],
+      ["c", 0.1979883004921],
+      ["g", 0.1975473066391],
+      ["t", 0.3015094502008],
+  ]
+
+  def make_repeat_fasta(id, desc, src, n)
+      puts ">#{id} #{desc}"
+      v = nil
+      width = 60
+      l = src.length
+      s = src * ((n / l) + 1)
+      s.slice!(n, l)
+      puts(s.scan(/.{1,#{width}}/).join("\n"))
+  end
+
+  def make_random_fasta(id, desc, table, n)
+      puts ">#{id} #{desc}"
+      rand, v = nil,nil
+      width = 60
+      chunk = 1 * width
+      prob = 0.0
+      table.each{|v| v[1]= (prob += v[1])}
+      for i in 1..(n/width)
+          puts((1..width).collect{
+              rand = gen_random(1.0)
+              table.find{|v| v[1]>rand}[0]
+          }.join)
+      end
+      if n%width != 0
+          puts((1..(n%width)).collect{
+              rand = gen_random(1.0)
+              table.find{|v| v[1]>rand}[0]
+          }.join)
+      end
+  end
+
+
+  n = (ARGV[0] or 250_000).to_i
+
+  make_repeat_fasta('ONE', 'Homo sapiens alu', alu, n*2)
+  make_random_fasta('TWO', 'IUB ambiguity codes', iub, n*3)
+  make_random_fasta('THREE', 'Homo sapiens frequency', homosapiens, n*5)
+  EOS
+benchmark:
+  - name: so_reverse_complement
+    prelude: |
+      script = File.join(File.dirname($0), 'bm_so_fasta.rb')
+      File.write(script, bm_so_fasta)
+
+      def prepare_fasta_output n
+        filebase = File.join(File.dirname($0), 'fasta.output')
+        script = File.join(File.dirname($0), 'bm_so_fasta.rb')
+        file = "#{filebase}.#{n}"
+
+        unless FileTest.exist?(file)
+          STDERR.puts "preparing #{file}"
+
+          open(file, 'w'){|f|
+            ARGV[0] = n
+            $stdout = f
+            load script
+            $stdout = STDOUT
+          }
+        end
+      end
+      prepare_fasta_output(2_500_000)
+    script: |
+      # The Great Computer Language Shootout
+      # http://shootout.alioth.debian.org/
+      #
+      # Contributed by Peter Bjarke Olsen
+      # Modified by Doug King
+
+      seq=Array.new
+
+      def revcomp(seq)
+        seq.reverse!.tr!('wsatugcyrkmbdhvnATUGCYRKMBDHVN','WSTAACGRYMKVHDBNTAACGRYMKVHDBN')
+        stringlen=seq.length
+        0.step(stringlen-1,60) {|x| print seq.slice(x,60) , "\n"}
+      end
+
+      input = open(File.join(File.dirname($0), 'fasta.output.2500000'), 'rb')
+
+      while input.gets
+        if $_ =~ />/
+          if seq.length != 0
+            revcomp(seq.join)
+            seq=Array.new
+          end
+          puts $_
+        else
+          $_.sub(/\n/,'')
+          seq.push $_
+        end
+      end
+      revcomp(seq.join)
+    loop_count: 1

common.mk

@@ -41,6 +41,8 @@ GEM_HOME =
 GEM_PATH =
 GEM_VENDOR =
 
+BENCHMARK_DRIVER_GIT_URL = https://github.com/benchmark-driver/benchmark-driver
+BENCHMARK_DRIVER_GIT_REF = v0.13.2
 SIMPLECOV_GIT_URL = git://github.com/colszowka/simplecov.git
 SIMPLECOV_GIT_REF = v0.15.0
 SIMPLECOV_HTML_GIT_URL = git://github.com/colszowka/simplecov-html.git
@@ -1117,14 +1119,16 @@ OPTS =
 # for example,
 #  $ make benchmark COMPARE_RUBY="ruby-trunk" OPTS="-e ruby-2.2.2"
 # This command compares trunk and built-ruby and 2.2.2
-benchmark: miniruby$(EXEEXT) PHONY
+benchmark: miniruby$(EXEEXT) update-benchmark-driver PHONY
-	$(BASERUBY) $(srcdir)/benchmark/driver.rb -v \
+	$(BASERUBY) $(srcdir)/benchmark/driver.rb \
-	            --executables="$(COMPARE_RUBY) -I$(srcdir)/lib -I. -I$(EXTOUT)/common --disable-gem; built-ruby::$(MINIRUBY) -r$(srcdir)/prelude --disable-gem" \
+	            --executables="compare-ruby::$(COMPARE_RUBY) -I$(EXTOUT)/common --disable-gem" \
+	            --executables="built-ruby::$(MINIRUBY) -r$(srcdir)/prelude --disable-gem" \
 	            --pattern='bm_' --directory=$(srcdir)/benchmark $(OPTS)
 
-benchmark-each: miniruby$(EXEEXT) PHONY
+benchmark-each: miniruby$(EXEEXT) update-benchmark-driver PHONY
-	$(BASERUBY) $(srcdir)/benchmark/driver.rb -v \
+	$(BASERUBY) $(srcdir)/benchmark/driver.rb \
-	            --executables="$(COMPARE_RUBY) -I$(srcdir)/lib -I. -I$(EXTOUT)/common --disable-gem; built-ruby::$(MINIRUBY) -r$(srcdir)/prelude --disable-gem" \
+	            --executables="compare-ruby::$(COMPARE_RUBY) -I$(EXTOUT)/common --disable-gem" \
+	            --executables="built-ruby::$(MINIRUBY) -r$(srcdir)/prelude --disable-gem" \
 	            --pattern=$(ITEM) --directory=$(srcdir)/benchmark $(OPTS)
 
 run.gdb:

win32/Makefile.sub

@@ -1182,6 +1182,10 @@ test-bundled-gems-run:
 	  exit /b %STATUS% \
 	)
 
+update-benchmark-driver:
+	$(GIT) clone https://github.com/benchmark-driver/benchmark-driver $(srcdir)/benchmark/benchmark-driver || \
+	  $(GIT) -C $(srcdir)/benchmark/benchmark-driver pull origin master
+
 $(ruby_pc): $(RBCONFIG)
 	@$(BOOTSTRAPRUBY) $(srcdir)/tool/expand-config.rb \
 		-output=$@ -mode=$(INSTALL_DATA_MODE) -config=rbconfig.rb \