benchmark: resurrect peak / size metrics

by adding runner plugins for them.

benchmark/lib/benchmark_driver/runner/peak.rb: added peak runner plugin
benchmark/lib/benchmark_driver/runner/size.rb: added size runner plugin
common.mk: allow using them

benchmark/memory_wrapper.rb: deleted in favor of those runner plugins

benchmark/README.md: document them

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@63924 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

benchmark/README.md

@@ -43,4 +43,8 @@ make benchmark ARGS=../benchmark/erb_render.yml
 
 # You can specify any option via $OPTS
 make benchmark OPTS="--help"
+
+# With `make benchmark`, some special runner plugins are available:
+#   -r peak, -r size
+make benchmark ITEM=vm2_bigarray OPTS="-r peak"
 ```

benchmark/lib/benchmark_driver/runner/peak.rb

@@ -0,0 +1,146 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'benchmark_driver/default_job'
+require 'benchmark_driver/default_job_parser'
+require 'tempfile'
+
+class BenchmarkDriver::Runner::Peak
+  METRIC = BenchmarkDriver::Metric.new(
+    name: 'Peak memory usage', unit: 'bytes', larger_better: false, worse_word: 'larger',
+  )
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output] output
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
+    @config = config
+    @output = output
+    @contexts = contexts
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Runner::Peak::Job>] jobs
+  def run(jobs)
+    if jobs.any? { |job| job.loop_count.nil? }
+      jobs = jobs.map do |job|
+        job.loop_count ? job : Job.new(job.to_h.merge(loop_count: 1))
+      end
+    end
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          job.runnable_contexts(@contexts).each do |context|
+            value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
+              run_benchmark(job, context: context)
+            end
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
+              @output.report(values: { METRIC => value }, loop_count: job.loop_count)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Context] context
+  # @return [BenchmarkDriver::Metrics]
+  def run_benchmark(job, context:)
+    benchmark = BenchmarkScript.new(
+      preludes:   [context.prelude, job.prelude],
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+    )
+
+    memory_status = File.expand_path('../../../../test/lib/memory_status', __dir__)
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      with_script(benchmark.render) do |path|
+        output = IO.popen([*context.executable.command, path, f.path, target, memory_status], &:read)
+        if $?.success?
+          Integer(f.read)
+        else
+          $stdout.print(output)
+          BenchmarkDriver::Result::ERROR
+        end
+      end
+    end
+  end
+
+  # Overridden by BenchmarkDriver::Runner::Size
+  def target
+    'peak'
+  end
+
+  def with_script(script)
+    if @config.verbose >= 2
+      sep = '-' * 30
+      $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
+    end
+
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      f.puts script
+      f.close
+      return yield(f.path)
+    end
+  end
+
+  # @param [String] prelude
+  # @param [String] script
+  # @param [String] teardown
+  # @param [Integer] loop_count
+  BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do
+    def render
+      prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n")
+      <<-RUBY
+#{prelude}
+#{while_loop(script, loop_count)}
+#{teardown}
+
+result_file, target, memory_status = ARGV
+require_relative memory_status
+
+ms = Memory::Status.new
+case target.to_sym
+when :peak
+  key = ms.respond_to?(:hwm) ? :hwm : :peak
+when :size
+  key = ms.respond_to?(:rss) ? :rss : :size
+else
+  raise('unexpected target: ' + target)
+end
+
+File.write(result_file, ms[key])
+      RUBY
+    end
+
+    private
+
+    def while_loop(content, times)
+      if !times.is_a?(Integer) || times <= 0
+        raise ArgumentError.new("Unexpected times: #{times.inspect}")
+      end
+
+      if times > 1
+        <<-RUBY
+__bmdv_i = 0
+while __bmdv_i < #{times}
+  #{content}
+  __bmdv_i += 1
+end
+        RUBY
+      else
+        content
+      end
+    end
+  end
+  private_constant :BenchmarkScript
+end

benchmark/lib/benchmark_driver/runner/size.rb

@@ -0,0 +1,20 @@
+require 'benchmark_driver/runner/peak'
+
+# Actually the same as BenchmarkDriver::Runner::Memory
+class BenchmarkDriver::Runner::Size < BenchmarkDriver::Runner::Peak
+  METRIC = BenchmarkDriver::Metric.new(
+    name: 'Max resident set size', unit: 'bytes', larger_better: false, worse_word: 'larger',
+  )
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  private
+
+  # Overriding BenchmarkDriver::Runner::Peak#target
+  def target
+    'size'
+  end
+end

benchmark/memory_wrapper.rb

@@ -1,16 +0,0 @@
-
-write_file, target, script_file = ARGV
-
-load(script_file)
-require_relative '../test/lib/memory_status'
-open(write_file, 'wb'){|f|
-  ms = Memory::Status.new
-  case target.to_sym
-  when :peak
-    key = ms.respond_to?(:hwm) ? :hwm : :peak
-  when :size
-    key = ms.respond_to?(:rss) ? :rss : :size
-  end
-
-  f.puts ms[key]
-}

common.mk

@@ -42,7 +42,7 @@ GEM_PATH =
 GEM_VENDOR =
 
 BENCHMARK_DRIVER_GIT_URL = https://github.com/benchmark-driver/benchmark-driver
-BENCHMARK_DRIVER_GIT_REF = v0.14.0
+BENCHMARK_DRIVER_GIT_REF = v0.14.3
 SIMPLECOV_GIT_URL = git://github.com/colszowka/simplecov.git
 SIMPLECOV_GIT_REF = v0.15.0
 SIMPLECOV_HTML_GIT_URL = git://github.com/colszowka/simplecov-html.git
@@ -1122,7 +1122,7 @@ OPTS =
 #  $ make benchmark COMPARE_RUBY="ruby-trunk" OPTS="-e ruby-2.2.2"
 # This command compares trunk and built-ruby and 2.2.2
 benchmark: miniruby$(EXEEXT) update-benchmark-driver PHONY
-	$(BASERUBY) -rrubygems $(srcdir)/benchmark/benchmark-driver/exe/benchmark-driver \
+	$(BASERUBY) -rrubygems -I$(srcdir)/benchmark/lib $(srcdir)/benchmark/benchmark-driver/exe/benchmark-driver \
 	            --executables="compare-ruby::$(COMPARE_RUBY) -I$(EXTOUT)/common --disable-gem" \
 	            --executables="built-ruby::$(MINIRUBY) -r$(srcdir)/prelude --disable-gem" \
 	            $(ARGS) $(OPTS)