From 19893a1c10e4e6dfbdb56ad78de1599b6c8f6981 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 2 Oct 2015 17:00:23 +0200 Subject: [PATCH] Basic setup for an RSpec based benchmark suite This benchmark suite uses benchmark-ips (https://github.com/evanphx/benchmark-ips) behind the scenes. Specs can be turned into benchmark specs by setting "benchmark" to "true" in the top-level describe block like so: describe SomeClass, benchmark: true do end Writing benchmarks can be done using custom RSpec matchers, for example: describe MaruTheCat, benchmark: true do describe '#jump_in_box' do it 'should run 1000 iterations per second' do maru = described_class.new expect { maru.jump_in_box }.to iterate_per_second(1000) end end end By default the "iterate_per_second" expectation requires a standard deviation under 30% (this is just an arbitrary default for now). You can change this by chaining "with_maximum_stddev" on the expectation: expect { maru.jump_in_box }.to iterate_per_second(1000) .with_maximum_stddev(10) This will change the expectation to require a maximum deviation of 10%. Alternatively you can use the it block style to write specs: describe MaruTheCat, benchmark: true do describe '#jump_in_box' do subject { -> { described_class.new } } it { is_expected.to iterate_per_second(1000) } end end Because "iterate_per_second" operates on a block, opposed to a static value, the "subject" method must return a Proc. This looks a bit goofy but I have been unable to find a nice way around this. --- .gitlab-ci.yml | 7 ++++ lib/tasks/spec.rake | 11 +++++- spec/benchmarks/models/user_spec.rb | 40 ++++++++++++++++++++ spec/spec_helper.rb | 3 +- spec/support/matchers/benchmark_matchers.rb | 42 +++++++++++++++++++++ 5 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 spec/benchmarks/models/user_spec.rb create mode 100644 spec/support/matchers/benchmark_matchers.rb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ddf4e31204a..a9aaf82ec1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,13 @@ spec:api: - ruby - mysql +spec:benchmark: + script: + - RAILS_ENV=test bundle exec rake spec:benchmark + tags: + - ruby + - mysql + spec:other: script: - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 831746815d7..3ae5c250694 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -19,11 +19,20 @@ namespace :spec do run_commands(cmds) end + desc 'GitLab | Rspec | Run benchmark specs' + task :benchmark do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @benchmark) + ] + run_commands(cmds) + end + desc 'GitLab | Rspec | Run other specs' task :other do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature) + %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark) ] run_commands(cmds) end diff --git a/spec/benchmarks/models/user_spec.rb b/spec/benchmarks/models/user_spec.rb new file mode 100644 index 00000000000..a08c84ffce4 --- /dev/null +++ b/spec/benchmarks/models/user_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe User, benchmark: true do + describe '.by_login' do + before do + %w{Alice Bob Eve}.each do |name| + create(:user, + email: "#{name}@gitlab.com", + username: name, + name: name) + end + end + + let(:iterations) { 1000 } + + describe 'using a capitalized username' do + subject { -> { User.by_login('Alice') } } + + it { is_expected.to iterate_per_second(iterations) } + end + + describe 'using a lowercase username' do + subject { -> { User.by_login('alice') } } + + it { is_expected.to iterate_per_second(iterations) } + end + + describe 'using a capitalized Email address' do + subject { -> { User.by_login('Alice@gitlab.com') } } + + it { is_expected.to iterate_per_second(iterations) } + end + + describe 'using a lowercase Email address' do + subject { -> { User.by_login('alice@gitlab.com') } } + + it { is_expected.to iterate_per_second(iterations) } + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index dfe855926c6..05bb32baa90 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -14,6 +14,7 @@ require File.expand_path("../../config/environment", __FILE__) require 'rspec/rails' require 'shoulda/matchers' require 'sidekiq/testing/inline' +require 'benchmark/ips' # Requires supporting ruby files with custom matchers and macros, etc, # in spec/support/ and its subdirectories. @@ -32,7 +33,7 @@ RSpec.configure do |config| config.include TestEnv config.include StubGitlabCalls config.include StubGitlabData - + config.include BenchmarkMatchers, benchmark: true config.infer_spec_type_from_file_location! config.raise_errors_for_deprecations! diff --git a/spec/support/matchers/benchmark_matchers.rb b/spec/support/matchers/benchmark_matchers.rb new file mode 100644 index 00000000000..45a1d49345f --- /dev/null +++ b/spec/support/matchers/benchmark_matchers.rb @@ -0,0 +1,42 @@ +module BenchmarkMatchers + extend RSpec::Matchers::DSL + + matcher :iterate_per_second do |min_iterations| + supports_block_expectations + + match do |block| + @max_stddev ||= 30 + + @entry = benchmark(&block) + + expect(@entry.ips).to be >= min_iterations + expect(@entry.stddev_percentage).to be <= @max_stddev + end + + chain :with_maximum_stddev do |value| + @max_stddev = value + end + + description do + "run at least #{min_iterations} iterations per second" + end + + failure_message do + ips = @entry.ips.round(2) + stddev = @entry.stddev_percentage.round(2) + + "expected at least #{min_iterations} iterations per second " \ + "with a maximum stddev of #{@max_stddev}%, instead of " \ + "#{ips} iterations per second with a stddev of #{stddev}%" + end + end + + # Benchmarks the given block and returns a Benchmark::IPS::Report::Entry. + def benchmark(&block) + report = Benchmark.ips(quiet: true) do |bench| + bench.report(&block) + end + + report.entries[0] + end +end