Basic setup for an RSpec based benchmark suite
This benchmark suite uses benchmark-ips (https://github.com/evanphx/benchmark-ips) behind the scenes. Specs can be turned into benchmark specs by setting "benchmark" to "true" in the top-level describe block like so: describe SomeClass, benchmark: true do end Writing benchmarks can be done using custom RSpec matchers, for example: describe MaruTheCat, benchmark: true do describe '#jump_in_box' do it 'should run 1000 iterations per second' do maru = described_class.new expect { maru.jump_in_box }.to iterate_per_second(1000) end end end By default the "iterate_per_second" expectation requires a standard deviation under 30% (this is just an arbitrary default for now). You can change this by chaining "with_maximum_stddev" on the expectation: expect { maru.jump_in_box }.to iterate_per_second(1000) .with_maximum_stddev(10) This will change the expectation to require a maximum deviation of 10%. Alternatively you can use the it block style to write specs: describe MaruTheCat, benchmark: true do describe '#jump_in_box' do subject { -> { described_class.new } } it { is_expected.to iterate_per_second(1000) } end end Because "iterate_per_second" operates on a block, opposed to a static value, the "subject" method must return a Proc. This looks a bit goofy but I have been unable to find a nice way around this.
This commit is contained in:
parent
dbc05d4a62
commit
19893a1c10
|
@ -24,6 +24,13 @@ spec:api:
|
|||
- ruby
|
||||
- mysql
|
||||
|
||||
spec:benchmark:
|
||||
script:
|
||||
- RAILS_ENV=test bundle exec rake spec:benchmark
|
||||
tags:
|
||||
- ruby
|
||||
- mysql
|
||||
|
||||
spec:other:
|
||||
script:
|
||||
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other
|
||||
|
|
|
@ -19,11 +19,20 @@ namespace :spec do
|
|||
run_commands(cmds)
|
||||
end
|
||||
|
||||
desc 'GitLab | Rspec | Run benchmark specs'
|
||||
task :benchmark do
|
||||
cmds = [
|
||||
%W(rake gitlab:setup),
|
||||
%W(rspec spec --tag @benchmark)
|
||||
]
|
||||
run_commands(cmds)
|
||||
end
|
||||
|
||||
desc 'GitLab | Rspec | Run other specs'
|
||||
task :other do
|
||||
cmds = [
|
||||
%W(rake gitlab:setup),
|
||||
%W(rspec spec --tag ~@api --tag ~@feature)
|
||||
%W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark)
|
||||
]
|
||||
run_commands(cmds)
|
||||
end
|
||||
|
|
|
@ -0,0 +1,40 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe User, benchmark: true do
|
||||
describe '.by_login' do
|
||||
before do
|
||||
%w{Alice Bob Eve}.each do |name|
|
||||
create(:user,
|
||||
email: "#{name}@gitlab.com",
|
||||
username: name,
|
||||
name: name)
|
||||
end
|
||||
end
|
||||
|
||||
let(:iterations) { 1000 }
|
||||
|
||||
describe 'using a capitalized username' do
|
||||
subject { -> { User.by_login('Alice') } }
|
||||
|
||||
it { is_expected.to iterate_per_second(iterations) }
|
||||
end
|
||||
|
||||
describe 'using a lowercase username' do
|
||||
subject { -> { User.by_login('alice') } }
|
||||
|
||||
it { is_expected.to iterate_per_second(iterations) }
|
||||
end
|
||||
|
||||
describe 'using a capitalized Email address' do
|
||||
subject { -> { User.by_login('Alice@gitlab.com') } }
|
||||
|
||||
it { is_expected.to iterate_per_second(iterations) }
|
||||
end
|
||||
|
||||
describe 'using a lowercase Email address' do
|
||||
subject { -> { User.by_login('alice@gitlab.com') } }
|
||||
|
||||
it { is_expected.to iterate_per_second(iterations) }
|
||||
end
|
||||
end
|
||||
end
|
|
@ -14,6 +14,7 @@ require File.expand_path("../../config/environment", __FILE__)
|
|||
require 'rspec/rails'
|
||||
require 'shoulda/matchers'
|
||||
require 'sidekiq/testing/inline'
|
||||
require 'benchmark/ips'
|
||||
|
||||
# Requires supporting ruby files with custom matchers and macros, etc,
|
||||
# in spec/support/ and its subdirectories.
|
||||
|
@ -32,7 +33,7 @@ RSpec.configure do |config|
|
|||
config.include TestEnv
|
||||
config.include StubGitlabCalls
|
||||
config.include StubGitlabData
|
||||
|
||||
config.include BenchmarkMatchers, benchmark: true
|
||||
|
||||
config.infer_spec_type_from_file_location!
|
||||
config.raise_errors_for_deprecations!
|
||||
|
|
|
@ -0,0 +1,42 @@
|
|||
module BenchmarkMatchers
|
||||
extend RSpec::Matchers::DSL
|
||||
|
||||
matcher :iterate_per_second do |min_iterations|
|
||||
supports_block_expectations
|
||||
|
||||
match do |block|
|
||||
@max_stddev ||= 30
|
||||
|
||||
@entry = benchmark(&block)
|
||||
|
||||
expect(@entry.ips).to be >= min_iterations
|
||||
expect(@entry.stddev_percentage).to be <= @max_stddev
|
||||
end
|
||||
|
||||
chain :with_maximum_stddev do |value|
|
||||
@max_stddev = value
|
||||
end
|
||||
|
||||
description do
|
||||
"run at least #{min_iterations} iterations per second"
|
||||
end
|
||||
|
||||
failure_message do
|
||||
ips = @entry.ips.round(2)
|
||||
stddev = @entry.stddev_percentage.round(2)
|
||||
|
||||
"expected at least #{min_iterations} iterations per second " \
|
||||
"with a maximum stddev of #{@max_stddev}%, instead of " \
|
||||
"#{ips} iterations per second with a stddev of #{stddev}%"
|
||||
end
|
||||
end
|
||||
|
||||
# Benchmarks the given block and returns a Benchmark::IPS::Report::Entry.
|
||||
def benchmark(&block)
|
||||
report = Benchmark.ips(quiet: true) do |bench|
|
||||
bench.report(&block)
|
||||
end
|
||||
|
||||
report.entries[0]
|
||||
end
|
||||
end
|
Loading…
Reference in New Issue