kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Skip contexts in quarantine 

This avoids running before/after blocks for tests that are in quarantine
This commit is contained in:
Mark Lapierre 2019-02-14 04:35:20 +00:00 committed by Sanad Liaquat
parent c3c43be39b
commit 8931420202
2 changed files with 237 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
qa/spec/spec_helper.rb
View File

@ -19,26 +19,16 @@ RSpec.configure do |config|
end
end
config.before do |example|
QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
# If quarantine is tagged, skip tests that have other metadata unless
# they're also tagged. This lets us run quarantined tests in a particular
# category without running tests in other categories.
# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
# using `--tag quarantine --tag smoke`, without this check we'd end up
# running that ldap test as well.
if config.inclusion_filter[:quarantine]
skip("Running tests tagged with all of #{config.inclusion_filter.rules.keys}") unless quarantine_and_optional_other_tag?(example, config)
config.before(:context) do
if self.class.metadata.keys.include?(:quarantine)
skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys)
end
end
config.before(:each, :quarantine) do |example|
# Skip tests in quarantine unless we explicitly focus on them
# We could use an exclusion filter, but this way the test report will list
# the quarantined tests when they're not run so that we're aware of them
skip('In quarantine') unless config.inclusion_filter[:quarantine]
config.before do |example|
QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys)
end
config.expect_with :rspec do |expectations|
@ -57,18 +47,41 @@ RSpec.configure do |config|
Kernel.srand config.seed
end
# Skip tests in quarantine unless we explicitly focus on them.
# Skip the entire context if a context is tagged. This avoids running before
# blocks unnecessarily.
# If quarantine is focussed, skip tests/contexts that have other metadata
# unless they're also focussed. This lets us run quarantined tests in a
# particular category without running tests in other categories.
# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
# using `--tag quarantine --tag smoke`, without this check we'd end up
# running that ldap test as well.
# We could use an exclusion filter, but this way the test report will list
# the quarantined tests when they're not run so that we're aware of them
def skip_or_run_quarantined_tests(metadata_keys, filter_keys)
included_filters = filters_other_than_quarantine(filter_keys)
if filter_keys.include?(:quarantine)
skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters)
else
skip('In quarantine') if metadata_keys.include?(:quarantine)
end
end
def filters_other_than_quarantine(filter_keys)
filter_keys.reject { |key| key == :quarantine }
end
# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
#
# Returns true if
# - the example metadata includes the quarantine tag
# - and the metadata and inclusion filter both have any other tag
# - or no other tags are in the inclusion filter
def quarantine_and_optional_other_tag?(example, config)
return false unless example.metadata.keys.include? :quarantine
# - the metadata includes the quarantine tag
# - and the metadata and inclusion filter both have any other tag
# - or no other tags are in the inclusion filter
def quarantine_and_optional_other_tag?(metadata_keys, included_filters)
return false unless metadata_keys.include? :quarantine
return true if included_filters.empty?
filters_other_than_quarantine = config.inclusion_filter.rules.keys.reject { |key| key == :quarantine }
return true if filters_other_than_quarantine.empty?
filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key }
included_filters.any? { |key| metadata_keys.include? key }
end

354
qa/spec/spec_helper_spec.rb
View File

@ -10,49 +10,82 @@ describe 'rspec config tests' do
end
end
context 'default' do
it_behaves_like 'passing tests'
end
context 'foo', :foo do
it_behaves_like 'passing tests'
end
context 'default' do
context 'quarantine', :quarantine do
it_behaves_like 'passing tests'
end
context 'bar quarantine', :bar, :quarantine do
it_behaves_like 'passing tests'
end
end
end
context 'default config' do
it 'tests are skipped if in quarantine' do
context 'with no tags focussed' do
before do
group.run
end
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
context 'in a context tagged :foo' do
it 'skips tests in quarantine' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to eq(2)
context 'in an untagged context' do
it 'skips tests in quarantine' do
context = group.children.find { |c| c.description == "default" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
ex = default_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = default_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
context 'in a context tagged :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
end
context "with 'quarantine' tagged" do
context 'with :quarantine focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = :quarantine
end
group.run
end
after do
RSpec.configure do |config|
@ -60,26 +93,44 @@ describe 'rspec config tests' do
end
end
it "only quarantined tests are run" do
group.run
context 'in an untagged context' do
it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "default" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(1)
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to be(1)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
context 'in a context tagged :foo' do
it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(1)
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to be(1)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
ex = default_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
context 'in a context tagged :quarantine' do
it 'runs all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
end
context "with 'foo' tagged" do
context 'with a non-quarantine tag (:foo) focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = :foo
@ -93,30 +144,43 @@ describe 'rspec config tests' do
end
end
it "tests are not run if not tagged 'foo'" do
default_context = group.children.find { |c| c.description == "default" }
expect(default_context.descendant_filtered_examples.count).to eq(0)
context 'in an untagged context' do
it 'runs no tests' do
context = group.children.find { |c| c.description == "default" }
expect(context.descendant_filtered_examples.count).to eq(0)
end
end
it "tests are skipped if in quarantine" do
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
context 'in a context tagged :foo' do
it 'skips quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = foo_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
context 'in a context tagged :quarantine' do
it 'runs no tests' do
context = group.children.find { |c| c.description == "quarantine" }
expect(context.descendant_filtered_examples.count).to eq(0)
end
end
end
context "with 'quarantine' and 'foo' tagged" do
context 'with :quarantine and a non-quarantine tag (:foo) focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = { quarantine: true, foo: true }
end
group.run
end
after do
RSpec.configure do |config|
@ -124,106 +188,67 @@ describe 'rspec config tests' do
end
end
it 'of tests tagged foo, only tests in quarantine run' do
group.run
context 'in an untagged context' do
it 'ignores untagged tests and skips tests even if in quarantine' do
context = group.children.find { |c| c.description == "default" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(1)
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end
end
it 'if tests are not tagged they are skipped, even if they are in quarantine' do
group.run
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to eq(1)
context 'in a context tagged :foo' do
it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = default_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end
end
context 'in a context tagged :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end
end
context 'in a context tagged :bar and :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end
end
end
context "with 'foo' and 'bar' tagged" do
before do
RSpec.configure do |config|
config.inclusion_filter = { bar: true, foo: true }
end
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it "runs tests tagged either 'foo' or 'bar'" do
group = RSpec.describe do
example 'foo', :foo do
end
example 'bar', :bar do
end
example 'foo and bar', :foo, :bar do
end
end
group.run
expect(group.examples.count).to eq(3)
ex = group.examples.find { |e| e.description == "foo" }
expect(ex.execution_result.status).to eq(:passed)
ex = group.examples.find { |e| e.description == "bar" }
expect(ex.execution_result.status).to eq(:passed)
ex = group.examples.find { |e| e.description == "foo and bar" }
expect(ex.execution_result.status).to eq(:passed)
end
it "skips quarantined tests tagged 'foo' and/or 'bar'" do
group = RSpec.describe do
example 'foo in quarantine', :foo, :quarantine do
end
example 'foo and bar in quarantine', :foo, :bar, :quarantine do
end
end
group.run
expect(group.examples.count).to eq(2)
ex = group.examples.find { |e| e.description == "foo in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = group.examples.find { |e| e.description == "foo and bar in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
it "ignores quarantined tests not tagged either 'foo' or 'bar'" do
group = RSpec.describe do
example 'in quarantine', :quarantine do
end
end
group.run
ex = group.examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to be_nil
end
end
context "with 'foo' and 'bar' and 'quarantined' tagged" do
context 'with :quarantine and multiple non-quarantine tags focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = { bar: true, foo: true, quarantine: true }
end
group.run
end
after do
RSpec.configure do |config|
@ -231,34 +256,49 @@ describe 'rspec config tests' do
end
end
it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do
group = RSpec.describe do
example 'foo', :foo do
end
example 'bar and quarantine', :bar, :quarantine do
end
example 'foo and bar', :foo, :bar do
end
example 'foo, bar, and quarantine', :foo, :bar, :quarantine do
end
context 'in a context tagged :foo' do
it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
end
end
group.run
expect(group.examples.count).to eq(4)
context 'in a context tagged :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]')
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
ex = group.examples.find { |e| e.description == "bar and quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
end
end
ex = group.examples.find { |e| e.description == "foo and bar" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]')
context 'in a context tagged :bar and :quarantine' do
it 'runs all tests' do
context = group.children.find { |c| c.description == "bar quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
end
end