Add "expected_errors" to corpus integrations
This change drops the concept of `excludes` from the corpus integration configuration. Instead with `expected_errors` you specify errors you expect to be raised by certain files. The corpus will fail if an expected error is not encountered when mutating a file, or if an error is encountered while mutating a file that was not whitelisted
This commit is contained in:
John Backus
parent
0e83f310da
commit
9f4e7ce45e
2 changed files with 190 additions and 47 deletions
|
|
@ -5,36 +5,100 @@
|
|||
mutation_coverage: false
|
||||
mutation_generation: true
|
||||
expect_coverage: 0 # not run
|
||||
exclude:
|
||||
# Binary encoded source subjected to limitations see README of unparser
|
||||
- core/array/pack/{b,h,u}_spec.rb
|
||||
- language/regexp/escapes_spec.rb
|
||||
- core/array/pack/shared/float.rb
|
||||
- core/array/pack/shared/integer.rb
|
||||
- core/array/pack/{c,m,w}_spec.rb
|
||||
- core/regexp/shared/quote.rb
|
||||
- core/encoding/compatible_spec.rb
|
||||
- core/io/readpartial_spec.rb
|
||||
- core/env/element_reference_spec.rb
|
||||
- core/dir/pwd_spec.rb
|
||||
- core/string/casecmp_spec.rb
|
||||
- core/string/unpack/{b,c,h,m,u,w}_spec.rb
|
||||
- core/string/unpack/b_spec.rb
|
||||
- core/string/unpack/shared/float.rb
|
||||
- core/string/unpack/shared/integer.rb
|
||||
- core/symbol/casecmp_spec.rb
|
||||
- optional/capi/integer_spec.rb
|
||||
expected_errors:
|
||||
"#<Parser::SyntaxError: invalid multibyte escape: /\xAA/>":
|
||||
- language/regexp/escapes_spec.rb
|
||||
"#<Parser::SyntaxError: literal contains escape sequences incompatible with UTF-8>":
|
||||
- core/array/fixtures/encoded_strings.rb
|
||||
- core/array/pack/shared/string.rb
|
||||
- core/array/pack/shared/unicode.rb
|
||||
- core/encoding/converter/convert_spec.rb
|
||||
- core/encoding/converter/last_error_spec.rb
|
||||
- core/encoding/converter/primitive_convert_spec.rb
|
||||
- core/encoding/converter/primitive_errinfo_spec.rb
|
||||
- core/encoding/converter/putback_spec.rb
|
||||
- core/encoding/fixtures/classes.rb
|
||||
- core/encoding/invalid_byte_sequence_error/error_bytes_spec.rb
|
||||
- core/encoding/invalid_byte_sequence_error/incomplete_input_spec.rb
|
||||
- core/encoding/invalid_byte_sequence_error/readagain_bytes_spec.rb
|
||||
- core/encoding/replicate_spec.rb
|
||||
- core/file/expand_path_spec.rb
|
||||
- core/io/gets_spec.rb
|
||||
- core/io/read_spec.rb
|
||||
- core/io/shared/gets_ascii.rb
|
||||
- core/io/write_spec.rb
|
||||
- core/marshal/dump_spec.rb
|
||||
- core/marshal/fixtures/marshal_data.rb
|
||||
- core/marshal/shared/load.rb
|
||||
- core/random/bytes_spec.rb
|
||||
- core/regexp/match_spec.rb
|
||||
- core/regexp/shared/new_ascii.rb
|
||||
- core/string/byteslice_spec.rb
|
||||
- core/string/codepoints_spec.rb
|
||||
- core/string/comparison_spec.rb
|
||||
- core/string/count_spec.rb
|
||||
- core/string/delete_spec.rb
|
||||
- core/string/element_set_spec.rb
|
||||
- core/string/encode_spec.rb
|
||||
- core/string/encoding_spec.rb
|
||||
- core/string/fixtures/iso-8859-9-encoding.rb
|
||||
- core/string/getbyte_spec.rb
|
||||
- core/string/gsub_spec.rb
|
||||
- core/string/new_spec.rb
|
||||
- core/string/scrub_spec.rb
|
||||
- core/string/shared/chars.rb
|
||||
- core/string/shared/codepoints.rb
|
||||
- core/string/shared/each_codepoint_without_block.rb
|
||||
- core/string/shared/encode.rb
|
||||
- core/string/shared/eql.rb
|
||||
- core/string/shared/succ.rb
|
||||
- core/string/slice_spec.rb
|
||||
- core/string/squeeze_spec.rb
|
||||
- core/string/unicode_normalize_spec.rb
|
||||
- core/string/valid_encoding_spec.rb
|
||||
- core/symbol/casecmp_spec.rb
|
||||
- core/time/_dump_spec.rb
|
||||
- core/time/_load_spec.rb
|
||||
- language/regexp/encoding_spec.rb
|
||||
- language/string_spec.rb
|
||||
- library/digest/md5/shared/constants.rb
|
||||
- library/digest/md5/shared/sample.rb
|
||||
- library/digest/sha1/shared/constants.rb
|
||||
- library/digest/sha256/shared/constants.rb
|
||||
- library/digest/sha384/shared/constants.rb
|
||||
- library/digest/sha512/shared/constants.rb
|
||||
- library/openssl/shared/constants.rb
|
||||
- library/socket/basicsocket/recv_spec.rb
|
||||
- library/socket/socket/gethostbyname_spec.rb
|
||||
- library/stringscanner/getch_spec.rb
|
||||
- library/stringscanner/shared/get_byte.rb
|
||||
- library/zlib/deflate/deflate_spec.rb
|
||||
- library/zlib/deflate/set_dictionary_spec.rb
|
||||
- library/zlib/gzipreader/each_byte_spec.rb
|
||||
- library/zlib/gzipreader/eof_spec.rb
|
||||
- library/zlib/gzipreader/getc_spec.rb
|
||||
- library/zlib/gzipreader/pos_spec.rb
|
||||
- library/zlib/gzipreader/read_spec.rb
|
||||
- library/zlib/gzipreader/rewind_spec.rb
|
||||
- library/zlib/inflate/append_spec.rb
|
||||
- library/zlib/inflate/inflate_spec.rb
|
||||
- library/zlib/inflate/set_dictionary_spec.rb
|
||||
- library/zlib/zstream/flush_next_out_spec.rb
|
||||
- optional/capi/encoding_spec.rb
|
||||
- optional/capi/string_spec.rb
|
||||
'#<RegexpError: invalid multibyte escape: /\xAA/>':
|
||||
- language/regexp/escapes_spec.rb
|
||||
- name: auom
|
||||
namespace: AUOM
|
||||
repo_uri: 'https://github.com/mbj/auom.git'
|
||||
mutation_coverage: true
|
||||
mutation_generation: true
|
||||
exclude: []
|
||||
expected_errors: {}
|
||||
expect_coverage: 1
|
||||
- name: axiom
|
||||
namespace: Axiom
|
||||
repo_uri: 'https://github.com/dkubb/axiom.git'
|
||||
mutation_coverage: false
|
||||
mutation_generation: true
|
||||
exclude: []
|
||||
expected_errors: {}
|
||||
expect_coverage: 1
|
||||
|
|
|
|||
|
|
@ -29,9 +29,11 @@ module MutantSpec
|
|||
START_MESSAGE = 'Starting - %s'.freeze
|
||||
FINISH_MESSAGE = 'Mutations - %4i - %s'.freeze
|
||||
|
||||
DEFAULT_MUTATION_COUNT = 0
|
||||
|
||||
include Adamantium, Anima.new(
|
||||
:exclude,
|
||||
:expect_coverage,
|
||||
:expected_errors,
|
||||
:mutation_coverage,
|
||||
:mutation_generation,
|
||||
:name,
|
||||
|
|
@ -80,21 +82,10 @@ module MutantSpec
|
|||
start: method(:start),
|
||||
in_processes: parallel_processes
|
||||
}
|
||||
total = Parallel.map(effective_ruby_paths, options) do |path|
|
||||
count = 0
|
||||
node =
|
||||
begin
|
||||
Parser::CurrentRuby.parse(path.read)
|
||||
rescue EncodingError, ArgumentError
|
||||
nil # Make rubocop happy
|
||||
end
|
||||
|
||||
if node
|
||||
count += Mutant::Mutator::REGISTRY.call(node).length
|
||||
end
|
||||
total = Parallel.map(effective_ruby_paths, options, &method(:count_mutations_and_check_errors))
|
||||
.inject(DEFAULT_MUTATION_COUNT, :+)
|
||||
|
||||
count
|
||||
end.inject(0, :+)
|
||||
took = Time.now - start
|
||||
puts MUTATION_GENERATION_MESSAGE % [total, took, total / took]
|
||||
self
|
||||
|
|
@ -124,6 +115,40 @@ module MutantSpec
|
|||
|
||||
private
|
||||
|
||||
# Count mutations and check error results against whitelist
|
||||
#
|
||||
# @param path [Pathname] path responsible for exception
|
||||
#
|
||||
# @return [Fixnum] mutations generated
|
||||
def count_mutations_and_check_errors(path)
|
||||
relative_path = path.relative_path_from(repo_path)
|
||||
|
||||
count = count_mutations(path)
|
||||
|
||||
expected_errors.assert_success(relative_path)
|
||||
|
||||
count
|
||||
rescue Exception => exception # rubocop:disable Lint/RescueException
|
||||
expected_errors.assert_error(relative_path, exception)
|
||||
|
||||
DEFAULT_MUTATION_COUNT
|
||||
end
|
||||
|
||||
# Count mutations generated for provided source file
|
||||
#
|
||||
# @param path [Pathname] path to a source file
|
||||
#
|
||||
# @raise [Exception] any error specified by integrations.yml
|
||||
#
|
||||
# @return [Fixnum] number of mutations generated
|
||||
def count_mutations(path)
|
||||
node = Parser::CurrentRuby.parse(path.read)
|
||||
|
||||
return DEFAULT_MUTATION_COUNT unless node
|
||||
|
||||
Mutant::Mutator::REGISTRY.call(node).length
|
||||
end
|
||||
|
||||
# Install mutant
|
||||
#
|
||||
# @return [undefined]
|
||||
|
|
@ -144,19 +169,10 @@ module MutantSpec
|
|||
#
|
||||
# @return [Array<Pathname>]
|
||||
def effective_ruby_paths
|
||||
paths = Pathname
|
||||
Pathname
|
||||
.glob(repo_path.join(RUBY_GLOB_PATTERN))
|
||||
.sort_by(&:size)
|
||||
.reverse
|
||||
|
||||
paths - excluded_paths
|
||||
end
|
||||
|
||||
# The excluded file paths
|
||||
#
|
||||
# @return [Array<Pathname>]
|
||||
def excluded_paths
|
||||
Pathname.glob(repo_path.join(EXCLUDE_GLOB_FORMAT % exclude.join(',')))
|
||||
end
|
||||
|
||||
# Number of parallel processes to use
|
||||
|
|
@ -226,6 +242,58 @@ module MutantSpec
|
|||
end
|
||||
end
|
||||
|
||||
# Mapping of files which we expect to cause errors during mutation generation
|
||||
class ErrorWhitelist
|
||||
class UnnecessaryExpectation < StandardError
|
||||
MESSAGE = 'Expected to encounter %s while mutating "%s"'.freeze
|
||||
|
||||
def initialize(*error_info)
|
||||
super(MESSAGE % error_info)
|
||||
end
|
||||
end
|
||||
|
||||
include Concord.new(:map), Adamantium
|
||||
|
||||
# Assert that we expect to encounter the provided exception for this path
|
||||
#
|
||||
# @param path [Pathname]
|
||||
# @param exception [Exception]
|
||||
#
|
||||
# @raise provided exception if we are not expecting this error
|
||||
#
|
||||
# This method is reraising exceptions but rubocop can't tell
|
||||
# rubocop:disable Style/SignalException
|
||||
#
|
||||
# @return [undefined]
|
||||
def assert_error(path, exception)
|
||||
original_error = exception.cause || exception
|
||||
|
||||
raise exception unless map.fetch(original_error.inspect, []).include?(path)
|
||||
end
|
||||
|
||||
# Assert that we expect to not encounter an error for the specified path
|
||||
#
|
||||
# @param path [Pathname]
|
||||
#
|
||||
# @raise [UnnecessaryExpectation] if we are expecting an exception for this path
|
||||
#
|
||||
# @return [undefined]
|
||||
def assert_success(path)
|
||||
map.each do |error, paths|
|
||||
fail UnnecessaryExpectation.new(error, path) if paths.include?(path)
|
||||
end
|
||||
end
|
||||
|
||||
# Return representation as hash
|
||||
#
|
||||
# @note this method is necessary for morpher loader to be invertible
|
||||
#
|
||||
# @return [Hash{Pathname => String}]
|
||||
def to_h
|
||||
map
|
||||
end
|
||||
end # ErrorWhitelist
|
||||
|
||||
# rubocop:disable ClosingParenthesisIndentation
|
||||
LOADER = Morpher.build do
|
||||
s(:block,
|
||||
|
|
@ -242,13 +310,24 @@ module MutantSpec
|
|||
s(:guard, s(:or, s(:primitive, TrueClass), s(:primitive, FalseClass)))),
|
||||
s(:key_symbolize, :mutation_generation,
|
||||
s(:guard, s(:or, s(:primitive, TrueClass), s(:primitive, FalseClass)))),
|
||||
s(:key_symbolize, :exclude, s(:map, s(:guard, s(:primitive, String))))
|
||||
s(:key_symbolize, :expected_errors,
|
||||
s(:block,
|
||||
s(:guard, s(:primitive, Hash)),
|
||||
s(:custom,
|
||||
[
|
||||
->(hash) { hash.map { |key, values| [key, values.map(&Pathname.method(:new))] }.to_h },
|
||||
->(hash) { hash.map { |key, values| [key, values.map(&:to_s)] }.to_h }
|
||||
]
|
||||
),
|
||||
s(:load_attribute_hash, s(:param, ErrorWhitelist))
|
||||
)
|
||||
)
|
||||
),
|
||||
s(:load_attribute_hash,
|
||||
# NOTE: The domain param has no DSL currently!
|
||||
Morpher::Evaluator::Transformer::Domain::Param.new(
|
||||
Project,
|
||||
%i[repo_uri name exclude mutation_coverage mutation_generation]
|
||||
%i[repo_uri name expected_errors mutation_coverage mutation_generation]
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue