From 9f4e7ce45e843d0ff7821386d11e0c40e3ed4606 Mon Sep 17 00:00:00 2001 From: John Backus Date: Sat, 23 Apr 2016 20:11:06 -0700 Subject: [PATCH] Add "expected_errors" to corpus integrations This change drops the concept of `excludes` from the corpus integration configuration. Instead with `expected_errors` you specify errors you expect to be raised by certain files. The corpus will fail if an expected error is not encountered when mutating a file, or if an error is encountered while mutating a file that was not whitelisted --- spec/integrations.yml | 106 ++++++++++++++++++++++++++------- spec/support/corpus.rb | 131 +++++++++++++++++++++++++++++++++-------- 2 files changed, 190 insertions(+), 47 deletions(-) diff --git a/spec/integrations.yml b/spec/integrations.yml index 39c00a28..77ec7925 100644 --- a/spec/integrations.yml +++ b/spec/integrations.yml @@ -5,36 +5,100 @@ mutation_coverage: false mutation_generation: true expect_coverage: 0 # not run - exclude: - # Binary encoded source subjected to limitations see README of unparser - - core/array/pack/{b,h,u}_spec.rb - - language/regexp/escapes_spec.rb - - core/array/pack/shared/float.rb - - core/array/pack/shared/integer.rb - - core/array/pack/{c,m,w}_spec.rb - - core/regexp/shared/quote.rb - - core/encoding/compatible_spec.rb - - core/io/readpartial_spec.rb - - core/env/element_reference_spec.rb - - core/dir/pwd_spec.rb - - core/string/casecmp_spec.rb - - core/string/unpack/{b,c,h,m,u,w}_spec.rb - - core/string/unpack/b_spec.rb - - core/string/unpack/shared/float.rb - - core/string/unpack/shared/integer.rb - - core/symbol/casecmp_spec.rb - - optional/capi/integer_spec.rb + expected_errors: + "#": + - language/regexp/escapes_spec.rb + "#": + - core/array/fixtures/encoded_strings.rb + - core/array/pack/shared/string.rb + - core/array/pack/shared/unicode.rb + - core/encoding/converter/convert_spec.rb + - core/encoding/converter/last_error_spec.rb + - core/encoding/converter/primitive_convert_spec.rb + - core/encoding/converter/primitive_errinfo_spec.rb + - core/encoding/converter/putback_spec.rb + - core/encoding/fixtures/classes.rb + - core/encoding/invalid_byte_sequence_error/error_bytes_spec.rb + - core/encoding/invalid_byte_sequence_error/incomplete_input_spec.rb + - core/encoding/invalid_byte_sequence_error/readagain_bytes_spec.rb + - core/encoding/replicate_spec.rb + - core/file/expand_path_spec.rb + - core/io/gets_spec.rb + - core/io/read_spec.rb + - core/io/shared/gets_ascii.rb + - core/io/write_spec.rb + - core/marshal/dump_spec.rb + - core/marshal/fixtures/marshal_data.rb + - core/marshal/shared/load.rb + - core/random/bytes_spec.rb + - core/regexp/match_spec.rb + - core/regexp/shared/new_ascii.rb + - core/string/byteslice_spec.rb + - core/string/codepoints_spec.rb + - core/string/comparison_spec.rb + - core/string/count_spec.rb + - core/string/delete_spec.rb + - core/string/element_set_spec.rb + - core/string/encode_spec.rb + - core/string/encoding_spec.rb + - core/string/fixtures/iso-8859-9-encoding.rb + - core/string/getbyte_spec.rb + - core/string/gsub_spec.rb + - core/string/new_spec.rb + - core/string/scrub_spec.rb + - core/string/shared/chars.rb + - core/string/shared/codepoints.rb + - core/string/shared/each_codepoint_without_block.rb + - core/string/shared/encode.rb + - core/string/shared/eql.rb + - core/string/shared/succ.rb + - core/string/slice_spec.rb + - core/string/squeeze_spec.rb + - core/string/unicode_normalize_spec.rb + - core/string/valid_encoding_spec.rb + - core/symbol/casecmp_spec.rb + - core/time/_dump_spec.rb + - core/time/_load_spec.rb + - language/regexp/encoding_spec.rb + - language/string_spec.rb + - library/digest/md5/shared/constants.rb + - library/digest/md5/shared/sample.rb + - library/digest/sha1/shared/constants.rb + - library/digest/sha256/shared/constants.rb + - library/digest/sha384/shared/constants.rb + - library/digest/sha512/shared/constants.rb + - library/openssl/shared/constants.rb + - library/socket/basicsocket/recv_spec.rb + - library/socket/socket/gethostbyname_spec.rb + - library/stringscanner/getch_spec.rb + - library/stringscanner/shared/get_byte.rb + - library/zlib/deflate/deflate_spec.rb + - library/zlib/deflate/set_dictionary_spec.rb + - library/zlib/gzipreader/each_byte_spec.rb + - library/zlib/gzipreader/eof_spec.rb + - library/zlib/gzipreader/getc_spec.rb + - library/zlib/gzipreader/pos_spec.rb + - library/zlib/gzipreader/read_spec.rb + - library/zlib/gzipreader/rewind_spec.rb + - library/zlib/inflate/append_spec.rb + - library/zlib/inflate/inflate_spec.rb + - library/zlib/inflate/set_dictionary_spec.rb + - library/zlib/zstream/flush_next_out_spec.rb + - optional/capi/encoding_spec.rb + - optional/capi/string_spec.rb + '#': + - language/regexp/escapes_spec.rb - name: auom namespace: AUOM repo_uri: 'https://github.com/mbj/auom.git' mutation_coverage: true mutation_generation: true - exclude: [] + expected_errors: {} expect_coverage: 1 - name: axiom namespace: Axiom repo_uri: 'https://github.com/dkubb/axiom.git' mutation_coverage: false mutation_generation: true - exclude: [] + expected_errors: {} expect_coverage: 1 diff --git a/spec/support/corpus.rb b/spec/support/corpus.rb index 4fc8c288..eea198e5 100644 --- a/spec/support/corpus.rb +++ b/spec/support/corpus.rb @@ -29,9 +29,11 @@ module MutantSpec START_MESSAGE = 'Starting - %s'.freeze FINISH_MESSAGE = 'Mutations - %4i - %s'.freeze + DEFAULT_MUTATION_COUNT = 0 + include Adamantium, Anima.new( - :exclude, :expect_coverage, + :expected_errors, :mutation_coverage, :mutation_generation, :name, @@ -80,21 +82,10 @@ module MutantSpec start: method(:start), in_processes: parallel_processes } - total = Parallel.map(effective_ruby_paths, options) do |path| - count = 0 - node = - begin - Parser::CurrentRuby.parse(path.read) - rescue EncodingError, ArgumentError - nil # Make rubocop happy - end - if node - count += Mutant::Mutator::REGISTRY.call(node).length - end + total = Parallel.map(effective_ruby_paths, options, &method(:count_mutations_and_check_errors)) + .inject(DEFAULT_MUTATION_COUNT, :+) - count - end.inject(0, :+) took = Time.now - start puts MUTATION_GENERATION_MESSAGE % [total, took, total / took] self @@ -124,6 +115,40 @@ module MutantSpec private + # Count mutations and check error results against whitelist + # + # @param path [Pathname] path responsible for exception + # + # @return [Fixnum] mutations generated + def count_mutations_and_check_errors(path) + relative_path = path.relative_path_from(repo_path) + + count = count_mutations(path) + + expected_errors.assert_success(relative_path) + + count + rescue Exception => exception # rubocop:disable Lint/RescueException + expected_errors.assert_error(relative_path, exception) + + DEFAULT_MUTATION_COUNT + end + + # Count mutations generated for provided source file + # + # @param path [Pathname] path to a source file + # + # @raise [Exception] any error specified by integrations.yml + # + # @return [Fixnum] number of mutations generated + def count_mutations(path) + node = Parser::CurrentRuby.parse(path.read) + + return DEFAULT_MUTATION_COUNT unless node + + Mutant::Mutator::REGISTRY.call(node).length + end + # Install mutant # # @return [undefined] @@ -144,19 +169,10 @@ module MutantSpec # # @return [Array] def effective_ruby_paths - paths = Pathname + Pathname .glob(repo_path.join(RUBY_GLOB_PATTERN)) .sort_by(&:size) .reverse - - paths - excluded_paths - end - - # The excluded file paths - # - # @return [Array] - def excluded_paths - Pathname.glob(repo_path.join(EXCLUDE_GLOB_FORMAT % exclude.join(','))) end # Number of parallel processes to use @@ -226,6 +242,58 @@ module MutantSpec end end + # Mapping of files which we expect to cause errors during mutation generation + class ErrorWhitelist + class UnnecessaryExpectation < StandardError + MESSAGE = 'Expected to encounter %s while mutating "%s"'.freeze + + def initialize(*error_info) + super(MESSAGE % error_info) + end + end + + include Concord.new(:map), Adamantium + + # Assert that we expect to encounter the provided exception for this path + # + # @param path [Pathname] + # @param exception [Exception] + # + # @raise provided exception if we are not expecting this error + # + # This method is reraising exceptions but rubocop can't tell + # rubocop:disable Style/SignalException + # + # @return [undefined] + def assert_error(path, exception) + original_error = exception.cause || exception + + raise exception unless map.fetch(original_error.inspect, []).include?(path) + end + + # Assert that we expect to not encounter an error for the specified path + # + # @param path [Pathname] + # + # @raise [UnnecessaryExpectation] if we are expecting an exception for this path + # + # @return [undefined] + def assert_success(path) + map.each do |error, paths| + fail UnnecessaryExpectation.new(error, path) if paths.include?(path) + end + end + + # Return representation as hash + # + # @note this method is necessary for morpher loader to be invertible + # + # @return [Hash{Pathname => String}] + def to_h + map + end + end # ErrorWhitelist + # rubocop:disable ClosingParenthesisIndentation LOADER = Morpher.build do s(:block, @@ -242,13 +310,24 @@ module MutantSpec s(:guard, s(:or, s(:primitive, TrueClass), s(:primitive, FalseClass)))), s(:key_symbolize, :mutation_generation, s(:guard, s(:or, s(:primitive, TrueClass), s(:primitive, FalseClass)))), - s(:key_symbolize, :exclude, s(:map, s(:guard, s(:primitive, String)))) + s(:key_symbolize, :expected_errors, + s(:block, + s(:guard, s(:primitive, Hash)), + s(:custom, + [ + ->(hash) { hash.map { |key, values| [key, values.map(&Pathname.method(:new))] }.to_h }, + ->(hash) { hash.map { |key, values| [key, values.map(&:to_s)] }.to_h } + ] + ), + s(:load_attribute_hash, s(:param, ErrorWhitelist)) + ) + ) ), s(:load_attribute_hash, # NOTE: The domain param has no DSL currently! Morpher::Evaluator::Transformer::Domain::Param.new( Project, - %i[repo_uri name exclude mutation_coverage mutation_generation] + %i[repo_uri name expected_errors mutation_coverage mutation_generation] ) ) )