diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 5fe8ddf69d7..b1f77bf242c 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -275,29 +275,23 @@ module Ci end def update_coverage - return unless project - coverage_regex = project.build_coverage_regex - return unless coverage_regex coverage = extract_coverage(trace, coverage_regex) - - if coverage.is_a? Numeric - update_attributes(coverage: coverage) - end + update_attributes(coverage: coverage) if coverage.present? end def extract_coverage(text, regex) - begin - matches = text.scan(Regexp.new(regex)).last - matches = matches.last if matches.kind_of?(Array) - coverage = matches.gsub(/\d+(\.\d+)?/).first + return unless regex - if coverage.present? - coverage.to_f - end - rescue - # if bad regex or something goes wrong we dont want to interrupt transition - # so we just silentrly ignore error for now + matches = text.scan(Regexp.new(regex)).last + matches = matches.last if matches.kind_of?(Array) + coverage = matches.gsub(/\d+(\.\d+)?/).first + + if coverage.present? + coverage.to_f end + rescue + # if bad regex or something goes wrong we dont want to interrupt transition + # so we just silentrly ignore error for now end def has_trace_file? @@ -522,6 +516,10 @@ module Ci self.update(artifacts_expire_at: nil) end + def coverage_regex + super || project.try(:build_coverage_regex) + end + def when read_attribute(:when) || build_attributes_from_config[:when] || 'on_success' end diff --git a/changelogs/unreleased/issue-20428.yml b/changelogs/unreleased/issue-20428.yml new file mode 100644 index 00000000000..60da1c14702 --- /dev/null +++ b/changelogs/unreleased/issue-20428.yml @@ -0,0 +1,4 @@ +--- +title: Add ability to define a coverage regex in the .gitlab-ci.yml +merge_request: 7447 +author: Leandro Camargo diff --git a/db/migrate/20161114024742_add_coverage_regex_to_builds.rb b/db/migrate/20161114024742_add_coverage_regex_to_builds.rb new file mode 100644 index 00000000000..88aa5d52b39 --- /dev/null +++ b/db/migrate/20161114024742_add_coverage_regex_to_builds.rb @@ -0,0 +1,13 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddCoverageRegexToBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def change + add_column :ci_builds, :coverage_regex, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 5efb4f6595c..c73c311ccb2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -215,6 +215,7 @@ ActiveRecord::Schema.define(version: 20170130204620) do t.datetime "queued_at" t.string "token" t.integer "lock_version" + t.string "coverage_regex" end add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index f11257be5c3..06810898cfe 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -76,6 +76,7 @@ There are a few reserved `keywords` that **cannot** be used as job names: | after_script | no | Define commands that run after each job's script | | variables | no | Define build variables | | cache | no | Define list of files that should be cached between subsequent runs | +| coverage | no | Define coverage settings for all jobs | ### image and services @@ -278,6 +279,23 @@ cache: untracked: true ``` +### coverage + +`coverage` allows you to configure how coverage will be filtered out from the +build outputs. Setting this up globally will make all the jobs to use this +setting for output filtering and extracting the coverage information from your +builds. + +Regular expressions are the only valid kind of value expected here. So, using +surrounding `/` is mandatory in order to consistently and explicitly represent +a regular expression string. You must escape special characters if you want to +match them literally. + +A simple example: +```yaml +coverage: /\(\d+\.\d+\) covered\./ +``` + ## Jobs `.gitlab-ci.yml` allows you to specify an unlimited number of jobs. Each job @@ -319,6 +337,7 @@ job_name: | before_script | no | Override a set of commands that are executed before build | | after_script | no | Override a set of commands that are executed after build | | environment | no | Defines a name of environment to which deployment is done by this build | +| coverage | no | Define coverage settings for a given job | ### script @@ -993,6 +1012,25 @@ job: - execute this after my script ``` +### job coverage + +This entry is pretty much the same as described in the global context in +[`coverage`](#coverage). The only difference is that, by setting it inside +the job level, whatever is set in there will take precedence over what has +been defined in the global level. A quick example of one overriding the +other would be: + +```yaml +coverage: /\(\d+\.\d+\) covered\./ + +job1: + coverage: /Code coverage: \d+\.\d+/ +``` + +In the example above, considering the context of the job `job1`, the coverage +regex that would be used is `/Code coverage: \d+\.\d+/` instead of +`/\(\d+\.\d+\) covered\./`. + ## Git Strategy > Introduced in GitLab 8.9 as an experimental feature. May change or be removed diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 7463bd719d5..649ee4d018b 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -61,6 +61,7 @@ module Ci allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', environment: job[:environment_name], + coverage_regex: job[:coverage], yaml_variables: yaml_variables(name), options: { image: job[:image], diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb new file mode 100644 index 00000000000..12a063059cb --- /dev/null +++ b/lib/gitlab/ci/config/entry/coverage.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents Coverage settings. + # + class Coverage < Node + include Validatable + + validations do + validates :config, regexp: true + end + + def value + @config[1...-1] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index a4ec8f0ff2f..ede97cc0504 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -33,8 +33,11 @@ module Gitlab entry :cache, Entry::Cache, description: 'Configure caching between build jobs.' + entry :coverage, Entry::Coverage, + description: 'Coverage configuration for this pipeline.' + helpers :before_script, :image, :services, :after_script, - :variables, :stages, :types, :cache, :jobs + :variables, :stages, :types, :cache, :coverage, :jobs def compose!(_deps = nil) super(self) do diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a55362f0b6b..69a5e6f433d 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when artifacts cache dependencies before_script - after_script variables environment] + after_script variables environment coverage] validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -71,9 +71,12 @@ module Gitlab entry :environment, Entry::Environment, description: 'Environment configuration for this job.' + entry :coverage, Entry::Coverage, + description: 'Coverage configuration for this job.' + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment + :artifacts, :commands, :environment, :coverage attributes :script, :tags, :allow_failure, :when, :dependencies @@ -130,6 +133,7 @@ module Gitlab variables: variables_defined? ? variables_value : nil, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, + coverage: coverage_defined? ? coverage_value : nil, artifacts: artifacts_value, after_script: after_script_value } end diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index f01975aab5c..9b9a0a8125a 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -28,17 +28,21 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_regexp(value) + !value.nil? && Regexp.new(value.to_s) && true + rescue RegexpError, TypeError + false + end + def validate_string_or_regexp(value) return true if value.is_a?(Symbol) return false unless value.is_a?(String) if value.first == '/' && value.last == '/' - Regexp.new(value[1...-1]) + validate_regexp(value[1...-1]) else true end - rescue RegexpError - false end def validate_boolean(value) diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index 28b0a9ffe01..16b234e6c59 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -9,15 +9,7 @@ module Gitlab include Validatable validations do - include LegacyValidationHelpers - - validate :array_of_strings_or_regexps - - def array_of_strings_or_regexps - unless validate_array_of_strings_or_regexps(config) - errors.add(:config, 'should be an array of strings or regexps') - end - end + validates :config, array_of_strings_or_regexps: true end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index 8632dd0e233..bd7428b1272 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -54,6 +54,51 @@ module Gitlab end end + class RegexpValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_regexp(value) + record.errors.add(attribute, 'must be a regular expression') + end + end + + private + + def look_like_regexp?(value) + value.is_a?(String) && value.start_with?('/') && + value.end_with?('/') + end + + def validate_regexp(value) + look_like_regexp?(value) && + Regexp.new(value.to_s[1...-1]) && + true + rescue RegexpError + false + end + end + + class ArrayOfStringsOrRegexpsValidator < RegexpValidator + def validate_each(record, attribute, value) + unless validate_array_of_strings_or_regexps(value) + record.errors.add(attribute, 'should be an array of strings or regexps') + end + end + + private + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp)) + end + + def validate_string_or_regexp(value) + return false unless value.is_a?(String) + return validate_regexp(value) if look_like_regexp?(value) + true + end + end + class TypeValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) type = options[:with] diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index f824e2e1efe..49349035b3b 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -4,6 +4,33 @@ module Ci describe GitlabCiYamlProcessor, lib: true do let(:path) { 'path' } + describe '#build_attributes' do + context 'Coverage entry' do + subject { described_class.new(config, path).build_attributes(:rspec) } + + let(:config_base) { { rspec: { script: "rspec" } } } + let(:config) { YAML.dump(config_base) } + + context 'when config has coverage set at the global scope' do + before do + config_base.update(coverage: '/\(\d+\.\d+\) covered/') + end + + context "and 'rspec' job doesn't have coverage set" do + it { is_expected.to include(coverage_regex: '\(\d+\.\d+\) covered') } + end + + context "but 'rspec' job also has coverage set" do + before do + config_base[:rspec][:coverage] = '/Code coverage: \d+\.\d+/' + end + + it { is_expected.to include(coverage_regex: 'Code coverage: \d+\.\d+') } + end + end + end + end + describe "#builds_for_ref" do let(:type) { 'test' } @@ -21,6 +48,7 @@ module Ci stage_idx: 1, name: "rspec", commands: "pwd\nrspec", + coverage_regex: nil, tag_list: [], options: {}, allow_failure: false, @@ -435,6 +463,7 @@ module Ci stage_idx: 1, name: "rspec", commands: "pwd\nrspec", + coverage_regex: nil, tag_list: [], options: { image: "ruby:2.1", @@ -463,6 +492,7 @@ module Ci stage_idx: 1, name: "rspec", commands: "pwd\nrspec", + coverage_regex: nil, tag_list: [], options: { image: "ruby:2.5", @@ -702,6 +732,7 @@ module Ci stage_idx: 1, name: "rspec", commands: "pwd\nrspec", + coverage_regex: nil, tag_list: [], options: { image: "ruby:2.1", @@ -913,6 +944,7 @@ module Ci stage_idx: 1, name: "normal_job", commands: "test", + coverage_regex: nil, tag_list: [], options: {}, when: "on_success", @@ -958,6 +990,7 @@ module Ci stage_idx: 0, name: "job1", commands: "execute-script-for-job", + coverage_regex: nil, tag_list: [], options: {}, when: "on_success", @@ -970,6 +1003,7 @@ module Ci stage_idx: 0, name: "job2", commands: "execute-script-for-job", + coverage_regex: nil, tag_list: [], options: {}, when: "on_success", diff --git a/spec/lib/gitlab/ci/config/entry/coverage_spec.rb b/spec/lib/gitlab/ci/config/entry/coverage_spec.rb new file mode 100644 index 00000000000..4c6bd859552 --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/coverage_spec.rb @@ -0,0 +1,54 @@ +require 'spec_helper' + +describe Gitlab::Ci::Config::Entry::Coverage do + let(:entry) { described_class.new(config) } + + describe 'validations' do + context "when entry config value doesn't have the surrounding '/'" do + let(:config) { 'Code coverage: \d+\.\d+' } + + describe '#errors' do + subject { entry.errors } + it { is_expected.to include(/coverage config must be a regular expression/) } + end + + describe '#valid?' do + subject { entry } + it { is_expected.not_to be_valid } + end + end + + context "when entry config value has the surrounding '/'" do + let(:config) { '/Code coverage: \d+\.\d+/' } + + describe '#value' do + subject { entry.value } + it { is_expected.to eq(config[1...-1]) } + end + + describe '#errors' do + subject { entry.errors } + it { is_expected.to be_empty } + end + + describe '#valid?' do + subject { entry } + it { is_expected.to be_valid } + end + end + + context 'when entry value is not valid' do + let(:config) { '(malformed regexp' } + + describe '#errors' do + subject { entry.errors } + it { is_expected.to include(/coverage config must be a regular expression/) } + end + + describe '#valid?' do + subject { entry } + it { is_expected.not_to be_valid } + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/global_spec.rb b/spec/lib/gitlab/ci/config/entry/global_spec.rb index e64c8d46bd8..d4f1780b174 100644 --- a/spec/lib/gitlab/ci/config/entry/global_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/global_spec.rb @@ -4,12 +4,17 @@ describe Gitlab::Ci::Config::Entry::Global do let(:global) { described_class.new(hash) } describe '.nodes' do - it 'can contain global config keys' do - expect(described_class.nodes).to include :before_script + it 'returns a hash' do + expect(described_class.nodes).to be_a(Hash) end - it 'returns a hash' do - expect(described_class.nodes).to be_a Hash + context 'when filtering all the entry/node names' do + it 'contains the expected node names' do + node_names = described_class.nodes.keys + expect(node_names).to match_array(%i[before_script image services + after_script variables stages + types cache coverage]) + end end end @@ -35,7 +40,7 @@ describe Gitlab::Ci::Config::Entry::Global do end it 'creates node object for each entry' do - expect(global.descendants.count).to eq 8 + expect(global.descendants.count).to eq 9 end it 'creates node object using valid class' do @@ -176,7 +181,7 @@ describe Gitlab::Ci::Config::Entry::Global do describe '#nodes' do it 'instantizes all nodes' do - expect(global.descendants.count).to eq 8 + expect(global.descendants.count).to eq 9 end it 'contains unspecified nodes' do diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index fc9b8b86dc4..d20f4ec207d 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -3,6 +3,20 @@ require 'spec_helper' describe Gitlab::Ci::Config::Entry::Job do let(:entry) { described_class.new(config, name: :rspec) } + describe '.nodes' do + context 'when filtering all the entry/node names' do + subject { described_class.nodes.keys } + + let(:result) do + %i[before_script script stage type after_script cache + image services only except variables artifacts + environment coverage] + end + + it { is_expected.to match_array result } + end + end + describe 'validations' do before { entry.compose! } diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 493bc2db21a..95b230e4f5c 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -222,6 +222,7 @@ CommitStatus: - queued_at - token - lock_version +- coverage_regex Ci::Variable: - id - project_id diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 47cd5075a7d..e20b394c525 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -221,6 +221,47 @@ describe Ci::Build, :models do end end + describe '#coverage_regex' do + subject { build.coverage_regex } + + context 'when project has build_coverage_regex set' do + let(:project_regex) { '\(\d+\.\d+\) covered' } + + before do + project.build_coverage_regex = project_regex + end + + context 'and coverage_regex attribute is not set' do + it { is_expected.to eq(project_regex) } + end + + context 'but coverage_regex attribute is also set' do + let(:build_regex) { 'Code coverage: \d+\.\d+' } + + before do + build.coverage_regex = build_regex + end + + it { is_expected.to eq(build_regex) } + end + end + + context 'when neither project nor build has coverage regex set' do + it { is_expected.to be_nil } + end + end + + describe '#update_coverage' do + context "regarding coverage_regex's value," do + it "saves the correct extracted coverage value" do + build.coverage_regex = '\(\d+.\d+\%\) covered' + allow(build).to receive(:trace) { 'Coverage 1033 / 1051 LOC (98.29%) covered' } + expect(build).to receive(:update_attributes).with(coverage: 98.29) { true } + expect(build.update_coverage).to be true + end + end + end + describe 'deployment' do describe '#last_deployment' do subject { build.last_deployment }