From a5549176e1fb5780fa68778571f0eda563dcf090 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 23 Aug 2021 06:09:01 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/CODEOWNERS | 4 +- app/models/instance_configuration.rb | 17 ++-- .../help/instance_configuration.html.haml | 2 +- .../_gitlab_ci.html.haml | 24 ------ .../_gitlab_pages.html.haml | 5 -- .../_size_limits.html.haml | 40 ++++++++++ db/ci_migrate | 1 - lib/gitlab/ci/config/entry/processable.rb | 3 +- lib/gitlab/config/entry/validators.rb | 12 ++- locale/gitlab.pot | 42 ++++++---- spec/lib/gitlab/ci/config/entry/job_spec.rb | 80 +++++++++++++++++++ spec/lib/gitlab/ci/config/entry/rules_spec.rb | 8 +- spec/models/instance_configuration_spec.rb | 48 ++++++++--- .../instance_configuration.html.haml_spec.rb | 9 ++- 14 files changed, 219 insertions(+), 76 deletions(-) delete mode 100644 app/views/help/instance_configuration/_gitlab_ci.html.haml create mode 100644 app/views/help/instance_configuration/_size_limits.html.haml delete mode 120000 db/ci_migrate diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index ae697582623..621841f2f3d 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -201,9 +201,9 @@ Dangerfile @gl-quality/eng-prod /lib/gitlab/auth/ldap/ @dblessing @mkozono [Templates] -/lib/gitlab/ci/templates/ @nolith @shinya.maeda @matteeyah +/lib/gitlab/ci/templates/ @gitlab-org/maintainers/cicd-templates /lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @DylanGriffith @mayra-cabrera @tkuah -/lib/gitlab/ci/templates/Security/ @gonzoyumo @twoodham @sethgitlab @thiagocsf +/lib/gitlab/ci/templates/Security/ @gonzoyumo @twoodham @sethgitlab @thiagocsf /lib/gitlab/ci/templates/Security/Container-Scanning.*.yml @gitlab-org/protect/container-security-backend [Project Alias] diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index 09a60e9dd10..9565dae08b5 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -13,7 +13,7 @@ class InstanceConfiguration { ssh_algorithms_hashes: ssh_algorithms_hashes, host: host, gitlab_pages: gitlab_pages, - gitlab_ci: gitlab_ci, + size_limits: size_limits, package_file_size_limits: package_file_size_limits, rate_limits: rate_limits }.deep_symbolize_keys end @@ -38,11 +38,16 @@ class InstanceConfiguration rescue Resolv::ResolvError end - def gitlab_ci - Settings.gitlab_ci - .to_h - .merge(artifacts_max_size: { value: Gitlab::CurrentSettings.max_artifacts_size.megabytes, - default: 100.megabytes }) + def size_limits + { + max_attachment_size: application_settings[:max_attachment_size].megabytes, + receive_max_input_size: application_settings[:receive_max_input_size]&.megabytes, + max_import_size: application_settings[:max_import_size] > 0 ? application_settings[:max_import_size].megabytes : nil, + diff_max_patch_bytes: application_settings[:diff_max_patch_bytes].bytes, + max_artifacts_size: application_settings[:max_artifacts_size].megabytes, + max_pages_size: application_settings[:max_pages_size] > 0 ? application_settings[:max_pages_size].megabytes : nil, + snippet_size_limit: application_settings[:snippet_size_limit]&.bytes + } end def package_file_size_limits diff --git a/app/views/help/instance_configuration.html.haml b/app/views/help/instance_configuration.html.haml index 88c531535b4..411a81cb976 100644 --- a/app/views/help/instance_configuration.html.haml +++ b/app/views/help/instance_configuration.html.haml @@ -7,7 +7,7 @@ = render 'help/instance_configuration/ssh_info' = render 'help/instance_configuration/gitlab_pages' - = render 'help/instance_configuration/gitlab_ci' + = render 'help/instance_configuration/size_limits' = render 'help/instance_configuration/package_registry' = render 'help/instance_configuration/rate_limits' %p diff --git a/app/views/help/instance_configuration/_gitlab_ci.html.haml b/app/views/help/instance_configuration/_gitlab_ci.html.haml deleted file mode 100644 index 53fa3f89873..00000000000 --- a/app/views/help/instance_configuration/_gitlab_ci.html.haml +++ /dev/null @@ -1,24 +0,0 @@ -- content_for :table_content do - %li= link_to _('GitLab CI'), '#gitlab-ci' - -- content_for :settings_content do - %h2#gitlab-ci - = _('GitLab CI') - - %p - = _('Below are the current settings regarding') - = succeed('.') { link_to(_('GitLab CI'), 'https://about.gitlab.com/gitlab-ci', target: '_blank') } - - .table-responsive - %table - %thead - %tr - %th= _('Setting') - %th= instance_configuration_host(@instance_configuration.settings[:host]) - %th= _('Default') - %tbody - %tr - - artifacts_size = @instance_configuration.settings[:gitlab_ci][:artifacts_max_size] - %td= _('Artifacts maximum size') - %td= instance_configuration_human_size_cell(artifacts_size[:value]) - %td= instance_configuration_human_size_cell(artifacts_size[:default]) diff --git a/app/views/help/instance_configuration/_gitlab_pages.html.haml b/app/views/help/instance_configuration/_gitlab_pages.html.haml index 55f043214f6..51835c202d6 100644 --- a/app/views/help/instance_configuration/_gitlab_pages.html.haml +++ b/app/views/help/instance_configuration/_gitlab_pages.html.haml @@ -28,8 +28,3 @@ %td= _('Port') %td %code= instance_configuration_cell_html(gitlab_pages[:port]) - %br - - %p - - link_to_gitlab_ci = link_to(_('GitLab CI'), '#gitlab-ci') - = _("The maximum size of your Pages site is regulated by the artifacts maximum size which is part of %{link_to_gitlab_ci}.").html_safe % { link_to_gitlab_ci: link_to_gitlab_ci } diff --git a/app/views/help/instance_configuration/_size_limits.html.haml b/app/views/help/instance_configuration/_size_limits.html.haml new file mode 100644 index 00000000000..b592eeed020 --- /dev/null +++ b/app/views/help/instance_configuration/_size_limits.html.haml @@ -0,0 +1,40 @@ +- size_limits = @instance_configuration.settings[:size_limits] +- content_for :table_content do + - if size_limits.present? + %li= link_to _('Size Limits'), '#size-limits' + +- content_for :settings_content do + - if size_limits.present? + %h2#size-limits + = _('Size Limits') + + %p + = _('There are several size limits in place.') + .table-responsive + %table + %thead + %tr + %th= _('Setting') + %th= instance_configuration_host(@instance_configuration.settings[:host]) + %tbody + %tr + %td= _('Maximum attachment size') + %td= instance_configuration_human_size_cell(size_limits[:max_attachment_size]) + %tr + %td= _('Maximum push size') + %td= instance_configuration_human_size_cell(size_limits[:receive_max_input_size]) + %tr + %td= _('Maximum import size') + %td= instance_configuration_human_size_cell(size_limits[:max_import_size]) + %tr + %td= _('Maximum diff patch size') + %td= instance_configuration_human_size_cell(size_limits[:diff_max_patch_bytes]) + %tr + %td= _('Maximum job artifact size') + %td= instance_configuration_human_size_cell(size_limits[:max_artifacts_size]) + %tr + %td= _('Maximum page size') + %td= instance_configuration_human_size_cell(size_limits[:max_pages_size]) + %tr + %td= _('Maximum snippet size') + %td= instance_configuration_human_size_cell(size_limits[:snippet_size_limit]) diff --git a/db/ci_migrate b/db/ci_migrate deleted file mode 120000 index 1f0710ccbe7..00000000000 --- a/db/ci_migrate +++ /dev/null @@ -1 +0,0 @@ -migrate \ No newline at end of file diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 3543b5493bd..2549c35ebd6 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -16,6 +16,7 @@ module Gitlab PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit allow_failure when needs resource_group].freeze + MAX_NESTING_LEVEL = 10 included do validations do @@ -31,7 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true - validates :rules, nested_array_of_hashes: true + validates :rules, nested_array_of_hashes_or_arrays: { max_level: MAX_NESTING_LEVEL } validates :resource_group, type: String end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 13c6eaf4993..5d2bf3cfebf 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -90,14 +90,22 @@ module Gitlab end end - class NestedArrayOfHashesValidator < ArrayOfHashesValidator + class NestedArrayOfHashesOrArraysValidator < ArrayOfHashesValidator include NestedArrayHelpers def validate_each(record, attribute, value) - unless validate_nested_array(value, 1, &method(:validate_array_of_hashes)) + max_level = options.fetch(:max_level, 1) + + unless validate_nested_array(value, max_level, &method(:validate_hash)) record.errors.add(attribute, 'should be an array containing hashes and arrays of hashes') end end + + private + + def validate_hash(value) + value.is_a?(Hash) + end end class ArrayOrStringValidator < ActiveModel::EachValidator diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 690790aa5da..886e476df0b 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4531,9 +4531,6 @@ msgstr "" msgid "Artifacts" msgstr "" -msgid "Artifacts maximum size" -msgstr "" - msgid "As we continue to build more features for SAST, we'd love your feedback on the SAST configuration feature in %{linkStart}this issue%{linkEnd}." msgstr "" @@ -5138,9 +5135,6 @@ msgstr "" msgid "Begin with the selected commit" msgstr "" -msgid "Below are the current settings regarding" -msgstr "" - msgid "Below are the fingerprints for the current instance SSH host keys." msgstr "" @@ -10660,9 +10654,6 @@ msgstr "" msgid "Decrease" msgstr "" -msgid "Default" -msgstr "" - msgid "Default CI/CD configuration file" msgstr "" @@ -15233,9 +15224,6 @@ msgstr "" msgid "GitLab Billing Team." msgstr "" -msgid "GitLab CI" -msgstr "" - msgid "GitLab Import" msgstr "" @@ -20622,6 +20610,9 @@ msgstr "" msgid "Maximum artifacts size (MB)" msgstr "" +msgid "Maximum attachment size" +msgstr "" + msgid "Maximum attachment size (MB)" msgstr "" @@ -20640,6 +20631,9 @@ msgstr "" msgid "Maximum delay (Minutes)" msgstr "" +msgid "Maximum diff patch size" +msgstr "" + msgid "Maximum diff patch size (Bytes)" msgstr "" @@ -20667,9 +20661,15 @@ msgstr "" msgid "Maximum files in a diff" msgstr "" +msgid "Maximum import size" +msgstr "" + msgid "Maximum import size (MB)" msgstr "" +msgid "Maximum job artifact size" +msgstr "" + msgid "Maximum job timeout" msgstr "" @@ -20703,6 +20703,12 @@ msgstr "" msgid "Maximum page reached" msgstr "" +msgid "Maximum page size" +msgstr "" + +msgid "Maximum push size" +msgstr "" + msgid "Maximum push size (MB)" msgstr "" @@ -20727,6 +20733,9 @@ msgstr "" msgid "Maximum size of pages (MB)" msgstr "" +msgid "Maximum snippet size" +msgstr "" + msgid "Maximum time between updates that a mirror can have when scheduled to synchronize." msgstr "" @@ -30949,6 +30958,9 @@ msgstr "" msgid "Size" msgstr "" +msgid "Size Limits" +msgstr "" + msgid "Size limit per repository (MB)" msgstr "" @@ -33337,9 +33349,6 @@ msgstr "" msgid "The maximum number of tags that a single worker accepts for cleanup. If the number of tags goes above this limit, the list of tags to delete is truncated to this number. To remove this limit, set it to 0." msgstr "" -msgid "The maximum size of your Pages site is regulated by the artifacts maximum size which is part of %{link_to_gitlab_ci}." -msgstr "" - msgid "The merge conflicts for this merge request cannot be resolved through GitLab. Please try to resolve them locally." msgstr "" @@ -33637,6 +33646,9 @@ msgstr "" msgid "There are several rate limits in place to protect the system." msgstr "" +msgid "There are several size limits in place." +msgstr "" + msgid "There is a halted Elasticsearch migration" msgstr "" diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 4a90e765d4b..0bb26babfc0 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -169,6 +169,22 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do it { expect(entry).to be_valid } end end + + context 'when rules are used' do + let(:config) { { script: 'ls', cache: { key: 'test' }, rules: rules } } + + let(:rules) do + [ + { if: '$CI_PIPELINE_SOURCE == "schedule"', when: 'never' }, + [ + { if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' }, + { if: '$CI_PIPELINE_SOURCE == "merge_request_event"' } + ] + ] + end + + it { expect(entry).to be_valid } + end end context 'when entry value is not correct' do @@ -485,6 +501,70 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do end end end + + context 'when invalid rules are used' do + let(:config) { { script: 'ls', cache: { key: 'test' }, rules: rules } } + + context 'with rules nested more than max allowed levels' do + let(:sample_rule) { { if: '$THIS == "other"', when: 'always' } } + + let(:rules) do + [ + { if: '$THIS == "that"', when: 'always' }, + [ + { if: '$SKIP', when: 'never' }, + [ + sample_rule, + [ + sample_rule, + [ + sample_rule, + [ + sample_rule, + [ + sample_rule, + [ + sample_rule, + [ + sample_rule, + [ + sample_rule, + [ + sample_rule, + [ + sample_rule, + [sample_rule] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + end + + it { expect(entry).not_to be_valid } + end + + context 'with rules with invalid keys' do + let(:rules) do + [ + { invalid_key: 'invalid' }, + [ + { if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' }, + { if: '$CI_PIPELINE_SOURCE == "merge_request_event"' } + ] + ] + end + + it { expect(entry).not_to be_valid } + end + end end end diff --git a/spec/lib/gitlab/ci/config/entry/rules_spec.rb b/spec/lib/gitlab/ci/config/entry/rules_spec.rb index 91252378541..cfec33003e4 100644 --- a/spec/lib/gitlab/ci/config/entry/rules_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/rules_spec.rb @@ -53,7 +53,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules do let(:config) do [ { if: '$THIS == "that"', when: 'always' }, - [{ if: '$SKIP', when: 'never' }] + [{ if: '$SKIP', when: 'never' }, { if: '$THIS == "other"', when: 'always' }] ] end @@ -64,11 +64,11 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules do let(:config) do [ { if: '$THIS == "that"', when: 'always' }, - [{ if: '$SKIP', when: 'never' }, [{ if: '$THIS == "other"', when: 'aways' }]] + [{ if: '$SKIP', when: 'never' }, [{ if: '$THIS == "other"', when: 'always' }]] ] end - it { is_expected.not_to be_valid } + it { is_expected.to be_valid } end end @@ -119,7 +119,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules do context 'with rules nested more than one level' do let(:first_rule) { { if: '$THIS == "that"', when: 'always' } } let(:second_rule) { { if: '$SKIP', when: 'never' } } - let(:third_rule) { { if: '$THIS == "other"', when: 'aways' } } + let(:third_rule) { { if: '$THIS == "other"', when: 'always' } } let(:config) do [ diff --git a/spec/models/instance_configuration_spec.rb b/spec/models/instance_configuration_spec.rb index 9544f0fe6ec..551e6e7572c 100644 --- a/spec/models/instance_configuration_spec.rb +++ b/spec/models/instance_configuration_spec.rb @@ -76,24 +76,46 @@ RSpec.describe InstanceConfiguration do end end - describe '#gitlab_ci' do - let(:gitlab_ci) { subject.settings[:gitlab_ci] } - - it 'returns Settings.gitalb_ci' do - gitlab_ci.delete(:artifacts_max_size) - - expect(gitlab_ci).to eq(Settings.gitlab_ci.symbolize_keys) + describe '#size_limits' do + before do + Gitlab::CurrentSettings.current_application_settings.update!( + max_attachment_size: 10, + receive_max_input_size: 20, + max_import_size: 30, + diff_max_patch_bytes: 409600, + max_artifacts_size: 50, + max_pages_size: 60, + snippet_size_limit: 70 + ) end - it 'returns the key artifacts_max_size' do - expect(gitlab_ci.keys).to include(:artifacts_max_size) + it 'returns size limits from application settings' do + size_limits = subject.settings[:size_limits] + + expect(size_limits[:max_attachment_size]).to eq(10.megabytes) + expect(size_limits[:receive_max_input_size]).to eq(20.megabytes) + expect(size_limits[:max_import_size]).to eq(30.megabytes) + expect(size_limits[:diff_max_patch_bytes]).to eq(400.kilobytes) + expect(size_limits[:max_artifacts_size]).to eq(50.megabytes) + expect(size_limits[:max_pages_size]).to eq(60.megabytes) + expect(size_limits[:snippet_size_limit]).to eq(70.bytes) end - it 'returns the key artifacts_max_size with values' do - stub_application_setting(max_artifacts_size: 200) + it 'returns nil if receive_max_input_size not set' do + Gitlab::CurrentSettings.current_application_settings.update!(receive_max_input_size: nil) - expect(gitlab_ci[:artifacts_max_size][:default]).to eq(100.megabytes) - expect(gitlab_ci[:artifacts_max_size][:value]).to eq(200.megabytes) + size_limits = subject.settings[:size_limits] + + expect(size_limits[:receive_max_input_size]).to be_nil + end + + it 'returns nil if set to 0 (unlimited)' do + Gitlab::CurrentSettings.current_application_settings.update!(max_import_size: 0, max_pages_size: 0) + + size_limits = subject.settings[:size_limits] + + expect(size_limits[:max_import_size]).to be_nil + expect(size_limits[:max_pages_size]).to be_nil end end diff --git a/spec/views/help/instance_configuration.html.haml_spec.rb b/spec/views/help/instance_configuration.html.haml_spec.rb index 7b431bb4180..c4542046a9d 100644 --- a/spec/views/help/instance_configuration.html.haml_spec.rb +++ b/spec/views/help/instance_configuration.html.haml_spec.rb @@ -9,6 +9,7 @@ RSpec.describe 'help/instance_configuration' do let(:ssh_settings) { settings[:ssh_algorithms_hashes] } before do + create(:plan, name: 'plan1', title: 'Plan 1') assign(:instance_configuration, instance_configuration) end @@ -17,7 +18,9 @@ RSpec.describe 'help/instance_configuration' do expect(rendered).to have_link(nil, href: '#ssh-host-keys-fingerprints') if ssh_settings.any? expect(rendered).to have_link(nil, href: '#gitlab-pages') - expect(rendered).to have_link(nil, href: '#gitlab-ci') + expect(rendered).to have_link(nil, href: '#size-limits') + expect(rendered).to have_link(nil, href: '#package-registry') + expect(rendered).to have_link(nil, href: '#rate-limits') end it 'has several sections' do @@ -25,7 +28,9 @@ RSpec.describe 'help/instance_configuration' do expect(rendered).to have_css('h2#ssh-host-keys-fingerprints') if ssh_settings.any? expect(rendered).to have_css('h2#gitlab-pages') - expect(rendered).to have_css('h2#gitlab-ci') + expect(rendered).to have_css('h2#size-limits') + expect(rendered).to have_css('h2#package-registry') + expect(rendered).to have_css('h2#rate-limits') end end end