From 3174adc79907c808088a4ede8b891facb9142025 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 26 May 2020 00:08:21 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- app/assets/stylesheets/framework/common.scss | 1 - .../admin/application_settings_controller.rb | 1 + app/models/application_setting.rb | 18 +++++++ .../application_setting_implementation.rb | 11 +++++ .../jc-weighted-repository-storages.yml | 5 ++ ...orages_weighted_to_application_settings.rb | 13 +++++ db/structure.sql | 2 + doc/administration/raketasks/maintenance.md | 14 ++++++ doc/user/application_security/dast/index.md | 49 +++++++++++-------- lib/api/settings.rb | 1 + .../application_settings_controller_spec.rb | 7 +++ spec/models/application_setting_spec.rb | 21 ++++++++ 12 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 changelogs/unreleased/jc-weighted-repository-storages.yml create mode 100644 db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb diff --git a/app/assets/stylesheets/framework/common.scss b/app/assets/stylesheets/framework/common.scss index 991fc9f2645..aed5086e6ba 100644 --- a/app/assets/stylesheets/framework/common.scss +++ b/app/assets/stylesheets/framework/common.scss @@ -405,7 +405,6 @@ img.emoji { .prepend-top-16 { margin-top: 16px; } .prepend-top-20 { margin-top: 20px; } .prepend-top-32 { margin-top: 32px; } -.prepend-left-2 { margin-left: 2px; } .prepend-left-4 { margin-left: 4px; } .prepend-left-5 { margin-left: 5px; } .prepend-left-8 { margin-left: 8px; } diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index 355662bbb38..11e917ae94a 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -214,6 +214,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController [ *::ApplicationSettingsHelper.visible_attributes, *::ApplicationSettingsHelper.external_authorization_service_attributes, + *ApplicationSetting.repository_storages_weighted_attributes, :lets_encrypt_notification_email, :lets_encrypt_terms_of_service_accepted, :domain_blacklist_file, diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index e88d9ba4e63..fc1af49e3b0 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -19,6 +19,12 @@ class ApplicationSetting < ApplicationRecord belongs_to :instance_administrators_group, class_name: "Group" + def self.repository_storages_weighted_attributes + @repository_storages_weighted_atributes ||= Gitlab.config.repositories.storages.keys.map { |k| "repository_storages_weighted_#{k}".to_sym }.freeze + end + + store_accessor :repository_storages_weighted, *Gitlab.config.repositories.storages.keys, prefix: true + # Include here so it can override methods from # `add_authentication_token_field` # We don't prepend for now because otherwise we'll need to @@ -39,6 +45,7 @@ class ApplicationSetting < ApplicationRecord cache_markdown_field :after_sign_up_text default_value_for :id, 1 + default_value_for :repository_storages_weighted, {} chronic_duration_attr_writer :archive_builds_in_human_readable, :archive_builds_in_seconds @@ -152,6 +159,7 @@ class ApplicationSetting < ApplicationRecord validates :repository_storages, presence: true validate :check_repository_storages + validate :check_repository_storages_weighted validates :auto_devops_domain, allow_blank: true, @@ -271,6 +279,10 @@ class ApplicationSetting < ApplicationRecord validates :allowed_key_types, presence: true + repository_storages_weighted_attributes.each do |attribute| + validates attribute, allow_nil: true, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 100 } + end + validates_each :restricted_visibility_levels do |record, attr, value| value&.each do |level| unless Gitlab::VisibilityLevel.options.value?(level) @@ -434,6 +446,12 @@ class ApplicationSetting < ApplicationRecord recaptcha_enabled || login_recaptcha_protection_enabled end + repository_storages_weighted_attributes.each do |attribute| + define_method :"#{attribute}=" do |value| + super(value.to_i) + end + end + private def parsed_grafana_url diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 69292fd11fb..40cc941c1b6 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -104,6 +104,7 @@ module ApplicationSettingImplementation login_recaptcha_protection_enabled: false, repository_checks_enabled: true, repository_storages: ['default'], + repository_storages_weighted: { default: 100 }, require_two_factor_authentication: false, restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], session_expire_delay: Settings.gitlab['session_expire_delay'], @@ -262,6 +263,10 @@ module ApplicationSettingImplementation Array(read_attribute(:repository_storages)) end + def repository_storages_weighted + read_attribute(:repository_storages_weighted) + end + def commit_email_hostname super.presence || self.class.default_commit_email_hostname end @@ -422,6 +427,12 @@ module ApplicationSettingImplementation invalid.empty? end + def check_repository_storages_weighted + invalid = repository_storages_weighted.keys - Gitlab.config.repositories.storages.keys + errors.add(:repository_storages_weighted, "can't include: %{invalid_storages}" % { invalid_storages: invalid.join(", ") }) unless + invalid.empty? + end + def terms_exist return unless enforce_terms? diff --git a/changelogs/unreleased/jc-weighted-repository-storages.yml b/changelogs/unreleased/jc-weighted-repository-storages.yml new file mode 100644 index 00000000000..060c1a26494 --- /dev/null +++ b/changelogs/unreleased/jc-weighted-repository-storages.yml @@ -0,0 +1,5 @@ +--- +title: Save repository storages in application settings with weights +merge_request: 31645 +author: +type: added diff --git a/db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb b/db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb new file mode 100644 index 00000000000..b9d4f65989a --- /dev/null +++ b/db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddRepositoryStoragesWeightedToApplicationSettings < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + add_column :application_settings, :repository_storages_weighted, :jsonb, default: {}, null: false + end + + def down + remove_column :application_settings, :repository_storages_weighted + end +end diff --git a/db/structure.sql b/db/structure.sql index 3930ee67db0..d35001d7f26 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -440,6 +440,7 @@ CREATE TABLE public.application_settings ( group_owners_can_manage_default_branch_protection boolean DEFAULT true NOT NULL, container_registry_vendor text DEFAULT ''::text NOT NULL, container_registry_version text DEFAULT ''::text NOT NULL, + repository_storages_weighted jsonb DEFAULT '{}'::jsonb NOT NULL, container_registry_features text[] DEFAULT '{}'::text[] NOT NULL, spam_check_endpoint_url text, spam_check_endpoint_enabled boolean DEFAULT false NOT NULL, @@ -13888,6 +13889,7 @@ COPY "schema_migrations" (version) FROM STDIN; 20200508050301 20200508091106 20200508140959 +20200508203901 20200511080113 20200511083541 20200511092246 diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index eee68c0da0a..23186b17faa 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -265,6 +265,20 @@ database: gitlabhq_production up migration_id migration_name ``` +## Run incomplete database migrations + +Database migrations can be stuck in an incomplete state. That is, they'll have a `down` +status in the output of the `sudo gitlab-rake db:migrate:status` command. + +To complete these migrations, use the following Rake task: + +```shell +sudo gitlab-rake db:migrate +``` + +After the command completes, run `sudo gitlab-rake db:migrate:status` to check if all +migrations are completed (have an `up` status). + ## Import common metrics Sometimes you may need to re-import the common metrics that power the Metrics dashboards. diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index e057f03f39a..3ca88acf35a 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -442,7 +442,7 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia | Environment variable | Required | Description | |-----------------------------| -----------|--------------------------------------------------------------------------------| -| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address from which to download the analyzer. | +| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address from which to download the analyzer. | | `DAST_WEBSITE` | no| The URL of the website to scan. `DAST_API_SPECIFICATION` must be specified if this is omitted. | | `DAST_API_SPECIFICATION` | no | The API specification to import. `DAST_WEBSITE` must be specified if this is omitted. | | `DAST_AUTH_URL` | no | The authentication URL of the website to scan. Not supported for API scans. | @@ -458,7 +458,17 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia | `DAST_API_HOST_OVERRIDE` | no | Used to override domains defined in API specification files. | | `DAST_EXCLUDE_RULES` | no | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from the scan report. Currently, excluded rules will get executed but the alerts from them will be suppressed. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. | | `DAST_REQUEST_HEADERS` | no | Set to a comma-separated list of request header names and values. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | -| `DAST_ZAP_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_DEBUG` | no | Enable debug message output. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_SPIDER_MINS` | no | The maximum duration of the spider scan in minutes. Set to zero for unlimited. Defaults to one minute, or unlimited when the scan is a full scan. | +| `DAST_HTML_REPORT` | no | The file name of the HTML report written at the end of a scan. | +| `DAST_MARKDOWN_REPORT` | no | The file name of the Markdown report written at the end of a scan. | +| `DAST_XML_REPORT` | no | The file name of the XML report written at the end of a scan. | +| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | no | Include alpha passive and active scan rules. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_ZAP_CLI_OPTIONS` | no | ZAP Server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. | +| `DAST_ZAP_GENERATE_CONFIG` | no | Generate sample ZAP config file for use with `DAST_ZAP_CONFIG_FILE`. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_ZAP_CONFIG_FILE` | no | Name of config file used to determine thresholds of vulnerability rules. | +| `DAST_ZAP_CONFIG_URL` | no | URL of config file used to determine thresholds of vulnerability rules. | ### DAST command-line options @@ -475,8 +485,9 @@ dast: - /analyze --help ``` -You must then overwrite the `script` command to pass in the appropriate argument. -For example, debug messages can be enabled by using `-d`, as shown in the following configuration: +You must then overwrite the `script` command to pass in the appropriate +argument. For example, passive scanning can be delayed using option `-D`. The following +configuration delays passive scanning by five minutes: ```yaml include: @@ -485,7 +496,7 @@ include: dast: script: - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -d -t $DAST_WEBSITE + - /analyze -D 300 -t $DAST_WEBSITE ``` ### Custom ZAProxy configuration @@ -500,10 +511,8 @@ when used. An example of how to rewrite the Authorization header value with `TOK include: template: DAST.gitlab-ci.yml -dast: - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -z"-config replacer.full_list\(0\).description=auth -config replacer.full_list\(0\).enabled=true -config replacer.full_list\(0\).matchtype=REQ_HEADER -config replacer.full_list\(0\).matchstr=Authorization -config replacer.full_list\(0\).regex=false -config replacer.full_list\(0\).replacement=TOKEN" -t $DAST_WEBSITE +variables: + DAST_ZAP_CLI_OPTIONS: "-config replacer.full_list(0).description=auth -config replacer.full_list(0).enabled=true -config replacer.full_list(0).matchtype=REQ_HEADER -config replacer.full_list(0).matchstr=Authorization -config replacer.full_list(0).regex=false -config replacer.full_list(0).replacement=TOKEN" ``` ### Cloning the project's repository @@ -625,18 +634,18 @@ vulnerabilities in your groups, projects and pipelines. Read more about the ## Bleeding-edge vulnerability definitions -ZAProxy first creates rules in the `alpha` class. After a testing period with the -community, they are promoted to `beta`. DAST uses `beta` definitions by default. -To request `alpha` definitions, use `-a` as shown in the following configuration: +ZAP first creates rules in the `alpha` class. After a testing period with +the community, they are promoted to `beta`. DAST uses `beta` definitions by +default. To request `alpha` definitions, use the +`DAST_INCLUDE_ALPHA_VULNERABILITIES` environment variable as shown in the +following configuration: ```yaml include: template: DAST.gitlab-ci.yml -dast: - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -a -t $DAST_WEBSITE +variables: + DAST_INCLUDE_ALPHA_VULNERABILITIES: true ``` ## Interacting with the vulnerabilities @@ -688,16 +697,14 @@ This results in the following error: ``` Fortunately, it's straightforward to increase the amount of memory available -for DAST by overwriting the `script` key in the DAST template: +for DAST by using the `DAST_ZAP_CLI_OPTIONS` environment variable: ```yaml include: - template: DAST.gitlab-ci.yml -dast: - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -t $DAST_WEBSITE -z"-Xmx3072m" +variables: + DAST_ZAP_CLI_OPTIONS: "-Xmx3072m" ``` Here, DAST is being allocated 3072 MB. diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 0056927fec9..6687242a604 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -113,6 +113,7 @@ module API end optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' + optional :repository_storages_weighted, type: Hash, desc: 'Storage paths for new projects with a weighted value between 0 and 100' optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' diff --git a/spec/controllers/admin/application_settings_controller_spec.rb b/spec/controllers/admin/application_settings_controller_spec.rb index 33764818f79..938a30115b0 100644 --- a/spec/controllers/admin/application_settings_controller_spec.rb +++ b/spec/controllers/admin/application_settings_controller_spec.rb @@ -120,6 +120,13 @@ describe Admin::ApplicationSettingsController do expect(ApplicationSetting.current.namespace_storage_size_limit).not_to eq(-100) end + it 'updates repository_storages_weighted setting' do + put :update, params: { application_setting: { repository_storages_weighted_default: 75 } } + + expect(response).to redirect_to(general_admin_application_settings_path) + expect(ApplicationSetting.current.repository_storages_weighted_default).to eq(75) + end + context 'external policy classification settings' do let(:settings) do { diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 8d963d41fce..a26fb33acee 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -105,6 +105,14 @@ describe ApplicationSetting do it { is_expected.not_to allow_value(false).for(:hashed_storage_enabled) } + it { is_expected.not_to allow_value(101).for(:repository_storages_weighted_default) } + it { is_expected.not_to allow_value(-1).for(:repository_storages_weighted_default) } + it { is_expected.to allow_value(100).for(:repository_storages_weighted_default) } + it { is_expected.to allow_value(0).for(:repository_storages_weighted_default) } + it { is_expected.to allow_value(50).for(:repository_storages_weighted_default) } + it { is_expected.to allow_value(nil).for(:repository_storages_weighted_default) } + it { is_expected.not_to allow_value({ default: 100, shouldntexist: 50 }).for(:repository_storages_weighted) } + context 'grafana_url validations' do before do subject.instance_variable_set(:@parsed_grafana_url, nil) @@ -786,4 +794,17 @@ describe ApplicationSetting do end it_behaves_like 'application settings examples' + + describe 'repository_storages_weighted_attributes' do + it 'returns the keys for repository_storages_weighted' do + expect(subject.class.repository_storages_weighted_attributes).to eq([:repository_storages_weighted_default]) + end + end + + it 'does not allow to set weight for non existing storage' do + setting.repository_storages_weighted = { invalid_storage: 100 } + + expect(setting).not_to be_valid + expect(setting.errors.messages[:repository_storages_weighted]).to match_array(["can't include: invalid_storage"]) + end end