Add latest changes from gitlab-org/gitlab@master

app/assets/stylesheets/framework/common.scss

@@ -405,7 +405,6 @@ img.emoji {
 .prepend-top-16 { margin-top: 16px; }
 .prepend-top-20 { margin-top: 20px; }
 .prepend-top-32 { margin-top: 32px; }
-.prepend-left-2 { margin-left: 2px; }
 .prepend-left-4 { margin-left: 4px; }
 .prepend-left-5 { margin-left: 5px; }
 .prepend-left-8 { margin-left: 8px; }

app/controllers/admin/application_settings_controller.rb

@@ -214,6 +214,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
     [
       *::ApplicationSettingsHelper.visible_attributes,
       *::ApplicationSettingsHelper.external_authorization_service_attributes,
+      *ApplicationSetting.repository_storages_weighted_attributes,
       :lets_encrypt_notification_email,
       :lets_encrypt_terms_of_service_accepted,
       :domain_blacklist_file,

app/models/application_setting.rb

@@ -19,6 +19,12 @@ class ApplicationSetting < ApplicationRecord
 
   belongs_to :instance_administrators_group, class_name: "Group"
 
+  def self.repository_storages_weighted_attributes
+    @repository_storages_weighted_atributes ||= Gitlab.config.repositories.storages.keys.map { |k| "repository_storages_weighted_#{k}".to_sym }.freeze
+  end
+
+  store_accessor :repository_storages_weighted, *Gitlab.config.repositories.storages.keys, prefix: true
+
   # Include here so it can override methods from
   # `add_authentication_token_field`
   # We don't prepend for now because otherwise we'll need to
@@ -39,6 +45,7 @@ class ApplicationSetting < ApplicationRecord
   cache_markdown_field :after_sign_up_text
 
   default_value_for :id, 1
+  default_value_for :repository_storages_weighted, {}
 
   chronic_duration_attr_writer :archive_builds_in_human_readable, :archive_builds_in_seconds
 
@@ -152,6 +159,7 @@ class ApplicationSetting < ApplicationRecord
 
   validates :repository_storages, presence: true
   validate :check_repository_storages
+  validate :check_repository_storages_weighted
 
   validates :auto_devops_domain,
             allow_blank: true,
@@ -271,6 +279,10 @@ class ApplicationSetting < ApplicationRecord
 
   validates :allowed_key_types, presence: true
 
+  repository_storages_weighted_attributes.each do |attribute|
+    validates attribute, allow_nil: true, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 100 }
+  end
+
   validates_each :restricted_visibility_levels do |record, attr, value|
     value&.each do |level|
       unless Gitlab::VisibilityLevel.options.value?(level)
@@ -434,6 +446,12 @@ class ApplicationSetting < ApplicationRecord
     recaptcha_enabled || login_recaptcha_protection_enabled
   end
 
+  repository_storages_weighted_attributes.each do |attribute|
+    define_method :"#{attribute}=" do |value|
+      super(value.to_i)
+    end
+  end
+
   private
 
   def parsed_grafana_url

app/models/application_setting_implementation.rb

@@ -104,6 +104,7 @@ module ApplicationSettingImplementation
         login_recaptcha_protection_enabled: false,
         repository_checks_enabled: true,
         repository_storages: ['default'],
+        repository_storages_weighted: { default: 100 },
         require_two_factor_authentication: false,
         restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],
         session_expire_delay: Settings.gitlab['session_expire_delay'],
@@ -262,6 +263,10 @@ module ApplicationSettingImplementation
     Array(read_attribute(:repository_storages))
   end
 
+  def repository_storages_weighted
+    read_attribute(:repository_storages_weighted)
+  end
+
   def commit_email_hostname
     super.presence || self.class.default_commit_email_hostname
   end
@@ -422,6 +427,12 @@ module ApplicationSettingImplementation
       invalid.empty?
   end
 
+  def check_repository_storages_weighted
+    invalid = repository_storages_weighted.keys - Gitlab.config.repositories.storages.keys
+    errors.add(:repository_storages_weighted, "can't include: %{invalid_storages}" % { invalid_storages: invalid.join(", ") }) unless
+      invalid.empty?
+  end
+
   def terms_exist
     return unless enforce_terms?
 

changelogs/unreleased/jc-weighted-repository-storages.yml

@@ -0,0 +1,5 @@
+---
+title: Save repository storages in application settings with weights
+merge_request: 31645
+author:
+type: added

db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddRepositoryStoragesWeightedToApplicationSettings < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    add_column :application_settings, :repository_storages_weighted, :jsonb, default: {}, null: false
+  end
+
+  def down
+    remove_column :application_settings, :repository_storages_weighted
+  end
+end

db/structure.sql

@@ -440,6 +440,7 @@ CREATE TABLE public.application_settings (
     group_owners_can_manage_default_branch_protection boolean DEFAULT true NOT NULL,
     container_registry_vendor text DEFAULT ''::text NOT NULL,
     container_registry_version text DEFAULT ''::text NOT NULL,
+    repository_storages_weighted jsonb DEFAULT '{}'::jsonb NOT NULL,
     container_registry_features text[] DEFAULT '{}'::text[] NOT NULL,
     spam_check_endpoint_url text,
     spam_check_endpoint_enabled boolean DEFAULT false NOT NULL,
@@ -13888,6 +13889,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200508050301
 20200508091106
 20200508140959
+20200508203901
 20200511080113
 20200511083541
 20200511092246

doc/administration/raketasks/maintenance.md

@@ -265,6 +265,20 @@ database: gitlabhq_production
    up     migration_id    migration_name
 ```
 
+## Run incomplete database migrations
+
+Database migrations can be stuck in an incomplete state. That is, they'll have a `down`
+status in the output of the `sudo gitlab-rake db:migrate:status` command.
+
+To complete these migrations, use the following Rake task:
+
+```shell
+sudo gitlab-rake db:migrate
+```
+
+After the command completes, run `sudo gitlab-rake db:migrate:status` to check if all
+migrations are completed (have an `up` status).
+
 ## Import common metrics
 
 Sometimes you may need to re-import the common metrics that power the Metrics dashboards.

doc/user/application_security/dast/index.md

@@ -442,7 +442,7 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia
 
 | Environment variable        | Required   | Description                                                                    |
 |-----------------------------| -----------|--------------------------------------------------------------------------------|
-| `SECURE_ANALYZERS_PREFIX`   | no         | Set the Docker registry base address from which to download the analyzer.            |
+| `SECURE_ANALYZERS_PREFIX`   | no | Set the Docker registry base address from which to download the analyzer. |
 | `DAST_WEBSITE`  | no| The URL of the website to scan. `DAST_API_SPECIFICATION` must be specified if this is omitted. |
 | `DAST_API_SPECIFICATION`  | no | The API specification to import. `DAST_WEBSITE` must be specified if this is omitted. |
 | `DAST_AUTH_URL` | no | The authentication URL of the website to scan. Not supported for API scans. |
@@ -458,7 +458,17 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia
 | `DAST_API_HOST_OVERRIDE` | no | Used to override domains defined in API specification files. |
 | `DAST_EXCLUDE_RULES` | no | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from the scan report. Currently, excluded rules will get executed but the alerts from them will be suppressed. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. |
 | `DAST_REQUEST_HEADERS` | no | Set to a comma-separated list of request header names and values. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` |
-| `DAST_ZAP_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
+| `DAST_DEBUG` | no | Enable debug message output. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
+| `DAST_SPIDER_MINS` | no | The maximum duration of the spider scan in minutes. Set to zero for unlimited. Defaults to one minute, or unlimited when the scan is a full scan. |
+| `DAST_HTML_REPORT` | no | The file name of the HTML report written at the end of a scan. |
+| `DAST_MARKDOWN_REPORT` | no | The file name of the Markdown report written at the end of a scan. |
+| `DAST_XML_REPORT` | no | The file name of the XML report written at the end of a scan. |
+| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | no | Include alpha passive and active scan rules. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
+| `DAST_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
+| `DAST_ZAP_CLI_OPTIONS` | no | ZAP Server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. |
+| `DAST_ZAP_GENERATE_CONFIG` | no | Generate sample ZAP config file for use with `DAST_ZAP_CONFIG_FILE`. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
+| `DAST_ZAP_CONFIG_FILE` | no | Name of config file used to determine thresholds of vulnerability rules. |
+| `DAST_ZAP_CONFIG_URL` | no | URL of config file used to determine thresholds of vulnerability rules. |
 
 ### DAST command-line options
 
@@ -475,8 +485,9 @@ dast:
     - /analyze --help
 ```
 
-You must then overwrite the `script` command to pass in the appropriate argument.
-For example, debug messages can be enabled by using `-d`, as shown in the following configuration:
+You must then overwrite the `script` command to pass in the appropriate
+argument. For example, passive scanning can be delayed using option `-D`. The following
+configuration delays passive scanning by five minutes:
 
 ```yaml
 include:
@@ -485,7 +496,7 @@ include:
 dast:
   script:
     - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
-    - /analyze -d -t $DAST_WEBSITE
+    - /analyze -D 300 -t $DAST_WEBSITE
 ```
 
 ### Custom ZAProxy configuration
@@ -500,10 +511,8 @@ when used. An example of how to rewrite the Authorization header value with `TOK
 include:
   template: DAST.gitlab-ci.yml
 
-dast:
-  script:
-    - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
-    - /analyze -z"-config replacer.full_list\(0\).description=auth -config replacer.full_list\(0\).enabled=true -config replacer.full_list\(0\).matchtype=REQ_HEADER -config replacer.full_list\(0\).matchstr=Authorization -config replacer.full_list\(0\).regex=false -config replacer.full_list\(0\).replacement=TOKEN" -t $DAST_WEBSITE
+variables:
+  DAST_ZAP_CLI_OPTIONS: "-config replacer.full_list(0).description=auth -config replacer.full_list(0).enabled=true -config replacer.full_list(0).matchtype=REQ_HEADER -config replacer.full_list(0).matchstr=Authorization -config replacer.full_list(0).regex=false -config replacer.full_list(0).replacement=TOKEN"
 ```
 
 ### Cloning the project's repository
@@ -625,18 +634,18 @@ vulnerabilities in your groups, projects and pipelines. Read more about the
 
 ## Bleeding-edge vulnerability definitions
 
-ZAProxy first creates rules in the `alpha` class. After a testing period with the
-community, they are promoted to `beta`. DAST uses `beta` definitions by default.
-To request `alpha` definitions, use `-a` as shown in the following configuration:
+ZAP first creates rules in the `alpha` class. After a testing period with
+the community, they are promoted to `beta`. DAST uses `beta` definitions by
+default. To request `alpha` definitions, use the
+`DAST_INCLUDE_ALPHA_VULNERABILITIES` environment variable as shown in the
+following configuration:
 
 ```yaml
 include:
   template: DAST.gitlab-ci.yml
 
-dast:
-  script:
-    - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
-    - /analyze -a -t $DAST_WEBSITE
+variables:
+  DAST_INCLUDE_ALPHA_VULNERABILITIES: true
 ```
 
 ## Interacting with the vulnerabilities
@@ -688,16 +697,14 @@ This results in the following error:
 ```
 
 Fortunately, it's straightforward to increase the amount of memory available
-for DAST by overwriting the `script` key in the DAST template:
+for DAST by using the `DAST_ZAP_CLI_OPTIONS` environment variable:
 
 ```yaml
 include:
   - template: DAST.gitlab-ci.yml
 
-dast:
-  script:
-    - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
-    - /analyze -t $DAST_WEBSITE -z"-Xmx3072m"
+variables:
+  DAST_ZAP_CLI_OPTIONS: "-Xmx3072m"
 ```
 
 Here, DAST is being allocated 3072 MB.

lib/api/settings.rb

@@ -113,6 +113,7 @@ module API
       end
       optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
       optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects'
+      optional :repository_storages_weighted, type: Hash, desc: 'Storage paths for new projects with a weighted value between 0 and 100'
       optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication'
       given require_two_factor_authentication: ->(val) { val } do
         requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'

spec/controllers/admin/application_settings_controller_spec.rb

@@ -120,6 +120,13 @@ describe Admin::ApplicationSettingsController do
       expect(ApplicationSetting.current.namespace_storage_size_limit).not_to eq(-100)
     end
 
+    it 'updates repository_storages_weighted setting' do
+      put :update, params: { application_setting: { repository_storages_weighted_default: 75 } }
+
+      expect(response).to redirect_to(general_admin_application_settings_path)
+      expect(ApplicationSetting.current.repository_storages_weighted_default).to eq(75)
+    end
+
     context 'external policy classification settings' do
       let(:settings) do
         {

spec/models/application_setting_spec.rb

@@ -105,6 +105,14 @@ describe ApplicationSetting do
 
     it { is_expected.not_to allow_value(false).for(:hashed_storage_enabled) }
 
+    it { is_expected.not_to allow_value(101).for(:repository_storages_weighted_default) }
+    it { is_expected.not_to allow_value(-1).for(:repository_storages_weighted_default) }
+    it { is_expected.to allow_value(100).for(:repository_storages_weighted_default) }
+    it { is_expected.to allow_value(0).for(:repository_storages_weighted_default) }
+    it { is_expected.to allow_value(50).for(:repository_storages_weighted_default) }
+    it { is_expected.to allow_value(nil).for(:repository_storages_weighted_default) }
+    it { is_expected.not_to allow_value({ default: 100, shouldntexist: 50 }).for(:repository_storages_weighted) }
+
     context 'grafana_url validations' do
       before do
         subject.instance_variable_set(:@parsed_grafana_url, nil)
@@ -786,4 +794,17 @@ describe ApplicationSetting do
   end
 
   it_behaves_like 'application settings examples'
+
+  describe 'repository_storages_weighted_attributes' do
+    it 'returns the keys for repository_storages_weighted' do
+      expect(subject.class.repository_storages_weighted_attributes).to eq([:repository_storages_weighted_default])
+    end
+  end
+
+  it 'does not allow to set weight for non existing storage' do
+    setting.repository_storages_weighted = { invalid_storage: 100 }
+
+    expect(setting).not_to be_valid
+    expect(setting.errors.messages[:repository_storages_weighted]).to match_array(["can't include: invalid_storage"])
+  end
 end