Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
parent
100a5e4bea
commit
3174adc799
|
@ -405,7 +405,6 @@ img.emoji {
|
||||||
.prepend-top-16 { margin-top: 16px; }
|
.prepend-top-16 { margin-top: 16px; }
|
||||||
.prepend-top-20 { margin-top: 20px; }
|
.prepend-top-20 { margin-top: 20px; }
|
||||||
.prepend-top-32 { margin-top: 32px; }
|
.prepend-top-32 { margin-top: 32px; }
|
||||||
.prepend-left-2 { margin-left: 2px; }
|
|
||||||
.prepend-left-4 { margin-left: 4px; }
|
.prepend-left-4 { margin-left: 4px; }
|
||||||
.prepend-left-5 { margin-left: 5px; }
|
.prepend-left-5 { margin-left: 5px; }
|
||||||
.prepend-left-8 { margin-left: 8px; }
|
.prepend-left-8 { margin-left: 8px; }
|
||||||
|
|
|
@ -214,6 +214,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
|
||||||
[
|
[
|
||||||
*::ApplicationSettingsHelper.visible_attributes,
|
*::ApplicationSettingsHelper.visible_attributes,
|
||||||
*::ApplicationSettingsHelper.external_authorization_service_attributes,
|
*::ApplicationSettingsHelper.external_authorization_service_attributes,
|
||||||
|
*ApplicationSetting.repository_storages_weighted_attributes,
|
||||||
:lets_encrypt_notification_email,
|
:lets_encrypt_notification_email,
|
||||||
:lets_encrypt_terms_of_service_accepted,
|
:lets_encrypt_terms_of_service_accepted,
|
||||||
:domain_blacklist_file,
|
:domain_blacklist_file,
|
||||||
|
|
|
@ -19,6 +19,12 @@ class ApplicationSetting < ApplicationRecord
|
||||||
|
|
||||||
belongs_to :instance_administrators_group, class_name: "Group"
|
belongs_to :instance_administrators_group, class_name: "Group"
|
||||||
|
|
||||||
|
def self.repository_storages_weighted_attributes
|
||||||
|
@repository_storages_weighted_atributes ||= Gitlab.config.repositories.storages.keys.map { |k| "repository_storages_weighted_#{k}".to_sym }.freeze
|
||||||
|
end
|
||||||
|
|
||||||
|
store_accessor :repository_storages_weighted, *Gitlab.config.repositories.storages.keys, prefix: true
|
||||||
|
|
||||||
# Include here so it can override methods from
|
# Include here so it can override methods from
|
||||||
# `add_authentication_token_field`
|
# `add_authentication_token_field`
|
||||||
# We don't prepend for now because otherwise we'll need to
|
# We don't prepend for now because otherwise we'll need to
|
||||||
|
@ -39,6 +45,7 @@ class ApplicationSetting < ApplicationRecord
|
||||||
cache_markdown_field :after_sign_up_text
|
cache_markdown_field :after_sign_up_text
|
||||||
|
|
||||||
default_value_for :id, 1
|
default_value_for :id, 1
|
||||||
|
default_value_for :repository_storages_weighted, {}
|
||||||
|
|
||||||
chronic_duration_attr_writer :archive_builds_in_human_readable, :archive_builds_in_seconds
|
chronic_duration_attr_writer :archive_builds_in_human_readable, :archive_builds_in_seconds
|
||||||
|
|
||||||
|
@ -152,6 +159,7 @@ class ApplicationSetting < ApplicationRecord
|
||||||
|
|
||||||
validates :repository_storages, presence: true
|
validates :repository_storages, presence: true
|
||||||
validate :check_repository_storages
|
validate :check_repository_storages
|
||||||
|
validate :check_repository_storages_weighted
|
||||||
|
|
||||||
validates :auto_devops_domain,
|
validates :auto_devops_domain,
|
||||||
allow_blank: true,
|
allow_blank: true,
|
||||||
|
@ -271,6 +279,10 @@ class ApplicationSetting < ApplicationRecord
|
||||||
|
|
||||||
validates :allowed_key_types, presence: true
|
validates :allowed_key_types, presence: true
|
||||||
|
|
||||||
|
repository_storages_weighted_attributes.each do |attribute|
|
||||||
|
validates attribute, allow_nil: true, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 100 }
|
||||||
|
end
|
||||||
|
|
||||||
validates_each :restricted_visibility_levels do |record, attr, value|
|
validates_each :restricted_visibility_levels do |record, attr, value|
|
||||||
value&.each do |level|
|
value&.each do |level|
|
||||||
unless Gitlab::VisibilityLevel.options.value?(level)
|
unless Gitlab::VisibilityLevel.options.value?(level)
|
||||||
|
@ -434,6 +446,12 @@ class ApplicationSetting < ApplicationRecord
|
||||||
recaptcha_enabled || login_recaptcha_protection_enabled
|
recaptcha_enabled || login_recaptcha_protection_enabled
|
||||||
end
|
end
|
||||||
|
|
||||||
|
repository_storages_weighted_attributes.each do |attribute|
|
||||||
|
define_method :"#{attribute}=" do |value|
|
||||||
|
super(value.to_i)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def parsed_grafana_url
|
def parsed_grafana_url
|
||||||
|
|
|
@ -104,6 +104,7 @@ module ApplicationSettingImplementation
|
||||||
login_recaptcha_protection_enabled: false,
|
login_recaptcha_protection_enabled: false,
|
||||||
repository_checks_enabled: true,
|
repository_checks_enabled: true,
|
||||||
repository_storages: ['default'],
|
repository_storages: ['default'],
|
||||||
|
repository_storages_weighted: { default: 100 },
|
||||||
require_two_factor_authentication: false,
|
require_two_factor_authentication: false,
|
||||||
restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],
|
restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],
|
||||||
session_expire_delay: Settings.gitlab['session_expire_delay'],
|
session_expire_delay: Settings.gitlab['session_expire_delay'],
|
||||||
|
@ -262,6 +263,10 @@ module ApplicationSettingImplementation
|
||||||
Array(read_attribute(:repository_storages))
|
Array(read_attribute(:repository_storages))
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def repository_storages_weighted
|
||||||
|
read_attribute(:repository_storages_weighted)
|
||||||
|
end
|
||||||
|
|
||||||
def commit_email_hostname
|
def commit_email_hostname
|
||||||
super.presence || self.class.default_commit_email_hostname
|
super.presence || self.class.default_commit_email_hostname
|
||||||
end
|
end
|
||||||
|
@ -422,6 +427,12 @@ module ApplicationSettingImplementation
|
||||||
invalid.empty?
|
invalid.empty?
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def check_repository_storages_weighted
|
||||||
|
invalid = repository_storages_weighted.keys - Gitlab.config.repositories.storages.keys
|
||||||
|
errors.add(:repository_storages_weighted, "can't include: %{invalid_storages}" % { invalid_storages: invalid.join(", ") }) unless
|
||||||
|
invalid.empty?
|
||||||
|
end
|
||||||
|
|
||||||
def terms_exist
|
def terms_exist
|
||||||
return unless enforce_terms?
|
return unless enforce_terms?
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
title: Save repository storages in application settings with weights
|
||||||
|
merge_request: 31645
|
||||||
|
author:
|
||||||
|
type: added
|
|
@ -0,0 +1,13 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
class AddRepositoryStoragesWeightedToApplicationSettings < ActiveRecord::Migration[6.0]
|
||||||
|
DOWNTIME = false
|
||||||
|
|
||||||
|
def up
|
||||||
|
add_column :application_settings, :repository_storages_weighted, :jsonb, default: {}, null: false
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
remove_column :application_settings, :repository_storages_weighted
|
||||||
|
end
|
||||||
|
end
|
|
@ -440,6 +440,7 @@ CREATE TABLE public.application_settings (
|
||||||
group_owners_can_manage_default_branch_protection boolean DEFAULT true NOT NULL,
|
group_owners_can_manage_default_branch_protection boolean DEFAULT true NOT NULL,
|
||||||
container_registry_vendor text DEFAULT ''::text NOT NULL,
|
container_registry_vendor text DEFAULT ''::text NOT NULL,
|
||||||
container_registry_version text DEFAULT ''::text NOT NULL,
|
container_registry_version text DEFAULT ''::text NOT NULL,
|
||||||
|
repository_storages_weighted jsonb DEFAULT '{}'::jsonb NOT NULL,
|
||||||
container_registry_features text[] DEFAULT '{}'::text[] NOT NULL,
|
container_registry_features text[] DEFAULT '{}'::text[] NOT NULL,
|
||||||
spam_check_endpoint_url text,
|
spam_check_endpoint_url text,
|
||||||
spam_check_endpoint_enabled boolean DEFAULT false NOT NULL,
|
spam_check_endpoint_enabled boolean DEFAULT false NOT NULL,
|
||||||
|
@ -13888,6 +13889,7 @@ COPY "schema_migrations" (version) FROM STDIN;
|
||||||
20200508050301
|
20200508050301
|
||||||
20200508091106
|
20200508091106
|
||||||
20200508140959
|
20200508140959
|
||||||
|
20200508203901
|
||||||
20200511080113
|
20200511080113
|
||||||
20200511083541
|
20200511083541
|
||||||
20200511092246
|
20200511092246
|
||||||
|
|
|
@ -265,6 +265,20 @@ database: gitlabhq_production
|
||||||
up migration_id migration_name
|
up migration_id migration_name
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Run incomplete database migrations
|
||||||
|
|
||||||
|
Database migrations can be stuck in an incomplete state. That is, they'll have a `down`
|
||||||
|
status in the output of the `sudo gitlab-rake db:migrate:status` command.
|
||||||
|
|
||||||
|
To complete these migrations, use the following Rake task:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
sudo gitlab-rake db:migrate
|
||||||
|
```
|
||||||
|
|
||||||
|
After the command completes, run `sudo gitlab-rake db:migrate:status` to check if all
|
||||||
|
migrations are completed (have an `up` status).
|
||||||
|
|
||||||
## Import common metrics
|
## Import common metrics
|
||||||
|
|
||||||
Sometimes you may need to re-import the common metrics that power the Metrics dashboards.
|
Sometimes you may need to re-import the common metrics that power the Metrics dashboards.
|
||||||
|
|
|
@ -458,7 +458,17 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia
|
||||||
| `DAST_API_HOST_OVERRIDE` | no | Used to override domains defined in API specification files. |
|
| `DAST_API_HOST_OVERRIDE` | no | Used to override domains defined in API specification files. |
|
||||||
| `DAST_EXCLUDE_RULES` | no | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from the scan report. Currently, excluded rules will get executed but the alerts from them will be suppressed. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. |
|
| `DAST_EXCLUDE_RULES` | no | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from the scan report. Currently, excluded rules will get executed but the alerts from them will be suppressed. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. |
|
||||||
| `DAST_REQUEST_HEADERS` | no | Set to a comma-separated list of request header names and values. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` |
|
| `DAST_REQUEST_HEADERS` | no | Set to a comma-separated list of request header names and values. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` |
|
||||||
| `DAST_ZAP_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
|
| `DAST_DEBUG` | no | Enable debug message output. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
|
||||||
|
| `DAST_SPIDER_MINS` | no | The maximum duration of the spider scan in minutes. Set to zero for unlimited. Defaults to one minute, or unlimited when the scan is a full scan. |
|
||||||
|
| `DAST_HTML_REPORT` | no | The file name of the HTML report written at the end of a scan. |
|
||||||
|
| `DAST_MARKDOWN_REPORT` | no | The file name of the Markdown report written at the end of a scan. |
|
||||||
|
| `DAST_XML_REPORT` | no | The file name of the XML report written at the end of a scan. |
|
||||||
|
| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | no | Include alpha passive and active scan rules. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
|
||||||
|
| `DAST_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
|
||||||
|
| `DAST_ZAP_CLI_OPTIONS` | no | ZAP Server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. |
|
||||||
|
| `DAST_ZAP_GENERATE_CONFIG` | no | Generate sample ZAP config file for use with `DAST_ZAP_CONFIG_FILE`. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. |
|
||||||
|
| `DAST_ZAP_CONFIG_FILE` | no | Name of config file used to determine thresholds of vulnerability rules. |
|
||||||
|
| `DAST_ZAP_CONFIG_URL` | no | URL of config file used to determine thresholds of vulnerability rules. |
|
||||||
|
|
||||||
### DAST command-line options
|
### DAST command-line options
|
||||||
|
|
||||||
|
@ -475,8 +485,9 @@ dast:
|
||||||
- /analyze --help
|
- /analyze --help
|
||||||
```
|
```
|
||||||
|
|
||||||
You must then overwrite the `script` command to pass in the appropriate argument.
|
You must then overwrite the `script` command to pass in the appropriate
|
||||||
For example, debug messages can be enabled by using `-d`, as shown in the following configuration:
|
argument. For example, passive scanning can be delayed using option `-D`. The following
|
||||||
|
configuration delays passive scanning by five minutes:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
include:
|
include:
|
||||||
|
@ -485,7 +496,7 @@ include:
|
||||||
dast:
|
dast:
|
||||||
script:
|
script:
|
||||||
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
|
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
|
||||||
- /analyze -d -t $DAST_WEBSITE
|
- /analyze -D 300 -t $DAST_WEBSITE
|
||||||
```
|
```
|
||||||
|
|
||||||
### Custom ZAProxy configuration
|
### Custom ZAProxy configuration
|
||||||
|
@ -500,10 +511,8 @@ when used. An example of how to rewrite the Authorization header value with `TOK
|
||||||
include:
|
include:
|
||||||
template: DAST.gitlab-ci.yml
|
template: DAST.gitlab-ci.yml
|
||||||
|
|
||||||
dast:
|
variables:
|
||||||
script:
|
DAST_ZAP_CLI_OPTIONS: "-config replacer.full_list(0).description=auth -config replacer.full_list(0).enabled=true -config replacer.full_list(0).matchtype=REQ_HEADER -config replacer.full_list(0).matchstr=Authorization -config replacer.full_list(0).regex=false -config replacer.full_list(0).replacement=TOKEN"
|
||||||
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
|
|
||||||
- /analyze -z"-config replacer.full_list\(0\).description=auth -config replacer.full_list\(0\).enabled=true -config replacer.full_list\(0\).matchtype=REQ_HEADER -config replacer.full_list\(0\).matchstr=Authorization -config replacer.full_list\(0\).regex=false -config replacer.full_list\(0\).replacement=TOKEN" -t $DAST_WEBSITE
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Cloning the project's repository
|
### Cloning the project's repository
|
||||||
|
@ -625,18 +634,18 @@ vulnerabilities in your groups, projects and pipelines. Read more about the
|
||||||
|
|
||||||
## Bleeding-edge vulnerability definitions
|
## Bleeding-edge vulnerability definitions
|
||||||
|
|
||||||
ZAProxy first creates rules in the `alpha` class. After a testing period with the
|
ZAP first creates rules in the `alpha` class. After a testing period with
|
||||||
community, they are promoted to `beta`. DAST uses `beta` definitions by default.
|
the community, they are promoted to `beta`. DAST uses `beta` definitions by
|
||||||
To request `alpha` definitions, use `-a` as shown in the following configuration:
|
default. To request `alpha` definitions, use the
|
||||||
|
`DAST_INCLUDE_ALPHA_VULNERABILITIES` environment variable as shown in the
|
||||||
|
following configuration:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
include:
|
include:
|
||||||
template: DAST.gitlab-ci.yml
|
template: DAST.gitlab-ci.yml
|
||||||
|
|
||||||
dast:
|
variables:
|
||||||
script:
|
DAST_INCLUDE_ALPHA_VULNERABILITIES: true
|
||||||
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
|
|
||||||
- /analyze -a -t $DAST_WEBSITE
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Interacting with the vulnerabilities
|
## Interacting with the vulnerabilities
|
||||||
|
@ -688,16 +697,14 @@ This results in the following error:
|
||||||
```
|
```
|
||||||
|
|
||||||
Fortunately, it's straightforward to increase the amount of memory available
|
Fortunately, it's straightforward to increase the amount of memory available
|
||||||
for DAST by overwriting the `script` key in the DAST template:
|
for DAST by using the `DAST_ZAP_CLI_OPTIONS` environment variable:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
include:
|
include:
|
||||||
- template: DAST.gitlab-ci.yml
|
- template: DAST.gitlab-ci.yml
|
||||||
|
|
||||||
dast:
|
variables:
|
||||||
script:
|
DAST_ZAP_CLI_OPTIONS: "-Xmx3072m"
|
||||||
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
|
|
||||||
- /analyze -t $DAST_WEBSITE -z"-Xmx3072m"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Here, DAST is being allocated 3072 MB.
|
Here, DAST is being allocated 3072 MB.
|
||||||
|
|
|
@ -113,6 +113,7 @@ module API
|
||||||
end
|
end
|
||||||
optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
|
optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
|
||||||
optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects'
|
optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects'
|
||||||
|
optional :repository_storages_weighted, type: Hash, desc: 'Storage paths for new projects with a weighted value between 0 and 100'
|
||||||
optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication'
|
optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication'
|
||||||
given require_two_factor_authentication: ->(val) { val } do
|
given require_two_factor_authentication: ->(val) { val } do
|
||||||
requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
|
requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
|
||||||
|
|
|
@ -120,6 +120,13 @@ describe Admin::ApplicationSettingsController do
|
||||||
expect(ApplicationSetting.current.namespace_storage_size_limit).not_to eq(-100)
|
expect(ApplicationSetting.current.namespace_storage_size_limit).not_to eq(-100)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
it 'updates repository_storages_weighted setting' do
|
||||||
|
put :update, params: { application_setting: { repository_storages_weighted_default: 75 } }
|
||||||
|
|
||||||
|
expect(response).to redirect_to(general_admin_application_settings_path)
|
||||||
|
expect(ApplicationSetting.current.repository_storages_weighted_default).to eq(75)
|
||||||
|
end
|
||||||
|
|
||||||
context 'external policy classification settings' do
|
context 'external policy classification settings' do
|
||||||
let(:settings) do
|
let(:settings) do
|
||||||
{
|
{
|
||||||
|
|
|
@ -105,6 +105,14 @@ describe ApplicationSetting do
|
||||||
|
|
||||||
it { is_expected.not_to allow_value(false).for(:hashed_storage_enabled) }
|
it { is_expected.not_to allow_value(false).for(:hashed_storage_enabled) }
|
||||||
|
|
||||||
|
it { is_expected.not_to allow_value(101).for(:repository_storages_weighted_default) }
|
||||||
|
it { is_expected.not_to allow_value(-1).for(:repository_storages_weighted_default) }
|
||||||
|
it { is_expected.to allow_value(100).for(:repository_storages_weighted_default) }
|
||||||
|
it { is_expected.to allow_value(0).for(:repository_storages_weighted_default) }
|
||||||
|
it { is_expected.to allow_value(50).for(:repository_storages_weighted_default) }
|
||||||
|
it { is_expected.to allow_value(nil).for(:repository_storages_weighted_default) }
|
||||||
|
it { is_expected.not_to allow_value({ default: 100, shouldntexist: 50 }).for(:repository_storages_weighted) }
|
||||||
|
|
||||||
context 'grafana_url validations' do
|
context 'grafana_url validations' do
|
||||||
before do
|
before do
|
||||||
subject.instance_variable_set(:@parsed_grafana_url, nil)
|
subject.instance_variable_set(:@parsed_grafana_url, nil)
|
||||||
|
@ -786,4 +794,17 @@ describe ApplicationSetting do
|
||||||
end
|
end
|
||||||
|
|
||||||
it_behaves_like 'application settings examples'
|
it_behaves_like 'application settings examples'
|
||||||
|
|
||||||
|
describe 'repository_storages_weighted_attributes' do
|
||||||
|
it 'returns the keys for repository_storages_weighted' do
|
||||||
|
expect(subject.class.repository_storages_weighted_attributes).to eq([:repository_storages_weighted_default])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does not allow to set weight for non existing storage' do
|
||||||
|
setting.repository_storages_weighted = { invalid_storage: 100 }
|
||||||
|
|
||||||
|
expect(setting).not_to be_valid
|
||||||
|
expect(setting.errors.messages[:repository_storages_weighted]).to match_array(["can't include: invalid_storage"])
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in New Issue