Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2020-03-19 09:09:27 +00:00
parent cf58004721
commit 2af90cef2e
41 changed files with 810 additions and 49 deletions

View File

@ -256,7 +256,8 @@ export default class Clusters {
eventHub.$on('uninstallApplication', data => this.uninstallApplication(data));
eventHub.$on('setCrossplaneProviderStack', data => this.setCrossplaneProviderStack(data));
eventHub.$on('setIngressModSecurityEnabled', data => this.setIngressModSecurityEnabled(data));
eventHub.$on('resetIngressModSecurityEnabled', id => this.resetIngressModSecurityEnabled(id));
eventHub.$on('setIngressModSecurityMode', data => this.setIngressModSecurityMode(data));
eventHub.$on('resetIngressModSecurityChanges', id => this.resetIngressModSecurityChanges(id));
// Add event listener to all the banner close buttons
this.addBannerCloseHandler(this.unreachableContainer, 'unreachable');
this.addBannerCloseHandler(this.authenticationFailureContainer, 'authentication_failure');
@ -271,7 +272,8 @@ export default class Clusters {
eventHub.$off('setCrossplaneProviderStack');
eventHub.$off('uninstallApplication');
eventHub.$off('setIngressModSecurityEnabled');
eventHub.$off('resetIngressModSecurityEnabled');
eventHub.$off('setIngressModSecurityMode');
eventHub.$off('resetIngressModSecurityChanges');
}
initPolling(method, successCallback, errorCallback) {
@ -525,8 +527,14 @@ export default class Clusters {
this.store.updateAppProperty(id, 'modsecurity_enabled', modSecurityEnabled);
}
resetIngressModSecurityEnabled(id) {
setIngressModSecurityMode({ id, modSecurityMode }) {
this.store.updateAppProperty(id, 'isEditingModSecurityMode', true);
this.store.updateAppProperty(id, 'modsecurity_mode', modSecurityMode);
}
resetIngressModSecurityChanges(id) {
this.store.updateAppProperty(id, 'isEditingModSecurityEnabled', false);
this.store.updateAppProperty(id, 'isEditingModSecurityMode', false);
}
destroy() {

View File

@ -313,6 +313,7 @@ Crossplane runs inside your Kubernetes cluster and supports secure connectivity
:install-failed="applications.ingress.installFailed"
:install-application-request-params="{
modsecurity_enabled: applications.ingress.modsecurity_enabled,
modsecurity_mode: applications.ingress.modsecurity_mode,
}"
:uninstallable="applications.ingress.uninstallable"
:uninstall-successful="applications.ingress.uninstallSuccessful"

View File

@ -1,8 +1,17 @@
<script>
import _ from 'lodash';
import { __ } from '../../locale';
import { APPLICATION_STATUS, INGRESS } from '~/clusters/constants';
import { GlAlert, GlSprintf, GlLink, GlToggle, GlButton } from '@gitlab/ui';
import { escape as esc } from 'lodash';
import { s__, __ } from '../../locale';
import { APPLICATION_STATUS, INGRESS, LOGGING_MODE, BLOCKING_MODE } from '~/clusters/constants';
import {
GlAlert,
GlSprintf,
GlLink,
GlToggle,
GlButton,
GlDropdown,
GlDropdownItem,
GlIcon,
} from '@gitlab/ui';
import eventHub from '~/clusters/event_hub';
import modSecurityLogo from 'images/cluster_app_logos/modsecurity.png';
@ -17,6 +26,9 @@ export default {
GlLink,
GlToggle,
GlButton,
GlDropdown,
GlDropdownItem,
GlIcon,
},
props: {
ingress: {
@ -28,10 +40,23 @@ export default {
required: false,
default: '',
},
modes: {
type: Object,
required: false,
default: () => ({
[LOGGING_MODE]: {
name: s__('ClusterIntegration|Logging mode'),
},
[BLOCKING_MODE]: {
name: s__('ClusterIntegration|Blocking mode'),
},
}),
},
},
data: () => ({
modSecurityLogo,
hasValueChanged: false,
initialValue: null,
initialMode: null,
}),
computed: {
modSecurityEnabled: {
@ -39,19 +64,30 @@ export default {
return this.ingress.modsecurity_enabled;
},
set(isEnabled) {
if (this.initialValue === null) {
this.initialValue = this.ingress.modsecurity_enabled;
}
eventHub.$emit('setIngressModSecurityEnabled', {
id: INGRESS,
modSecurityEnabled: isEnabled,
});
if (this.hasValueChanged) {
this.resetStatus();
} else {
this.hasValueChanged = true;
}
},
},
hasValueChanged() {
return this.modSecurityEnabledChanged || this.modSecurityModeChanged;
},
modSecurityEnabledChanged() {
return this.initialValue !== null && this.initialValue !== this.ingress.modsecurity_enabled;
},
modSecurityModeChanged() {
return (
this.ingress.modsecurity_enabled &&
this.initialMode !== null &&
this.initialMode !== this.ingress.modsecurity_mode
);
},
ingressModSecurityDescription() {
return _.escape(this.ingressModSecurityHelpPath);
return esc(this.ingressModSecurityHelpPath);
},
saving() {
return [UPDATING].includes(this.ingress.status);
@ -73,18 +109,40 @@ export default {
this.saving || (this.hasValueChanged && [INSTALLED, UPDATED].includes(this.ingress.status))
);
},
modSecurityModeName() {
return this.modes[this.ingress.modsecurity_mode].name;
},
},
methods: {
updateApplication() {
eventHub.$emit('updateApplication', {
id: INGRESS,
params: { modsecurity_enabled: this.ingress.modsecurity_enabled },
params: {
modsecurity_enabled: this.ingress.modsecurity_enabled,
modsecurity_mode: this.ingress.modsecurity_mode,
},
});
this.resetStatus();
},
resetStatus() {
eventHub.$emit('resetIngressModSecurityEnabled', INGRESS);
this.hasValueChanged = false;
if (this.initialMode !== null) {
this.ingress.modsecurity_mode = this.initialMode;
}
if (this.initialValue !== null) {
this.ingress.modsecurity_enabled = this.initialValue;
}
this.initialValue = null;
this.initialMode = null;
eventHub.$emit('resetIngressModSecurityChanges', INGRESS);
},
selectMode(modeKey) {
if (this.initialMode === null) {
this.initialMode = this.ingress.modsecurity_mode;
}
eventHub.$emit('setIngressModSecurityMode', {
id: INGRESS,
modSecurityMode: modeKey,
});
},
},
};
@ -144,7 +202,35 @@ export default {
label-position="right"
/>
</div>
<div v-if="showButtons">
<div
v-if="ingress.modsecurity_enabled"
class="gl-responsive-table-row-layout mt-3"
role="row"
>
<div class="table-section section-wrap" role="gridcell">
<strong>
{{ s__('ClusterIntegration|Global default') }}
<gl-icon name="earth" class="align-text-bottom" />
</strong>
<div class="form-group">
<p class="form-text text-muted">
<strong>
{{
s__(
'ClusterIntegration|Set the global mode for the WAF in this cluster. This can be overridden at the environmental level.',
)
}}
</strong>
</p>
</div>
<gl-dropdown :text="modSecurityModeName" :disabled="saveButtonDisabled">
<gl-dropdown-item v-for="(mode, key) in modes" :key="key" @click="selectMode(key)">
{{ mode.name }}
</gl-dropdown-item>
</gl-dropdown>
</div>
</div>
<div v-if="showButtons" class="mt-3">
<gl-button
class="btn-success inline mr-1"
:loading="saving"

View File

@ -66,3 +66,6 @@ export const APPLICATIONS = [
];
export const INGRESS_DOMAIN_SUFFIX = '.nip.io';
export const LOGGING_MODE = 'logging';
export const BLOCKING_MODE = 'blocking';

View File

@ -53,9 +53,11 @@ export default class ClusterStore {
...applicationInitialState,
title: s__('ClusterIntegration|Ingress'),
modsecurity_enabled: false,
modsecurity_mode: null,
externalIp: null,
externalHostname: null,
isEditingModSecurityEnabled: false,
isEditingModSecurityMode: false,
updateFailed: false,
},
cert_manager: {
@ -214,6 +216,9 @@ export default class ClusterStore {
if (!this.state.applications.ingress.isEditingModSecurityEnabled) {
this.state.applications.ingress.modsecurity_enabled = serverAppEntry.modsecurity_enabled;
}
if (!this.state.applications.ingress.isEditingModSecurityMode) {
this.state.applications.ingress.modsecurity_mode = serverAppEntry.modsecurity_mode;
}
} else if (appId === CERT_MANAGER) {
this.state.applications.cert_manager.email =
this.state.applications.cert_manager.email || serverAppEntry.email;

View File

@ -47,7 +47,7 @@ class Clusters::ApplicationsController < Clusters::BaseController
end
def cluster_application_params
params.permit(:application, :hostname, :email, :stack, :modsecurity_enabled)
params.permit(:application, :hostname, :email, :stack, :modsecurity_enabled, :modsecurity_mode)
end
def cluster_application_destroy_params

View File

@ -65,6 +65,14 @@ module Types
calls_gitaly: true,
null: false
field :ssh_url_to_repo, type: GraphQL::STRING_TYPE,
description: 'SSH URL to the snippet repository',
null: true
field :http_url_to_repo, type: GraphQL::STRING_TYPE,
description: 'HTTP URL to the snippet repository',
null: true
markdown_field :description_html, null: true, method: :description
end
end

View File

@ -6,6 +6,9 @@ module Clusters
VERSION = '1.29.7'
INGRESS_CONTAINER_NAME = 'nginx-ingress-controller'
MODSECURITY_LOG_CONTAINER_NAME = 'modsecurity-log'
MODSECURITY_MODE_LOGGING = "DetectionOnly"
MODSECURITY_MODE_BLOCKING = "On"
MODSECURITY_OWASP_RULES_FILE = "/etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf"
self.table_name = 'clusters_applications_ingress'
@ -18,11 +21,14 @@ module Clusters
default_value_for :ingress_type, :nginx
default_value_for :modsecurity_enabled, true
default_value_for :version, VERSION
default_value_for :modsecurity_mode, :logging
enum ingress_type: {
nginx: 1
}
enum modsecurity_mode: { logging: 0, blocking: 1 }
FETCH_IP_ADDRESS_DELAY = 30.seconds
MODSEC_SIDECAR_INITIAL_DELAY_SECONDS = 10
@ -82,7 +88,8 @@ module Clusters
"controller" => {
"config" => {
"enable-modsecurity" => "true",
"enable-owasp-modsecurity-crs" => "true",
"enable-owasp-modsecurity-crs" => "false",
"modsecurity-snippet" => modsecurity_snippet_content,
"modsecurity.conf" => modsecurity_config_content
},
"extraContainers" => [
@ -157,6 +164,11 @@ module Clusters
def application_jupyter_nil_or_installable?
cluster.application_jupyter.nil? || cluster.application_jupyter&.installable?
end
def modsecurity_snippet_content
sec_rule_engine = logging? ? MODSECURITY_MODE_LOGGING : MODSECURITY_MODE_BLOCKING
"SecRuleEngine #{sec_rule_engine}\nInclude #{MODSECURITY_OWASP_RULES_FILE}"
end
end
end
end

View File

@ -312,6 +312,10 @@ class Snippet < ApplicationRecord
Digest::SHA256.hexdigest("#{title}#{description}#{created_at}#{updated_at}")
end
def versioned_enabled_for?(user)
repository_exists? && ::Feature.enabled?(:version_snippets, user)
end
class << self
# Searches for snippets with a matching title, description or file name.
#

View File

@ -11,6 +11,14 @@ class SnippetPresenter < Gitlab::View::Presenter::Delegated
Gitlab::UrlBuilder.build(snippet, raw: true)
end
def ssh_url_to_repo
snippet.ssh_url_to_repo if snippet.versioned_enabled_for?(current_user)
end
def http_url_to_repo
snippet.http_url_to_repo if snippet.versioned_enabled_for?(current_user)
end
def can_read_snippet?
can_access_resource?("read")
end

View File

@ -15,4 +15,5 @@ class ClusterApplicationEntity < Grape::Entity
expose :can_uninstall?, as: :can_uninstall
expose :available_domains, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:available_domains) }
expose :pages_domain, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:pages_domain) }
expose :modsecurity_mode, if: -> (e, _) { e.respond_to?(:modsecurity_mode) }
end

View File

@ -31,6 +31,10 @@ module Clusters
application.modsecurity_enabled = params[:modsecurity_enabled] || false
end
if application.has_attribute?(:modsecurity_mode)
application.modsecurity_mode = params[:modsecurity_mode] || 0
end
if application.respond_to?(:oauth_application)
application.oauth_application = create_oauth_application(application, request)
end

View File

@ -0,0 +1,5 @@
---
title: Resolve Improve format support message in issue design
merge_request: 27409
author:
type: fixed

View File

@ -0,0 +1,5 @@
---
title: Add option for switching between blocking and logging for WAF
merge_request: 27133
author:
type: added

View File

@ -0,0 +1,19 @@
# frozen_string_literal: true
# See https://docs.gitlab.com/ee/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddModsecurityModeToIngressApplication < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column_with_default(:clusters_applications_ingress, :modsecurity_mode, :smallint, default: 0, allow_null: false)
end
def down
remove_column :clusters_applications_ingress, :modsecurity_mode
end
end

View File

@ -1199,6 +1199,7 @@ ActiveRecord::Schema.define(version: 2020_03_13_123934) do
t.string "external_ip"
t.string "external_hostname"
t.boolean "modsecurity_enabled"
t.integer "modsecurity_mode", limit: 2, default: 0, null: false
t.index ["cluster_id"], name: "index_clusters_applications_ingress_on_cluster_id", unique: true
end

View File

@ -1016,6 +1016,15 @@ When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']`
When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/issues/2521) is resolved.
### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly node
If this error occurs even though file permissions are correct, it's likely that
the Gitaly node is experiencing
[clock drift](https://en.wikipedia.org/wiki/Clock_drift).
Please ensure that the GitLab and Gitaly nodes are synchronized and use an NTP time
server to keep them synchronized if possible.
### Praefect
Praefect is an experimental daemon that allows for replication of the Git data.

View File

@ -573,6 +573,127 @@ Particular attention should be shown to:
Congratulations! You have configured a highly available Praefect cluster.
### Failover
There are two ways to do a failover from one internal Gitaly node to another as the primary. Manually, or automatically.
As an example, in this `config.toml` we have one virtual storage named "default" with two internal Gitaly nodes behind it.
One is deemed the "primary". This means that read and write traffic will go to `internal_storage_0`, and writes
will get replicated to `internal_storage_1`:
```toml
socket_path = "/path/to/Praefect.socket"
# failover_enabled will enable automatic failover
failover_enabled = false
[logging]
format = "json"
level = "info"
[[virtual_storage]]
name = "default"
[[virtual_storage.node]]
name = "internal_storage_0"
address = "tcp://localhost:9999"
primary = true
token = "supersecret"
[[virtual_storage.node]]
name = "internal_storage_1"
address = "tcp://localhost:9998"
token = "supersecret"
```
#### Manual failover
In order to failover from using one internal Gitaly node to using another, a manual failover step can be used. Unless `failover_enabled` is set to `true`
in the `config.toml`, the only way to fail over from one primary to using another node as the primary is to do a manual failover.
1. Move `primary = true` from the current `[[virtual_storage.node]]` to another node in `/etc/gitlab/gitlab.rb`:
```ruby
praefect['virtual_storages'] = {
'praefect' => {
'gitaly-1' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
# no longer the primary
},
'gitaly-2' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
# this is the new primary
'primary' => true
},
'gitaly-3' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
}
}
}
```
1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
On a restart, Praefect will send write traffic to `internal_storage_1`. `internal_storage_0` is the new secondary now,
and replication jobs will be created to replicate repository data to `internal_storage_0` **from** `internal_storage_1`
#### Automatic failover
When automatic failover is enabled, Praefect will do automatic detection of the health of internal Gitaly nodes. If the
primary has a certain amount of healthchecks fail, it will decide to promote one of the secondaries to be primary, and
demote the primary to be a secondary.
1. To enable automatic failover, edit `/etc/gitlab/gitlab.rb`:
```ruby
# failover_enabled turns on automatic failover
praefect['failover_enabled'] = true
praefect['virtual_storages'] = {
'praefect' => {
'gitaly-1' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
'primary' => true
},
'gitaly-2' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN'
},
'gitaly-3' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN'
}
}
}
```
1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
Below is the picture when Praefect starts up with the config.toml above:
```mermaid
graph TD
A[Praefect] -->|Mutator RPC| B(internal_storage_0)
B --> |Replication|C[internal_storage_1]
```
Let's say suddenly `internal_storage_0` goes down. Praefect will detect this and
automatically switch over to `internal_storage_1`, and `internal_storage_0` will serve as a secondary:
```mermaid
graph TD
A[Praefect] -->|Mutator RPC| B(internal_storage_1)
B --> |Replication|C[internal_storage_0]
```
NOTE: **Note:**: Currently this feature is supported for setups that only have 1 Praefect instance. Praefect instances running,
for example behind a load balancer, `failover_enabled` should be disabled. The reason is The reason is because there
is no coordination that currently happens across different Praefect instances, so there could be a situation where
two Praefect instances think two different Gitaly nodes are the primary.
## Migrating existing repositories to Praefect
If your GitLab instance already has repositories, these won't be migrated

View File

@ -110,7 +110,7 @@ them.
| [PgBouncer](../../development/architecture.md#pgbouncer) | Database Pool Manager | [PgBouncer HA configuration](pgbouncer.md) **(PREMIUM ONLY)** |
| [Redis](../../development/architecture.md#redis)[^3] with Redis Sentinel | Key/Value store for shared data with HA watcher service | [Redis HA configuration](redis.md) |
| [Gitaly](../../development/architecture.md#gitaly)[^2] [^5] [^7] | Recommended high-level storage for Git repository data | [Gitaly HA configuration](gitaly.md) |
| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/Background jobs | |
| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/Background jobs | [Sidekiq configuration](sidekiq.md) |
| [GitLab application nodes](../../development/architecture.md#unicorn)[^1] | (Unicorn / Puma, Workhorse) - Web-requests (UI, API, Git over HTTP) | [GitLab app HA/scaling configuration](gitlab.md) |
| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling/HA](monitoring_node.md) |

View File

@ -0,0 +1,186 @@
---
type: reference
---
# Configuring Sidekiq
This section discusses how to configure an external Sidekiq instance.
Sidekiq requires connection to the Redis, PostgreSQL and Gitaly instance.
To configure the Sidekiq node:
1. SSH into the Sidekiq server.
1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package
you want using steps 1 and 2 from the GitLab downloads page.
**Do not complete any other steps on the download page.**
1. Open `/etc/gitlab/gitlab.rb` with your editor.
1. Generate the Sidekiq configuration:
```ruby
sidekiq['listen_address'] = "10.10.1.48"
## Optional: Enable extra Sidekiq processes
sidekiq_cluster['enable'] = true
sidekiq_cluster['enable'] = true
"elastic_indexer"
]
```
1. Setup Sidekiq's connection to Redis:
```ruby
## Must be the same in every sentinel node
redis['master_name'] = 'gitlab-redis'
## The same password for Redis authentication you set up for the master node.
redis['master_password'] = 'YOUR_PASSOWORD'
## A list of sentinels with `host` and `port`
gitlab_rails['redis_sentinels'] = [
{'host' => '10.10.1.34', 'port' => 26379},
{'host' => '10.10.1.35', 'port' => 26379},
{'host' => '10.10.1.36', 'port' => 26379},
]
```
1. Setup Sidekiq's connection to Gitaly:
```ruby
git_data_dirs({
'default' => { 'gitaly_address' => 'tcp://gitaly:8075' },
})
gitlab_rails['gitaly_token'] = 'YOUR_TOKEN'
```
1. Setup Sidekiq's connection to Postgres:
```ruby
gitlab_rails['db_host'] = '10.10.1.30'
gitlab_rails['db_password'] = 'YOUR_PASSOWORD'
gitlab_rails['db_port'] = '5432'
gitlab_rails['db_adapter'] = 'postgresql'
gitlab_rails['db_encoding'] = 'unicode'
gitlab_rails['auto_migrate'] = false
```
Remember to add the Sidekiq nodes to the Postgres whitelist:
```ruby
postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.10.1.30/32 10.10.1.31/32 10.10.1.32/32 10.10.1.33/32 10.10.1.38/32)
```
1. Disable other services:
```ruby
nginx['enable'] = false
grafana['enable'] = false
prometheus['enable'] = false
gitlab_rails['auto_migrate'] = false
alertmanager['enable'] = false
gitaly['enable'] = false
gitlab_monitor['enable'] = false
gitlab_workhorse['enable'] = false
nginx['enable'] = false
postgres_exporter['enable'] = false
postgresql['enable'] = false
redis['enable'] = false
redis_exporter['enable'] = false
unicorn['enable'] = false
gitlab_exporter['enable'] = false
```
1. Run `gitlab-ctl reconfigure`.
## Example configuration
Here's what the ending `/etc/gitlab/gitlab.rb` would look like:
```ruby
########################################
##### Services Disabled ###
########################################
nginx['enable'] = false
grafana['enable'] = false
prometheus['enable'] = false
gitlab_rails['auto_migrate'] = false
alertmanager['enable'] = false
gitaly['enable'] = false
gitlab_monitor['enable'] = false
gitlab_workhorse['enable'] = false
nginx['enable'] = false
postgres_exporter['enable'] = false
postgresql['enable'] = false
redis['enable'] = false
redis_exporter['enable'] = false
unicorn['enable'] = false
gitlab_exporter['enable'] = false
########################################
#### Redis ###
########################################
## Must be the same in every sentinel node
redis['master_name'] = 'gitlab-redis'
## The same password for Redis authentication you set up for the master node.
redis['master_password'] = 'YOUR_PASSOWORD'
## A list of sentinels with `host` and `port`
gitlab_rails['redis_sentinels'] = [
{'host' => '10.10.1.34', 'port' => 26379},
{'host' => '10.10.1.35', 'port' => 26379},
{'host' => '10.10.1.36', 'port' => 26379},
]
#######################################
### Gitaly ###
#######################################
git_data_dirs({
'default' => { 'gitaly_address' => 'tcp://gitaly:8075' },
})
gitlab_rails['gitaly_token'] = 'YOUR_TOKEN'
#######################################
### Postgres ###
#######################################
gitlab_rails['db_host'] = '10.10.1.30'
gitlab_rails['db_password'] = 'YOUR_PASSOWORD'
gitlab_rails['db_port'] = '5432'
gitlab_rails['db_adapter'] = 'postgresql'
gitlab_rails['db_encoding'] = 'unicode'
gitlab_rails['auto_migrate'] = false
#######################################
### Sidekiq configuration ###
#######################################
sidekiq['listen_address'] = "10.10.1.48"
#######################################
### Monitoring configuration ###
#######################################
consul['enable'] = true
consul['monitoring_service_discovery'] = true
consul['configuration'] = {
bind_addr: '10.10.1.48',
retry_join: %w(10.10.1.34 10.10.1.35 10.10.1.36)
}
# Set the network addresses that the exporters will listen on
node_exporter['listen_address'] = '10.10.1.48:9100'
# Rails Status for prometheus
gitlab_rails['monitoring_whitelist'] = ['10.10.1.42', '127.0.0.1']
```
## Further reading
Related Sidekiq configuration:
1. [Extra Sidekiq processes](../operations/extra_sidekiq_processes.md)
1. [Using the GitLab-Sidekiq chart](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/)

View File

@ -353,10 +353,6 @@ configuring a different storage driver. By default the GitLab Container Registry
is configured to use the filesystem driver, which makes use of [storage path](#container-registry-storage-path)
configuration.
NOTE: **Note:** Enabling a storage driver other than `filesystem` would mean
that your Docker client needs to be able to access the storage backend directly.
In that case, you must use an address that resolves and is accessible outside GitLab server. The Docker client will continue to authenticate via GitLab but data transfer will be direct to and from the storage backend.
The different supported drivers are:
| Driver | Description |
@ -425,6 +421,55 @@ storage:
NOTE: **Note:**
`your-s3-bucket` should only be the name of a bucket that exists, and can't include subdirectories.
### Disable redirect for storage driver
By default, users accessing a registry configured with a remote backend are redirected to the default backend for the storage driver. For example, registries can be configured using the `s3` storage driver, which redirects requests to a remote S3 bucket to alleviate load on the GitLab server.
However, this behaviour is undesirable for registries used by internal hosts that usually can't access public servers. To disable redirects, set the `disable` flag to true as follows. This makes all traffic to always go through the Registry service. This results in improved security (less surface attack as the storage backend is not publicly accessible), but worse performance (all traffic is redirected via the service).
**Omnibus GitLab installations**
1. Edit `/etc/gitlab/gitlab.rb`:
```ruby
registry['storage'] = {
's3' => {
'accesskey' => 's3-access-key',
'secretkey' => 's3-secret-key-for-access-key',
'bucket' => 'your-s3-bucket',
'region' => 'your-s3-region',
'regionendpoint' => 'your-s3-regionendpoint'
},
'redirect' => {
'disable' => true
}
}
```
1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
**Installations from source**
1. Add the `redirect` flag to your registry configuration YML file:
```yml
storage:
s3:
accesskey: 'AKIAKIAKI'
secretkey: 'secret123'
bucket: 'gitlab-registry-bucket-AKIAKIAKI'
region: 'your-s3-region'
regionendpoint: 'your-s3-regionendpoint'
redirect:
disable: true
cache:
blobdescriptor: inmemory
delete:
enabled: true
```
1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect.
### Storage limitations
Currently, there is no storage limitation, which means a user can upload an

View File

@ -840,7 +840,7 @@ GitLab Rails console:
```ruby
projects_and_size = []
# a list of projects you want to look at, can get these however
# You need to specify the projects that you want to look through. You can get these in any manner.
projects = Project.last(100)
projects.each do |p|

View File

@ -7355,6 +7355,11 @@ type Snippet implements Noteable {
"""
fileName: String
"""
HTTP URL to the snippet repository
"""
httpUrlToRepo: String
"""
Id of the snippet
"""
@ -7395,6 +7400,11 @@ type Snippet implements Noteable {
"""
rawUrl: String!
"""
SSH URL to the snippet repository
"""
sshUrlToRepo: String
"""
Title of the snippet
"""

View File

@ -22213,6 +22213,20 @@
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "httpUrlToRepo",
"description": "HTTP URL to the snippet repository",
"args": [
],
"type": {
"kind": "SCALAR",
"name": "String",
"ofType": null
},
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "id",
"description": "Id of the snippet",
@ -22320,6 +22334,20 @@
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "sshUrlToRepo",
"description": "SSH URL to the snippet repository",
"args": [
],
"type": {
"kind": "SCALAR",
"name": "String",
"ofType": null
},
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "title",
"description": "Title of the snippet",

View File

@ -1161,9 +1161,11 @@ Represents a snippet entry
| `description` | String | Description of the snippet |
| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` |
| `fileName` | String | File Name of the snippet |
| `httpUrlToRepo` | String | HTTP URL to the snippet repository |
| `id` | ID! | Id of the snippet |
| `project` | Project | The project the snippet is associated with |
| `rawUrl` | String! | Raw URL of the snippet |
| `sshUrlToRepo` | String | SSH URL to the snippet repository |
| `title` | String! | Title of the snippet |
| `updatedAt` | Time! | Timestamp this snippet was updated |
| `userPermissions` | SnippetPermissions! | Permissions for the current user on the resource |

View File

@ -19,6 +19,7 @@ type: index
- [Send email confirmation on sign-up](user_email_confirmation.md)
- [Security of running jobs](https://docs.gitlab.com/runner/security/)
- [Proxying images](asset_proxy.md)
- [CI/CD environment variables](cicd_environment_variables.md)
## Securing your GitLab installation

View File

@ -0,0 +1,11 @@
---
type: reference
---
# CI/CD Environment Variables
Environment variables are applied to environments via the runner and can be set from the project's **Settings > CI/CD** page.
The values are encrypted using [aes-256-cbc](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) and stored in the database.
This data can only be decrypted with a valid [secrets file](../raketasks/backup_restore.md#when-the-secrets-file-is-lost).

View File

@ -36,7 +36,7 @@ A user can be deactivated from the Admin Area. To do this:
Please note that for the deactivation option to be visible to an admin, the user:
- Must be currently active.
- Should not have any activity in the last 180 days.
- Must not have any signins or activity in the last 180 days.
Users can also be deactivated using the [GitLab API](../../api/users.md#deactivate-user).

View File

@ -10,6 +10,7 @@ module API
expose :web_url do |snippet|
Gitlab::UrlBuilder.build(snippet)
end
expose :ssh_url_to_repo, :http_url_to_repo, if: ->(snippet) { snippet.versioned_enabled_for?(options[:current_user]) }
end
end
end

View File

@ -75,6 +75,7 @@ module Gitlab
def find_metric(metrics, metric)
return unless metrics
return unless metric.identifier
metrics.find { |m| m[:id] == metric.identifier }
end

View File

@ -4126,6 +4126,9 @@ msgstr ""
msgid "ClusterIntegration|Base domain"
msgstr ""
msgid "ClusterIntegration|Blocking mode"
msgstr ""
msgid "ClusterIntegration|CA Certificate"
msgstr ""
@ -4324,6 +4327,9 @@ msgstr ""
msgid "ClusterIntegration|Gitlab Integration"
msgstr ""
msgid "ClusterIntegration|Global default"
msgstr ""
msgid "ClusterIntegration|Google Cloud Platform project"
msgstr ""
@ -4483,6 +4489,9 @@ msgstr ""
msgid "ClusterIntegration|Loading subnetworks"
msgstr ""
msgid "ClusterIntegration|Logging mode"
msgstr ""
msgid "ClusterIntegration|Machine type"
msgstr ""
@ -4711,6 +4720,9 @@ msgstr ""
msgid "ClusterIntegration|Set a prefix for your namespaces. If not set, defaults to your project path. If modified, existing environments will use their current namespaces until the cluster cache is cleared."
msgstr ""
msgid "ClusterIntegration|Set the global mode for the WAF in this cluster. This can be overridden at the environmental level."
msgstr ""
msgid "ClusterIntegration|Show"
msgstr ""

View File

@ -38,6 +38,7 @@
"email": { "type": ["string", "null"] },
"stack": { "type": ["string", "null"] },
"modsecurity_enabled": { "type": ["boolean", "null"] },
"modsecurity_mode": {"type": ["integer", "0"]},
"update_available": { "type": ["boolean", "null"] },
"can_uninstall": { "type": "boolean" },
"available_domains": {

View File

@ -1,7 +1,7 @@
import { shallowMount } from '@vue/test-utils';
import IngressModsecuritySettings from '~/clusters/components/ingress_modsecurity_settings.vue';
import { APPLICATION_STATUS, INGRESS } from '~/clusters/constants';
import { GlAlert, GlToggle } from '@gitlab/ui';
import { GlAlert, GlToggle, GlDropdown } from '@gitlab/ui';
import eventHub from '~/clusters/event_hub';
const { UPDATING } = APPLICATION_STATUS;
@ -13,6 +13,7 @@ describe('IngressModsecuritySettings', () => {
modsecurity_enabled: false,
status: 'installable',
installed: false,
modsecurity_mode: 'logging',
};
const createComponent = (props = defaultProps) => {
@ -29,6 +30,7 @@ describe('IngressModsecuritySettings', () => {
const findSaveButton = () => wrapper.find('.btn-success');
const findCancelButton = () => wrapper.find('[variant="secondary"]');
const findModSecurityToggle = () => wrapper.find(GlToggle);
const findModSecurityDropdown = () => wrapper.find(GlDropdown);
describe('when ingress is installed', () => {
beforeEach(() => {
@ -44,22 +46,50 @@ describe('IngressModsecuritySettings', () => {
describe('with toggle changed by the user', () => {
beforeEach(() => {
findModSecurityToggle().vm.$emit('change');
wrapper.setProps({
ingress: {
...defaultProps,
installed: true,
status: 'installed',
modsecurity_enabled: true,
},
});
});
it('renders both save and cancel buttons', () => {
it('renders save and cancel buttons', () => {
expect(findSaveButton().exists()).toBe(true);
expect(findCancelButton().exists()).toBe(true);
});
describe('and the save changes button is clicked', () => {
describe('with dropdown changed by the user', () => {
beforeEach(() => {
findSaveButton().vm.$emit('click');
findModSecurityDropdown().vm.$children[1].$emit('click');
wrapper.setProps({
ingress: {
...defaultProps,
installed: true,
status: 'installed',
modsecurity_enabled: true,
modsecurity_mode: 'blocking',
},
});
});
it('triggers save event and pass current modsecurity value', () => {
expect(eventHub.$emit).toHaveBeenCalledWith('updateApplication', {
id: INGRESS,
params: { modsecurity_enabled: false },
it('renders both save and cancel buttons', () => {
expect(findSaveButton().exists()).toBe(true);
expect(findCancelButton().exists()).toBe(true);
});
describe('and the save changes button is clicked', () => {
beforeEach(() => {
findSaveButton().vm.$emit('click');
});
it('triggers save event and pass current modsecurity value', () => {
expect(eventHub.$emit).toHaveBeenCalledWith('updateApplication', {
id: INGRESS,
params: { modsecurity_enabled: true, modsecurity_mode: 'blocking' },
});
});
});
});
@ -70,7 +100,7 @@ describe('IngressModsecuritySettings', () => {
});
it('triggers reset event and hides both cancel and save changes button', () => {
expect(eventHub.$emit).toHaveBeenCalledWith('resetIngressModSecurityEnabled', INGRESS);
expect(eventHub.$emit).toHaveBeenCalledWith('resetIngressModSecurityChanges', INGRESS);
expect(findSaveButton().exists()).toBe(false);
expect(findCancelButton().exists()).toBe(false);
});

View File

@ -82,6 +82,7 @@ describe('Clusters Store', () => {
externalHostname: null,
installed: false,
isEditingModSecurityEnabled: false,
isEditingModSecurityMode: false,
installFailed: true,
uninstallable: false,
updateFailed: false,
@ -89,6 +90,7 @@ describe('Clusters Store', () => {
uninstallFailed: false,
validationError: null,
modsecurity_enabled: false,
modsecurity_mode: undefined,
},
runner: {
title: 'GitLab Runner',

View File

@ -3,12 +3,15 @@
require 'spec_helper'
describe GitlabSchema.types['Snippet'] do
let_it_be(:user) { create(:user) }
it 'has the correct fields' do
expected_fields = [:id, :title, :project, :author,
:file_name, :description,
:visibility_level, :created_at, :updated_at,
:web_url, :raw_url, :notes, :discussions,
:user_permissions, :description_html, :blob]
:web_url, :raw_url, :ssh_url_to_repo, :http_url_to_repo,
:notes, :discussions, :user_permissions,
:description_html, :blob]
expect(described_class).to have_graphql_fields(*expected_fields)
end
@ -17,8 +20,55 @@ describe GitlabSchema.types['Snippet'] do
it { expect(described_class).to require_graphql_authorizations(:read_snippet) }
end
shared_examples 'response without repository URLs' do
it 'does not respond with repository URLs' do
expect(response['sshUrlToRepo']).to be_nil
expect(response['httpUrlToRepo']).to be_nil
end
end
describe 'Repository URLs' do
let(:query) do
%(
{
snippets {
nodes {
sshUrlToRepo
httpUrlToRepo
}
}
}
)
end
let(:response) { subject.dig('data', 'snippets', 'nodes')[0] }
subject { GitlabSchema.execute(query, context: { current_user: user }).as_json }
context 'when snippet has repository' do
let!(:snippet) { create(:personal_snippet, :repository, :public, author: user) }
it 'responds with repository URLs' do
expect(response['sshUrlToRepo']).to eq(snippet.ssh_url_to_repo)
expect(response['httpUrlToRepo']).to eq(snippet.http_url_to_repo)
end
context 'when version_snippets feature is disabled' do
before do
stub_feature_flags(version_snippets: false)
end
it_behaves_like 'response without repository URLs'
end
end
context 'when snippet does not have a repository' do
let!(:snippet) { create(:personal_snippet, :public, author: user) }
it_behaves_like 'response without repository URLs'
end
end
describe '#blob' do
let_it_be(:user) { create(:user) }
let(:query_blob) { subject.dig('data', 'snippets', 'edges')[0]['node']['blob'] }
let(:query) do
%(

View File

@ -74,6 +74,16 @@ describe Gitlab::Metrics::Dashboard::Processor do
expect(actual_metrics_order).to eq expected_metrics_order
end
context 'when the project has multiple metrics in the same group' do
let!(:project_response_metric) { create(:prometheus_metric, project: project, group: :response) }
let!(:project_response_metric_2) { create(:prometheus_metric, project: project, group: :response) }
it 'includes multiple metrics' do
expect(all_metrics).to include get_metric_details(project_response_metric)
expect(all_metrics).to include get_metric_details(project_response_metric_2)
end
end
context 'when the dashboard should not include project metrics' do
let(:sequence) do
[

View File

@ -140,13 +140,10 @@ describe Clusters::Applications::Ingress do
end
describe '#values' do
let(:project) { build(:project) }
let(:cluster) { build(:cluster, projects: [project]) }
subject { ingress }
context 'when modsecurity_enabled is enabled' do
before do
allow(subject).to receive(:cluster).and_return(cluster)
allow(subject).to receive(:modsecurity_enabled).and_return(true)
end
@ -154,8 +151,24 @@ describe Clusters::Applications::Ingress do
expect(subject.values).to include("enable-modsecurity: 'true'")
end
it 'includes modsecurity core ruleset enablement' do
expect(subject.values).to include("enable-owasp-modsecurity-crs: 'true'")
it 'includes modsecurity core ruleset enablement set to false' do
expect(subject.values).to include("enable-owasp-modsecurity-crs: 'false'")
end
it 'includes modsecurity snippet with information related to security rules' do
expect(subject.values).to include("SecRuleEngine DetectionOnly")
expect(subject.values).to include("Include #{described_class::MODSECURITY_OWASP_RULES_FILE}")
end
context 'when modsecurity_mode is set to :blocking' do
before do
subject.blocking!
end
it 'includes modsecurity snippet with information related to security rules' do
expect(subject.values).to include("SecRuleEngine On")
expect(subject.values).to include("Include #{described_class::MODSECURITY_OWASP_RULES_FILE}")
end
end
it 'includes modsecurity.conf content' do
@ -176,7 +189,6 @@ describe Clusters::Applications::Ingress do
context 'when modsecurity_enabled is disabled' do
before do
allow(subject).to receive(:cluster).and_return(cluster)
allow(subject).to receive(:modsecurity_enabled).and_return(false)
end

View File

@ -713,4 +713,32 @@ describe Snippet do
it { is_expected.to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "#{snippet.project.full_path}/snippets/#{snippet.id}.git") }
end
end
describe '#versioned_enabled_for?' do
let_it_be(:user) { create(:user) }
subject { snippet.versioned_enabled_for?(user) }
context 'with repository and version_snippets enabled' do
let!(:snippet) { create(:personal_snippet, :repository, author: user) }
it { is_expected.to be_truthy }
end
context 'without repository' do
let!(:snippet) { create(:personal_snippet, author: user) }
it { is_expected.to be_falsy }
end
context 'without version_snippets feature disabled' do
let!(:snippet) { create(:personal_snippet, :repository, author: user) }
before do
stub_feature_flags(version_snippets: false)
end
it { is_expected.to be_falsy }
end
end
end

View File

@ -85,7 +85,7 @@ describe API::ProjectSnippets do
describe 'GET /projects/:project_id/snippets/:id' do
let(:user) { create(:user) }
let(:snippet) { create(:project_snippet, :public, project: project) }
let(:snippet) { create(:project_snippet, :public, :repository, project: project) }
it 'returns snippet json' do
get api("/projects/#{project.id}/snippets/#{snippet.id}", user)
@ -95,6 +95,18 @@ describe API::ProjectSnippets do
expect(json_response['title']).to eq(snippet.title)
expect(json_response['description']).to eq(snippet.description)
expect(json_response['file_name']).to eq(snippet.file_name)
expect(json_response['ssh_url_to_repo']).to eq(snippet.ssh_url_to_repo)
expect(json_response['http_url_to_repo']).to eq(snippet.http_url_to_repo)
end
context 'when feature flag :version_snippets is disabled' do
before do
stub_feature_flags(version_snippets: false)
get api("/projects/#{project.id}/snippets/#{snippet.id}", user)
end
it_behaves_like 'snippet response without repository URLs'
end
it 'returns 404 for invalid snippet id' do

View File

@ -139,8 +139,8 @@ describe API::Snippets do
describe 'GET /snippets/:id' do
let_it_be(:admin) { create(:user, :admin) }
let_it_be(:author) { create(:user) }
let_it_be(:private_snippet) { create(:personal_snippet, :private, author: author) }
let_it_be(:internal_snippet) { create(:personal_snippet, :internal, author: author) }
let_it_be(:private_snippet) { create(:personal_snippet, :repository, :private, author: author) }
let_it_be(:internal_snippet) { create(:personal_snippet, :repository, :internal, author: author) }
it 'requires authentication' do
get api("/snippets/#{private_snippet.id}", nil)
@ -157,6 +157,18 @@ describe API::Snippets do
expect(json_response['description']).to eq(private_snippet.description)
expect(json_response['file_name']).to eq(private_snippet.file_name)
expect(json_response['visibility']).to eq(private_snippet.visibility)
expect(json_response['ssh_url_to_repo']).to eq(private_snippet.ssh_url_to_repo)
expect(json_response['http_url_to_repo']).to eq(private_snippet.http_url_to_repo)
end
context 'when feature flag :version_snippets is disabled' do
before do
stub_feature_flags(version_snippets: false)
get api("/snippets/#{private_snippet.id}", author)
end
it_behaves_like 'snippet response without repository URLs'
end
it 'shows private snippets to an admin' do

View File

@ -41,3 +41,10 @@ RSpec.shared_examples 'update with repository actions' do
end
end
end
RSpec.shared_examples 'snippet response without repository URLs' do
it 'skip inclusion of repository URLs' do
expect(json_response).not_to have_key('ssh_url_to_repo')
expect(json_response).not_to have_key('http_url_to_repo')
end
end