kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-07-25 03:10:12 +00:00
parent 3fad3677d7
commit 5b9a8005ea
27 changed files with 343 additions and 397 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
data/deprecations/15-3-pipeline_activity_limit.yml Normal file
View File

@ -0,0 +1,33 @@
# This is a template for a feature deprecation.
#
# Please refer to the deprecation guidelines to confirm your understanding of GitLab's definitions.
# https://docs.gitlab.com/ee/development/deprecation_guidelines/#terminology
#
# Deprecations must be announced at least three releases prior to removal.
#
# If an End of Support period applies, the announcement should be shared with GitLab Support
# in the `#spt_managers` on Slack and mention `@gitlab-com/support` in this MR.
#
# Breaking changes must happen in a major release.
#
# For more information please refer to the handbook documentation here:
# https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations
#
# Please delete this line and above before submitting your merge request.
#
# REQUIRED FIELDS
#
- name: "Maximum number of active pipelines per project limit (`ci_active_pipelines`)" # (required) The name of the feature to be deprecated
announcement_milestone: "15.3" # (required) The milestone when this feature was first announced as deprecated.
announcement_date: "2022-08-22" # (required) The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post.
removal_milestone: "16.0" # (required) The milestone when this feature is planned to be removed
removal_date: "2023-05-22" # (required) The date of the milestone release when this feature is planned to be removed. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post.
breaking_change: false # (required) If this deprecation is a breaking change, set this value to true
reporter: jheimbuck_gl # (required) GitLab username of the person reporting the deprecation
stage: Verify # (required) String value of the stage that the feature was created in. e.g., Growth
issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/368195 # (required) Link to the deprecation issue in GitLab
body: | # (required) Do not modify this line, instead modify the lines below.
The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) was never enabled by default and will be removed in GitLab 16.0. This limit can also be configured in the rails console under [`ci_active_pipelines`](https://docs.gitlab.com/ee/administration/instance_limits.html#number-of-pipelines-running-concurrently). Instead, use the other recommended rate limits that offer similar protection:
- [**Pipelines rate limits**](https://docs.gitlab.com/ee/user/admin_area/settings/rate_limit_on_pipelines_creation.html).
- [**Total number of jobs in currently active pipelines**](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits).

9
db/migrate/20220720161816_add_namespace_id_to_vulnerability_reads_table.rb Normal file
View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddNamespaceIdToVulnerabilityReadsTable < Gitlab::Database::Migration[2.0]
enable_lock_retries!
def change
add_column :vulnerability_reads, :namespace_id, :bigint
end
end

159
db/migrate/20220720162123_update_vulnerability_reads_triggers_to_set_namespace_id.rb Normal file
View File

@ -0,0 +1,159 @@
# frozen_string_literal: true
class UpdateVulnerabilityReadsTriggersToSetNamespaceId < Gitlab::Database::Migration[2.0]
enable_lock_retries!
def up
execute(<<~SQL)
CREATE OR REPLACE FUNCTION insert_or_update_vulnerability_reads() RETURNS trigger
LANGUAGE plpgsql
AS $$
DECLARE
severity smallint;
state smallint;
report_type smallint;
resolved_on_default_branch boolean;
present_on_default_branch boolean;
namespace_id bigint;
BEGIN
IF (NEW.vulnerability_id IS NULL AND (TG_OP = 'INSERT' OR TG_OP = 'UPDATE')) THEN
RETURN NULL;
END IF;
IF (TG_OP = 'UPDATE' AND OLD.vulnerability_id IS NOT NULL AND NEW.vulnerability_id IS NOT NULL) THEN
RETURN NULL;
END IF;
SELECT
vulnerabilities.severity, vulnerabilities.state, vulnerabilities.report_type, vulnerabilities.resolved_on_default_branch, vulnerabilities.present_on_default_branch
INTO
severity, state, report_type, resolved_on_default_branch, present_on_default_branch
FROM
vulnerabilities
WHERE
vulnerabilities.id = NEW.vulnerability_id;
IF present_on_default_branch IS NOT true THEN
RETURN NULL;
END IF;
SELECT
projects.namespace_id
INTO
namespace_id
FROM
projects
WHERE
projects.id = NEW.project_id;
INSERT INTO vulnerability_reads (vulnerability_id, namespace_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.vulnerability_id, namespace_id, NEW.project_id, NEW.scanner_id, report_type, severity, state, resolved_on_default_branch, NEW.uuid::uuid, NEW.location->>'image', NEW.location->'kubernetes_resource'->>'agent_id', CAST(NEW.location->'kubernetes_resource'->>'agent_id' AS bigint))
ON CONFLICT(vulnerability_id) DO NOTHING;
RETURN NULL;
END
$$;
SQL
execute(<<~SQL)
CREATE OR REPLACE FUNCTION insert_vulnerability_reads_from_vulnerability() RETURNS trigger
LANGUAGE plpgsql
AS $$
DECLARE
scanner_id bigint;
uuid uuid;
location_image text;
cluster_agent_id text;
casted_cluster_agent_id bigint;
namespace_id bigint;
BEGIN
SELECT
v_o.scanner_id, v_o.uuid, v_o.location->>'image', v_o.location->'kubernetes_resource'->>'agent_id', CAST(v_o.location->'kubernetes_resource'->>'agent_id' AS bigint), projects.namespace_id
INTO
scanner_id, uuid, location_image, cluster_agent_id, casted_cluster_agent_id, namespace_id
FROM
vulnerability_occurrences v_o
INNER JOIN projects ON projects.id = v_o.project_id
WHERE
v_o.vulnerability_id = NEW.id
LIMIT 1;
INSERT INTO vulnerability_reads (vulnerability_id, namespace_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.id, namespace_id, NEW.project_id, scanner_id, NEW.report_type, NEW.severity, NEW.state, NEW.resolved_on_default_branch, uuid::uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
ON CONFLICT(vulnerability_id) DO NOTHING;
RETURN NULL;
END
$$;
SQL
end
def down
execute(<<~SQL)
CREATE OR REPLACE FUNCTION insert_or_update_vulnerability_reads() RETURNS trigger
LANGUAGE plpgsql
AS $$
DECLARE
severity smallint;
state smallint;
report_type smallint;
resolved_on_default_branch boolean;
present_on_default_branch boolean;
BEGIN
IF (NEW.vulnerability_id IS NULL AND (TG_OP = 'INSERT' OR TG_OP = 'UPDATE')) THEN
RETURN NULL;
END IF;
IF (TG_OP = 'UPDATE' AND OLD.vulnerability_id IS NOT NULL AND NEW.vulnerability_id IS NOT NULL) THEN
RETURN NULL;
END IF;
SELECT
vulnerabilities.severity, vulnerabilities.state, vulnerabilities.report_type, vulnerabilities.resolved_on_default_branch, vulnerabilities.present_on_default_branch
INTO
severity, state, report_type, resolved_on_default_branch, present_on_default_branch
FROM
vulnerabilities
WHERE
vulnerabilities.id = NEW.vulnerability_id;
IF present_on_default_branch IS NOT true THEN
RETURN NULL;
END IF;
INSERT INTO vulnerability_reads (vulnerability_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.vulnerability_id, NEW.project_id, NEW.scanner_id, report_type, severity, state, resolved_on_default_branch, NEW.uuid::uuid, NEW.location->>'image', NEW.location->'kubernetes_resource'->>'agent_id', CAST(NEW.location->'kubernetes_resource'->>'agent_id' AS bigint))
ON CONFLICT(vulnerability_id) DO NOTHING;
RETURN NULL;
END
$$;
SQL
execute(<<~SQL)
CREATE OR REPLACE FUNCTION insert_vulnerability_reads_from_vulnerability() RETURNS trigger
LANGUAGE plpgsql
AS $$
DECLARE
scanner_id bigint;
uuid uuid;
location_image text;
cluster_agent_id text;
casted_cluster_agent_id bigint;
BEGIN
SELECT
v_o.scanner_id, v_o.uuid, v_o.location->>'image', v_o.location->'kubernetes_resource'->>'agent_id', CAST(v_o.location->'kubernetes_resource'->>'agent_id' AS bigint)
INTO
scanner_id, uuid, location_image, cluster_agent_id, casted_cluster_agent_id
FROM
vulnerability_occurrences v_o
WHERE
v_o.vulnerability_id = NEW.id
LIMIT 1;
INSERT INTO vulnerability_reads (vulnerability_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.id, NEW.project_id, scanner_id, NEW.report_type, NEW.severity, NEW.state, NEW.resolved_on_default_branch, uuid::uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
ON CONFLICT(vulnerability_id) DO NOTHING;
RETURN NULL;
END
$$;
SQL
end
end

18
db/post_migrate/20220720171102_add_index_on_vulnerability_reads_namespace_id.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class AddIndexOnVulnerabilityReadsNamespaceId < Gitlab::Database::Migration[2.0]
INDEX_NAME = 'index_vuln_reads_on_namespace_id_state_severity_and_vuln_id'
disable_ddl_transaction!
def up
add_concurrent_index :vulnerability_reads,
[:namespace_id, :state, :severity, :vulnerability_id],
name: INDEX_NAME,
order: { vulnerability_id: :desc }
end
def down
remove_concurrent_index_by_name :vulnerability_reads, INDEX_NAME
end
end

15
db/post_migrate/20220720171801_add_foreign_key_on_vulnerability_reads_namespace_id.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
class AddForeignKeyOnVulnerabilityReadsNamespaceId < Gitlab::Database::Migration[2.0]
disable_ddl_transaction!
def up
add_concurrent_foreign_key :vulnerability_reads, :namespaces, column: :namespace_id, on_delete: :cascade
end
def down
with_lock_retries do
remove_foreign_key_if_exists :vulnerability_reads, column: :namespace_id
end
end
end

1
db/schema_migrations/20220720161816 Normal file
View File

@ -0,0 +1 @@
8933e0dfff73ebef359de9512e8a89d43da37991b03500aac1d1a1ba78d6a4a9

1
db/schema_migrations/20220720162123 Normal file
View File

@ -0,0 +1 @@
1b8fd9394fefd20a8a0ad281f10b4006063e436e3e41d5651e84b43dccd9c221

1
db/schema_migrations/20220720171102 Normal file
View File

@ -0,0 +1 @@
4daf9a0b571968640c4b6ddfa6dbc2d71ed1632b4dc4d350cd5cab4ae53c8178

1
db/schema_migrations/20220720171801 Normal file
View File

@ -0,0 +1 @@
161186e3bb37a3c03f7e6ec5f833e86bcc163af33847666006e9a0cf814eb876

32
db/structure.sql
View File

@ -54,6 +54,7 @@ DECLARE
report_type smallint;
resolved_on_default_branch boolean;
present_on_default_branch boolean;
namespace_id bigint;
BEGIN
IF (NEW.vulnerability_id IS NULL AND (TG_OP = 'INSERT' OR TG_OP = 'UPDATE')) THEN
RETURN NULL;
@ -76,8 +77,17 @@ BEGIN
RETURN NULL;
END IF;
INSERT INTO vulnerability_reads (vulnerability_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.vulnerability_id, NEW.project_id, NEW.scanner_id, report_type, severity, state, resolved_on_default_branch, NEW.uuid::uuid, NEW.location->>'image', NEW.location->'kubernetes_resource'->>'agent_id', CAST(NEW.location->'kubernetes_resource'->>'agent_id' AS bigint))
SELECT
projects.namespace_id
INTO
namespace_id
FROM
projects
WHERE
projects.id = NEW.project_id;
INSERT INTO vulnerability_reads (vulnerability_id, namespace_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.vulnerability_id, namespace_id, NEW.project_id, NEW.scanner_id, report_type, severity, state, resolved_on_default_branch, NEW.uuid::uuid, NEW.location->>'image', NEW.location->'kubernetes_resource'->>'agent_id', CAST(NEW.location->'kubernetes_resource'->>'agent_id' AS bigint))
ON CONFLICT(vulnerability_id) DO NOTHING;
RETURN NULL;
END
@ -103,19 +113,21 @@ DECLARE
location_image text;
cluster_agent_id text;
casted_cluster_agent_id bigint;
namespace_id bigint;
BEGIN
SELECT
v_o.scanner_id, v_o.uuid, v_o.location->>'image', v_o.location->'kubernetes_resource'->>'agent_id', CAST(v_o.location->'kubernetes_resource'->>'agent_id' AS bigint)
v_o.scanner_id, v_o.uuid, v_o.location->>'image', v_o.location->'kubernetes_resource'->>'agent_id', CAST(v_o.location->'kubernetes_resource'->>'agent_id' AS bigint), projects.namespace_id
INTO
scanner_id, uuid, location_image, cluster_agent_id, casted_cluster_agent_id
scanner_id, uuid, location_image, cluster_agent_id, casted_cluster_agent_id, namespace_id
FROM
vulnerability_occurrences v_o
vulnerability_occurrences v_o
INNER JOIN projects ON projects.id = v_o.project_id
WHERE
v_o.vulnerability_id = NEW.id
LIMIT 1;
INSERT INTO vulnerability_reads (vulnerability_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.id, NEW.project_id, scanner_id, NEW.report_type, NEW.severity, NEW.state, NEW.resolved_on_default_branch, uuid::uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
INSERT INTO vulnerability_reads (vulnerability_id, namespace_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.id, namespace_id, NEW.project_id, scanner_id, NEW.report_type, NEW.severity, NEW.state, NEW.resolved_on_default_branch, uuid::uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
ON CONFLICT(vulnerability_id) DO NOTHING;
RETURN NULL;
END
@ -22357,6 +22369,7 @@ CREATE TABLE vulnerability_reads (
location_image text,
cluster_agent_id text,
casted_cluster_agent_id bigint,
namespace_id bigint,
CONSTRAINT check_380451bdbe CHECK ((char_length(location_image) <= 2048)),
CONSTRAINT check_a105eb825a CHECK ((char_length(cluster_agent_id) <= 10))
);
@ -30044,6 +30057,8 @@ CREATE UNIQUE INDEX index_vuln_historical_statistics_on_project_id_and_date ON v
CREATE INDEX index_vuln_reads_on_casted_cluster_agent_id_where_it_is_null ON vulnerability_reads USING btree (casted_cluster_agent_id) WHERE (casted_cluster_agent_id IS NOT NULL);
CREATE INDEX index_vuln_reads_on_namespace_id_state_severity_and_vuln_id ON vulnerability_reads USING btree (namespace_id, state, severity, vulnerability_id DESC);
CREATE INDEX index_vuln_reads_on_project_id_state_severity_and_vuln_id ON vulnerability_reads USING btree (project_id, state, severity, vulnerability_id DESC);
CREATE INDEX index_vulnerabilites_common_finder_query ON vulnerabilities USING btree (project_id, state, report_type, severity, id);
@ -31947,6 +31962,9 @@ ALTER TABLE ONLY user_namespace_callouts
ALTER TABLE ONLY sbom_occurrences
ADD CONSTRAINT fk_4b88e5b255 FOREIGN KEY (component_version_id) REFERENCES sbom_component_versions(id) ON DELETE CASCADE;
ALTER TABLE ONLY vulnerability_reads
ADD CONSTRAINT fk_4f593f6c62 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
ALTER TABLE ONLY vulnerability_reads
ADD CONSTRAINT fk_5001652292 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;

14
doc/administration/monitoring/prometheus/index.md
View File

@ -465,3 +465,17 @@ If you are using Prometheus monitoring:
```
1. Optional. [Configure the storage retention size](index.md#configure-the-storage-retention-size).
### Monitoring node not receiving data
If the monitoring node is not receiving any data, check that the exporters are capturing data:
```shell
curl "http[s]://localhost:<EXPORTER LISTENING PORT>/metric"
```
or
```shell
curl "http[s]://localhost:<EXPORTER LISTENING PORT>/-/metric"
```

5
doc/administration/reference_architectures/10k_users.md
View File

@ -693,7 +693,8 @@ SSH in to any of the Patroni nodes on the **primary site**:
```
If the 'State' column for any node doesn't say "running", check the
[Troubleshooting section](troubleshooting.md) before proceeding.
[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server)
before proceeding.
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">
@ -2086,7 +2087,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en
failing to connect to PostgreSQL, it may be that your PgBouncer node's IP
address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb`
on your database nodes. Before proceeding, see
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md).

5
doc/administration/reference_architectures/25k_users.md
View File

@ -696,7 +696,8 @@ SSH in to any of the Patroni nodes on the **primary site**:
```
If the 'State' column for any node doesn't say "running", check the
[Troubleshooting section](troubleshooting.md) before proceeding.
[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server)
before proceeding.
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">
@ -2091,7 +2092,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en
failing to connect to PostgreSQL, it may be that your PgBouncer node's IP
address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb`
on your database nodes. Before proceeding, see
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md).

2
doc/administration/reference_architectures/2k_users.md
View File

@ -766,7 +766,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en
failing to connect to PostgreSQL, it may be that your PgBouncer node's IP
address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb`
on your database nodes. Before proceeding, see
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md).

5
doc/administration/reference_architectures/3k_users.md
View File

@ -978,7 +978,8 @@ SSH in to any of the Patroni nodes on the **primary site**:
```
If the 'State' column for any node doesn't say "running", check the
[Troubleshooting section](troubleshooting.md) before proceeding.
[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server)
before proceeding.
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">
@ -2018,7 +2019,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en
failing to connect to PostgreSQL, it may be that your PgBouncer node's IP
address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb`
on your database nodes. Before proceeding, see
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md).

5
doc/administration/reference_architectures/50k_users.md
View File

@ -703,7 +703,8 @@ SSH in to any of the Patroni nodes on the **primary site**:
```
If the 'State' column for any node doesn't say "running", check the
[Troubleshooting section](troubleshooting.md) before proceeding.
[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server)
before proceeding.
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">
@ -2107,7 +2108,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en
failing to connect to PostgreSQL, it may be that your PgBouncer node's IP
address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb`
on your database nodes. Before proceeding, see
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md).

5
doc/administration/reference_architectures/5k_users.md
View File

@ -975,7 +975,8 @@ SSH in to any of the Patroni nodes on the **primary site**:
```
If the 'State' column for any node doesn't say "running", check the
[Troubleshooting section](troubleshooting.md) before proceeding.
[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server)
before proceeding.
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">
@ -2018,7 +2019,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en
failing to connect to PostgreSQL, it may be that your PgBouncer node's IP
address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb`
on your database nodes. Before proceeding, see
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
[PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server).
1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md).

355
doc/administration/reference_architectures/troubleshooting.md
View File

@ -1,352 +1,11 @@
---
stage: Systems
group: Distribution
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
redirect_to: '../configure.md'
remove_date: '2022-10-24'
---
# Troubleshooting a reference architecture setup **(FREE SELF)**
This document was moved to [another location](../configure.md).
This page serves as the troubleshooting documentation if you followed one of
the [reference architectures](index.md#reference-architectures).
## Troubleshooting object storage
### S3 API compatibility issues
Not all S3 providers [are fully compatible](../../raketasks/backup_gitlab.md#other-s3-providers)
with the Fog library that GitLab uses. Symptoms include:
```plaintext
411 Length Required
```
### GitLab Pages can use object storage
If you intend to use [GitLab Pages](../../user/project/pages/index.md), you can
[configure object storage](../pages/index.md#using-object-storage).
NFS is still available if you prefer.
The [GitLab Pages Helm chart](https://docs.gitlab.com/charts/charts/gitlab/gitlab-pages/) is also available
for Kubernetes deployments.
### Incremental logging is required for CI to use object storage
If you configure GitLab to use object storage for CI logs and artifacts,
[you must also enable incremental logging](../job_logs.md#incremental-logging-architecture).
### Proxy Download
A number of the use cases for object storage allow client traffic to be redirected to the
object storage back end, like when Git clients request large files via LFS or when
downloading CI artifacts and logs.
When the files are stored on local block storage or NFS, GitLab has to act as a proxy.
With object storage, the default behavior is for GitLab to redirect to the object
storage device rather than proxy the request.
The `proxy_download` setting controls this behavior: the default is generally `false`.
Verify this in the documentation for each use case. Set it to `true` to make
GitLab proxy the files rather than redirect.
When not proxying files, GitLab returns an
[HTTP 302 redirect with a pre-signed, time-limited object storage URL](https://gitlab.com/gitlab-org/gitlab/-/issues/32117#note_218532298).
This can result in some of the following problems:
- If GitLab is using non-secure HTTP to access the object storage, clients may generate
`https->http` downgrade errors and refuse to process the redirect. The solution to this
is for GitLab to use HTTPS. LFS, for example, generates this error:
```plaintext
LFS: lfsapi/client: refusing insecure redirect, https->http
```
- Clients must trust the certificate authority that issued the object storage
certificate, or may return common TLS errors such as:
```plaintext
x509: certificate signed by unknown authority
```
- Clients need network access to the object storage. Errors that might result
if this access is not in place include:
```plaintext
Received status code 403 from server: Forbidden
```
### ETag mismatch
Using the default GitLab settings, some object storage back-ends such as
[MinIO](https://gitlab.com/gitlab-org/gitlab/-/issues/23188)
and [Alibaba](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564)
might generate `ETag mismatch` errors.
When using GitLab direct upload, the
[workaround for MinIO](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564#note_244497658)
is to use the `--compat` parameter on the server.
We are working on a fix to GitLab component Workhorse, and also
a workaround, in the mean time, to
[allow ETag verification to be disabled](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18175).
## Troubleshooting Redis
There are a lot of moving parts that needs to be taken care carefully
in order for the HA setup to work as expected.
Before proceeding with the troubleshooting below, check your firewall rules:
- Redis machines
- Accept TCP connection in `6379`
- Connect to the other Redis machines via TCP in `6379`
- Sentinel machines
- Accept TCP connection in `26379`
- Connect to other Sentinel machines via TCP in `26379`
- Connect to the Redis machines via TCP in `6379`
### Troubleshooting Redis replication
You can check if everything is correct by connecting to each server using
`redis-cli` application, and sending the `info replication` command as below.
```shell
/opt/gitlab/embedded/bin/redis-cli -h <redis-host-or-ip> -a '<redis-password>' info replication
```
When connected to a `Primary` Redis, you see the number of connected
`replicas`, and a list of each with connection details:
```plaintext
# Replication
role:master
connected_replicas:1
replica0:ip=10.133.5.21,port=6379,state=online,offset=208037514,lag=1
master_repl_offset:208037658
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:206989083
repl_backlog_histlen:1048576
```
When it's a `replica`, you see details of the primary connection and if
its `up` or `down`:
```plaintext
# Replication
role:replica
master_host:10.133.1.58
master_port:6379
master_link_status:up
master_last_io_seconds_ago:1
master_sync_in_progress:0
replica_repl_offset:208096498
replica_priority:100
replica_read_only:1
connected_replicas:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
```
### Troubleshooting Sentinel
If you get an error like: `Redis::CannotConnectError: No sentinels available.`,
there may be something wrong with your configuration files or it can be related
to [this issue](https://github.com/redis/redis-rb/issues/531).
You must make sure you are defining the same value in `redis['master_name']`
and `redis['master_password']` as you defined for your sentinel node.
The way the Redis connector `redis-rb` works with sentinel is a bit
non-intuitive. We try to hide the complexity in omnibus, but it still requires
a few extra configurations.
---
To make sure your configuration is correct:
1. SSH into your GitLab application server
1. Enter the Rails console:
```shell
# For Omnibus installations
sudo gitlab-rails console
# For source installations
sudo -u git rails console -e production
```
1. Run in the console:
```ruby
redis = Redis.new(Gitlab::Redis::SharedState.params)
redis.info
```
Keep this screen open and try to simulate a failover below.
1. To simulate a failover on primary Redis, SSH into the Redis server and run:
```shell
# port must match your primary redis port, and the sleep time must be a few seconds bigger than defined one
redis-cli -h localhost -p 6379 DEBUG sleep 20
```
1. Then back in the Rails console from the first step, run:
```ruby
redis.info
```
You should see a different port after a few seconds delay
(the failover/reconnect time).
## Troubleshooting Gitaly
For troubleshooting information, see Gitaly and Gitaly Cluster
[troubleshooting information](../gitaly/troubleshooting.md).
## Troubleshooting the GitLab Rails application
- `mount: wrong fs type, bad option, bad superblock on`
You have not installed the necessary NFS client utilities. See step 1 above.
- `mount: mount point /var/opt/gitlab/... does not exist`
This particular directory does not exist on the NFS server. Ensure
the share is exported and exists on the NFS server and try to remount.
## Troubleshooting Monitoring
If the monitoring node is not receiving any data, check that the exporters are
capturing data.
```shell
curl "http[s]://localhost:<EXPORTER LISTENING PORT>/metric"
```
or
```shell
curl "http[s]://localhost:<EXPORTER LISTENING PORT>/-/metric"
```
## Troubleshooting PgBouncer
In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs:
```shell
sudo gitlab-ctl tail pgbouncer
```
Additionally, you can check the output from `show databases` in the [administrative console](#pgbouncer-administrative-console). In the output, you would expect to see values in the `host` field for the `gitlabhq_production` database. Additionally, `current_connections` should be greater than 1.
### PgBouncer administrative console
As part of Omnibus GitLab, the `gitlab-ctl pgb-console` command is provided to automatically connect to the PgBouncer administrative console. See the [PgBouncer documentation](https://www.pgbouncer.org/usage.html#admin-console) for detailed instructions on how to interact with the console.
To start a session:
```shell
sudo gitlab-ctl pgb-console
```
The password you are prompted for is the `pgbouncer_user_password`
To get some basic information about the instance, run
```shell
pgbouncer=# show databases; show clients; show servers;
name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections
---------------------+-----------+------+---------------------+------------+-----------+--------------+-----------+-----------------+---------------------
gitlabhq_production | 127.0.0.1 | 5432 | gitlabhq_production | | 100 | 5 | | 0 | 1
pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0
(2 rows)
type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link
| remote_pid | tls
------+-----------+---------------------+--------+-----------+-------+------------+------------+---------------------+---------------------+-----------+------
+------------+-----
C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44590 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12444c0 |
| 0 |
C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44592 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12447c0 |
| 0 |
C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44594 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x1244940 |
| 0 |
C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44706 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:16:31 | 0x1244ac0 |
| 0 |
C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44708 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:15:15 | 0x1244c40 |
| 0 |
C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44794 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:15:15 | 0x1244dc0 |
| 0 |
C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44798 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:16:31 | 0x1244f40 |
| 0 |
C | pgbouncer | pgbouncer | active | 127.0.0.1 | 44660 | 127.0.0.1 | 6432 | 2018-04-24 22:13:51 | 2018-04-24 22:17:12 | 0x1244640 |
| 0 |
(8 rows)
type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | rem
ote_pid | tls
------+--------+---------------------+-------+-----------+------+------------+------------+---------------------+---------------------+-----------+------+----
--------+-----
S | gitlab | gitlabhq_production | idle | 127.0.0.1 | 5432 | 127.0.0.1 | 35646 | 2018-04-24 22:15:15 | 2018-04-24 22:17:10 | 0x124dca0 | |
19980 |
(1 row)
```
### Message: `LOG: invalid CIDR mask in address`
See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-cidr-mask-in-address).
### Message: `LOG: invalid IP mask "md5": Name or service not known`
See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-ip-mask-md5-name-or-service-not-known).
## Troubleshooting PostgreSQL with Patroni
In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs for PostgreSQL (which is run through Patroni):
```shell
sudo gitlab-ctl tail patroni
```
### Consul and PostgreSQL with Patroni changes not taking effect
Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it does not restart the services. However, not all changes can be activated by reloading.
To restart either service, run `gitlab-ctl restart consul` or `gitlab-ctl restart patroni` respectively.
For PostgreSQL with Patroni, to prevent the primary node from being failed over automatically, it's safest to stop all secondaries first, then restart the primary and finally restart the secondaries again.
On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](../consul.md#restart-consul) for instructions on how to restart the service.
### PgBouncer error `ERROR: pgbouncer cannot connect to server`
You may get this error when running `gitlab-rake gitlab:db:configure` or you
may see the error in the PgBouncer log file.
```plaintext
PG::ConnectionBad: ERROR: pgbouncer cannot connect to server
```
The problem may be that your PgBouncer node's IP address is not included in the
`trust_auth_cidr_addresses` setting in `/etc/gitlab/gitlab.rb` on the database nodes.
You can confirm that this is the issue by checking the PostgreSQL log on the primary
database node. If you see the following error then `trust_auth_cidr_addresses`
is the problem.
```plaintext
2018-03-29_13:59:12.11776 FATAL: no pg_hba.conf entry for host "123.123.123.123", user "pgbouncer", database "gitlabhq_production", SSL off
```
To fix the problem, add the IP address to `/etc/gitlab/gitlab.rb`.
```ruby
postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>)
```
[Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
<!-- This redirect file can be deleted after <2022-10-24>. -->
<!-- Redirects that point to other docs in the same project expire in three months. -->
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->

2
doc/api/pipelines.md
View File

@ -281,7 +281,7 @@ POST /projects/:id/pipeline
|-------------|---------|----------|---------------------|
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user |
| `ref` | string | yes | The branch or tag to run the pipeline on. |
| `variables` | array | no | An array containing the variables available in the pipeline, matching the structure `[{ 'key': 'UPLOAD_TO_S3', 'variable_type': 'file', 'value': 'true' }, {'key': 'TEST', 'value': 'test variable'}]`. If `variable_type` is excluded, it defaults to `env_var`. |
| `variables` | array | no | An [array of hashes](index.md#array-of-hashes) containing the variables available in the pipeline, matching the structure `[{ 'key': 'UPLOAD_TO_S3', 'variable_type': 'file', 'value': 'true' }, {'key': 'TEST', 'value': 'test variable'}]`. If `variable_type` is excluded, it defaults to `env_var`. |
```shell
curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/pipeline?ref=main"

2
doc/api/settings.md
View File

@ -403,7 +403,7 @@ listed in the descriptions of the relevant settings.
| `performance_bar_enabled` | boolean | no | (Deprecated: Pass `performance_bar_allowed_group_path: nil` instead) Allow enabling the performance bar. |
| `personal_access_token_prefix` | string | no | Prefix for all generated personal access tokens. |
| `pipeline_limit_per_project_user_sha` | integer | no | Maximum number of pipeline creation requests per minute per user and commit. Disabled by default. |
| `plantuml_enabled` | boolean | no | (**If enabled, requires:** `plantuml_url`) Enable PlantUML integration. Default is `false`. |
| `plantuml_enabled` | boolean | no | (**If enabled, requires:** `plantuml_url`) Enable [PlantUML integration](../administration/integration/plantuml.md). Default is `false`. |
| `plantuml_url` | string | required by: `plantuml_enabled` | The PlantUML instance URL for integration. |
| `polling_interval_multiplier` | decimal | no | Interval multiplier used by endpoints that perform polling. Set to `0` to disable polling. |
| `project_export_enabled` | boolean | no | Enable project export. |

2
doc/ci/secure_files/index.md
View File

@ -36,7 +36,7 @@ To add a secure file to a project:
1. On the top bar, select **Menu > Projects** and find your project.
1. On the left sidebar, select **Settings > CI/CD**.
1. In the **Secure Files** section, select **Manage**.
1. In the **Secure Files** section, select **Expand**.
1. Select **Upload File**.
1. Find the file to upload, select **Open**, and the file upload begins immediately.
The file shows up in the list when the upload is complete.

38
doc/development/go_guide/index.md
View File

@ -448,36 +448,28 @@ up to run `goimports -local gitlab.com/gitlab-org` so that it's applied to every
If initializing a slice, provide a capacity where possible to avoid extra
allocations.
<table>
<tr><th>:white_check_mark: Do</th><th>:x: Don't</th></tr>
<tr>
<td>
**Don't:**
```golang
s2 := make([]string, 0, size)
for _, val := range s1 {
s2 = append(s2, val)
}
```
```golang
var s2 []string
for _, val := range s1 {
s2 = append(s2, val)
}
```
</td>
<td>
**Do:**
```golang
var s2 []string
for _, val := range s1 {
s2 = append(s2, val)
}
```
</td>
</tr>
</table>
```golang
s2 := make([]string, 0, size)
for _, val := range s1 {
s2 = append(s2, val)
}
```
If no capacity is passed to `make` when creating a new slice, `append`
will continuously resize the slice's backing array if it cannot hold
the values. Providing the capacity ensures that allocations are kept
to a minimum. It is recommended that the [`prealloc`](https://github.com/alexkohler/prealloc)
to a minimum. It's recommended that the [`prealloc`](https://github.com/alexkohler/prealloc)
golanci-lint rule automatically check for this.
### Analyzer Tests

18
doc/update/deprecations.md
View File

@ -45,6 +45,24 @@ sole discretion of GitLab Inc.
<div class="announcement-milestone">
## Announced in 15.3
<div class="deprecation removal-160">
### Maximum number of active pipelines per project limit (`ci_active_pipelines`)
Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)
The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) was never enabled by default and will be removed in GitLab 16.0. This limit can also be configured in the rails console under [`ci_active_pipelines`](https://docs.gitlab.com/ee/administration/instance_limits.html#number-of-pipelines-running-concurrently). Instead, use the other recommended rate limits that offer similar protection:
- [**Pipelines rate limits**](https://docs.gitlab.com/ee/user/admin_area/settings/rate_limit_on_pipelines_creation.html).
- [**Total number of jobs in currently active pipelines**](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits).
</div>
</div>
<div class="announcement-milestone">
## Announced in 15.2
<div class="deprecation removal-160 breaking-change">

2
doc/update/package/convert_to_ee.md
View File

@ -22,6 +22,8 @@ that may require Support intervention.
The steps can be summed up to:
1. Make a [GitLab backup](../../raketasks/backup_gitlab.md).
1. Find the currently installed GitLab version:
**For Debian/Ubuntu**

2
doc/user/application_security/secret_detection/index.md
View File

@ -200,7 +200,7 @@ Secret Detection can be customized by defining available CI/CD variables:
| CI/CD variable | Default value | Description |
|-----------------------------------|---------------|-------------|
| `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. |
| `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs (see [`doublestar.Match`](https://pkg.go.dev/github.com/bmatcuk/doublestar/v4@v4.0.2#Match) for supported patterns), or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. |
| `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. |
| `SECRET_DETECTION_IMAGE_SUFFIX` | "" | Suffix added to the image name. If set to `-fips`, `FIPS-enabled` images are used for scan. See [FIPS-enabled images](#fips-enabled-images) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355519) in GitLab 14.10. |
| `SECRET_DETECTION_LOG_OPTIONS` | "" | [`git log`](https://git-scm.com/docs/git-log) options used to define commit ranges. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350660) in GitLab 15.1.|

4
doc/user/asciidoc.md
View File

@ -477,8 +477,8 @@ digraph G {
#### PlantUML
To make PlantUML available in GitLab, a GitLab administrator needs to enable it first.
Read more in [PlantUML & GitLab](../administration/integration/plantuml.md).
PlantUML integration is enabled on GitLab.com. To make PlantUML available in self-managed
installation of GitLab, a GitLab administrator [must enable it](../administration/integration/plantuml.md).
After PlantUML is enabled, enter your text in a `plantuml` block:

4
doc/user/markdown.md
View File

@ -190,8 +190,8 @@ end
#### PlantUML
To make PlantUML available in GitLab, a GitLab administrator must enable it. For more information, see the
[PlantUML & GitLab](../administration/integration/plantuml.md) page.
PlantUML integration is enabled on GitLab.com. To make PlantUML available in self-managed
installation of GitLab, a GitLab administrator [must enable it](../administration/integration/plantuml.md).
#### Kroki