diff --git a/app/helpers/search_helper.rb b/app/helpers/search_helper.rb index ae67c7b4178..409a3e65fe3 100644 --- a/app/helpers/search_helper.rb +++ b/app/helpers/search_helper.rb @@ -150,7 +150,7 @@ module SearchHelper } ] - if search_service.scope == 'issues' && Feature.enabled?(:search_sort_issues_by_popularity) + if search_service.scope == 'issues' options << { title: _('Popularity'), sortable: true, diff --git a/app/views/search/results/_issuable.html.haml b/app/views/search/results/_issuable.html.haml index 551f5c048bc..63524bbf00e 100644 --- a/app/views/search/results/_issuable.html.haml +++ b/app/views/search/results/_issuable.html.haml @@ -12,7 +12,7 @@ .description.term.gl-px-0 = highlight_and_truncate_issuable(issuable, @search_term, @search_highlight) .col-sm-3.gl-mt-3.gl-sm-mt-0.gl-text-right - - if Feature.enabled?(:search_sort_issues_by_popularity) && issuable.respond_to?(:upvotes_count) && issuable.upvotes_count > 0 + - if issuable.respond_to?(:upvotes_count) && issuable.upvotes_count > 0 %li.issuable-upvotes.gl-list-style-none.has-tooltip{ title: _('Upvotes') } = sprite_icon('thumb-up', css_class: "gl-vertical-align-middle") = issuable.upvotes_count diff --git a/config/feature_flags/development/search_sort_issues_by_popularity.yml b/config/feature_flags/development/search_sort_issues_by_popularity.yml deleted file mode 100644 index 64885f00792..00000000000 --- a/config/feature_flags/development/search_sort_issues_by_popularity.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: search_sort_issues_by_popularity -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65231 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/334974 -milestone: '14.1' -type: development -group: group::global search -default_enabled: false diff --git a/db/migrate/20210629031900_associate_existing_dast_builds_with_variables.rb b/db/migrate/20210629031900_associate_existing_dast_builds_with_variables.rb new file mode 100644 index 00000000000..5db39334550 --- /dev/null +++ b/db/migrate/20210629031900_associate_existing_dast_builds_with_variables.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +class AssociateExistingDastBuildsWithVariables < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + class Profile < ApplicationRecord + self.table_name = 'dast_profiles' + self.inheritance_column = :_type_disabled + end + + class ProfilesPipeline < ApplicationRecord + include EachBatch + + self.table_name = 'dast_profiles_pipelines' + self.inheritance_column = :_type_disabled + + belongs_to :profile, foreign_key: :dast_profile_id + end + + class Build < ApplicationRecord + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + + default_scope { where(name: :dast, stage: :dast) } # rubocop:disable Cop/DefaultScope + end + + class SiteProfilesBuild < ApplicationRecord + self.table_name = 'dast_site_profiles_builds' + self.inheritance_column = :_type_disabled + end + + BATCH_SIZE = 300 + + def up + process_batch do |batch| + bulk_inserts = [] + + grouped_builds = fetch_builds(batch).group_by(&:commit_id) + + batch.includes(:profile).each do |profile_pipeline| + builds = grouped_builds[profile_pipeline.ci_pipeline_id] + + next if builds.blank? + + builds.each do |build| + bulk_inserts.push(dast_site_profile_id: profile_pipeline.profile.dast_site_profile_id, ci_build_id: build.id) + end + end + + SiteProfilesBuild.insert_all(bulk_inserts, unique_by: :ci_build_id) + end + end + + def down + process_batch do |batch| + builds = fetch_builds(batch) + + SiteProfilesBuild + .where(ci_build_id: builds) + .delete_all + end + end + + private + + def fetch_builds(batch) + # pluck necessary to support ci table decomposition + # https://gitlab.com/groups/gitlab-org/-/epics/6289 + Build.where(commit_id: batch.pluck(:ci_pipeline_id)) + end + + def process_batch + ProfilesPipeline.each_batch(of: BATCH_SIZE, column: :ci_pipeline_id) do |batch| + yield(batch) + end + end +end diff --git a/db/post_migrate/20210804151444_prepare_indexes_for_ci_job_artifact_bigint_conversion.rb b/db/post_migrate/20210804151444_prepare_indexes_for_ci_job_artifact_bigint_conversion.rb new file mode 100644 index 00000000000..8115465e311 --- /dev/null +++ b/db/post_migrate/20210804151444_prepare_indexes_for_ci_job_artifact_bigint_conversion.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class PrepareIndexesForCiJobArtifactBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + prepare_async_index :ci_job_artifacts, :id_convert_to_bigint, unique: true, + name: :index_ci_job_artifact_on_id_convert_to_bigint + + prepare_async_index :ci_job_artifacts, [:project_id, :id_convert_to_bigint], where: 'file_type = 18', + name: :index_ci_job_artifacts_for_terraform_reports_bigint + + prepare_async_index :ci_job_artifacts, :id_convert_to_bigint, where: 'file_type = 18', + name: :index_ci_job_artifacts_id_for_terraform_reports_bigint + + prepare_async_index :ci_job_artifacts, [:expire_at, :job_id_convert_to_bigint], + name: :index_ci_job_artifacts_on_expire_at_and_job_id_bigint + + prepare_async_index :ci_job_artifacts, [:job_id_convert_to_bigint, :file_type], unique: true, + name: :index_ci_job_artifacts_on_job_id_and_file_type_bigint + end + + def down + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifacts_on_job_id_and_file_type_bigint + + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifacts_on_expire_at_and_job_id_bigint + + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifacts_id_for_terraform_reports_bigint + + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifacts_for_terraform_reports_bigint + + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifact_on_id_convert_to_bigint + end +end diff --git a/db/post_migrate/20210804153307_prepare_indexes_for_tagging_bigint_conversion.rb b/db/post_migrate/20210804153307_prepare_indexes_for_tagging_bigint_conversion.rb new file mode 100644 index 00000000000..98f90bafce3 --- /dev/null +++ b/db/post_migrate/20210804153307_prepare_indexes_for_tagging_bigint_conversion.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class PrepareIndexesForTaggingBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + prepare_async_index :taggings, :id_convert_to_bigint, unique: true, + name: :index_taggings_on_id_convert_to_bigint + + prepare_async_index :taggings, [:taggable_id_convert_to_bigint, :taggable_type], + name: :i_taggings_on_taggable_id_convert_to_bigint_and_taggable_type + + prepare_async_index :taggings, [:taggable_id_convert_to_bigint, :taggable_type, :context], + name: :i_taggings_on_taggable_bigint_and_taggable_type_and_context + + prepare_async_index :taggings, [:tag_id, :taggable_id_convert_to_bigint, :taggable_type, :context, :tagger_id, :tagger_type], + unique: true, name: :taggings_idx_tmp + end + + def down + unprepare_async_index_by_name :taggings, :taggings_idx_tmp + + unprepare_async_index_by_name :taggings, :i_taggings_on_taggable_bigint_and_taggable_type_and_context + + unprepare_async_index_by_name :taggings, :i_taggings_on_taggable_id_convert_to_bigint_and_taggable_type + + unprepare_async_index_by_name :taggings, :index_taggings_on_id_convert_to_bigint + end +end diff --git a/db/post_migrate/20210804154407_prepare_indexes_for_ci_stage_bigint_conversion.rb b/db/post_migrate/20210804154407_prepare_indexes_for_ci_stage_bigint_conversion.rb new file mode 100644 index 00000000000..82af595b2d3 --- /dev/null +++ b/db/post_migrate/20210804154407_prepare_indexes_for_ci_stage_bigint_conversion.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class PrepareIndexesForCiStageBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + prepare_async_index :ci_stages, :id_convert_to_bigint, unique: true, + name: :index_ci_stages_on_id_convert_to_bigint + + prepare_async_index :ci_stages, [:pipeline_id, :id_convert_to_bigint], where: 'status in (0, 1, 2, 8, 9, 10)', + name: :index_ci_stages_on_pipeline_id_and_id_convert_to_bigint + end + + def down + unprepare_async_index_by_name :ci_stages, :index_ci_stages_on_pipeline_id_and_id_convert_to_bigint + + unprepare_async_index_by_name :ci_stages, :index_ci_stages_on_id_convert_to_bigint + end +end diff --git a/db/schema_migrations/20210629031900 b/db/schema_migrations/20210629031900 new file mode 100644 index 00000000000..5f4949c36de --- /dev/null +++ b/db/schema_migrations/20210629031900 @@ -0,0 +1 @@ +4f20581b0d16157fbe984383417f0463d7e52252569480796aa3c73abf19c95f \ No newline at end of file diff --git a/db/schema_migrations/20210804151444 b/db/schema_migrations/20210804151444 new file mode 100644 index 00000000000..0d633c3da36 --- /dev/null +++ b/db/schema_migrations/20210804151444 @@ -0,0 +1 @@ +4a6676e9185a99070751c91c71a7a9e6a845426d68567abf80a2e414251e5805 \ No newline at end of file diff --git a/db/schema_migrations/20210804153307 b/db/schema_migrations/20210804153307 new file mode 100644 index 00000000000..ae7baaa4cf4 --- /dev/null +++ b/db/schema_migrations/20210804153307 @@ -0,0 +1 @@ +d73756410c7f37662c50bb05c372e6ac32ba81f232c07debcd42d1f679eb74ef \ No newline at end of file diff --git a/db/schema_migrations/20210804154407 b/db/schema_migrations/20210804154407 new file mode 100644 index 00000000000..dc0e40c7117 --- /dev/null +++ b/db/schema_migrations/20210804154407 @@ -0,0 +1 @@ +ab7ee98704e844de4a3ba3ae14ea64dd46539e63d49c7c7e0d67ed03ebc3bbd4 \ No newline at end of file diff --git a/doc/administration/git_protocol.md b/doc/administration/git_protocol.md index 6e391cb459e..e3e2db81fb0 100644 --- a/doc/administration/git_protocol.md +++ b/doc/administration/git_protocol.md @@ -113,14 +113,5 @@ URL to use SSH. ### Observe Git protocol version of connections -To observe what Git protocol versions are being used in a -production environment, you can use the following Prometheus query: - -```prometheus -sum(rate(gitaly_git_protocol_requests_total[1m])) by (grpc_method,git_protocol,grpc_service) -``` - - - -You can view what Git protocol versions are being used on GitLab.com at -. +For information on observing the Git protocol versions are being used in a production environment, +see the [relevant documentation](gitaly/index.md#useful-queries). diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index 79b92835d3e..5e8cbac42c1 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -684,12 +684,8 @@ To configure Gitaly with TLS: ### Observe type of Gitaly connections -[Prometheus](../monitoring/prometheus/index.md) can be used observe what type of connections Gitaly -is serving a production environment. Use the following Prometheus query: - -```prometheus -sum(rate(gitaly_connections_total[5m])) by (type) -``` +For information on observing the type of Gitaly connections being served, see the +[relevant documentation](index.md#useful-queries). ## `gitaly-ruby` @@ -781,26 +777,8 @@ repository. In the example above: - If another request comes in for a repository that has used up its 20 slots, that request gets queued. -You can observe the behavior of this queue using the Gitaly logs and Prometheus: - -- In the Gitaly logs, look for the string (or structured log field) `acquire_ms`. Messages that have - this field are reporting about the concurrency limiter. -- In Prometheus, look for the following metrics: - - - `gitaly_rate_limiting_in_progress`. - - `gitaly_rate_limiting_queued`. - - `gitaly_rate_limiting_seconds`. - -The metric definitions are available: - -- Directly from Prometheus `/metrics` endpoint configured for Gitaly. -- Using [Grafana Explore](https://grafana.com/docs/grafana/latest/explore/) on a - Grafana instance configured against Prometheus. - -NOTE: -Although the name of the Prometheus metric contains `rate_limiting`, it's a concurrency limiter, not -a rate limiter. If a Gitaly client makes 1,000 requests in a row very quickly, concurrency doesn't -exceed 1, and the concurrency limiter has no effect. +You can observe the behavior of this queue using the Gitaly logs and Prometheus. For more +information, see the [relevant documentation](index.md#monitor-gitaly). ## Background Repository Optimization @@ -854,30 +832,11 @@ server" and "Gitaly client" refers to the same machine. ### Verify authentication monitoring -Before rotating a Gitaly authentication token, verify that you can monitor the authentication -behavior of your GitLab installation using Prometheus. Use the following Prometheus query: +Before rotating a Gitaly authentication token, verify that you can +[monitor the authentication behavior](index.md#useful-queries) of your GitLab installation using +Prometheus. -```prometheus -sum(rate(gitaly_authentications_total[5m])) by (enforced, status) -``` - -In a system where authentication is configured correctly and where you have live traffic, you -see something like this: - -```prometheus -{enforced="true",status="ok"} 4424.985419441742 -``` - -There may also be other numbers with rate 0. We care only about the non-zero numbers. - -The only non-zero number should have `enforced="true",status="ok"`. If you have other non-zero -numbers, something is wrong in your configuration. - -The `status="ok"` number reflects your current request rate. In the example above, Gitaly is -handling about 4000 requests per second. - -Now that you have established that you can monitor the Gitaly authentication behavior of your GitLab -installation, you can begin the rest of the procedure. +You can then continue the rest of the procedure. ### Enable "auth transitioning" mode @@ -1084,9 +1043,8 @@ closed it. ### Observe the cache -The cache can be observed in logs and using metrics. - -#### Logs +The cache can be observed [using metrics](index.md#monitor-gitaly) and in the following logged +information: |Message|Fields|Description| |:---|:---|:---| @@ -1146,33 +1104,3 @@ Example: "time":"2021-03-25T14:57:53.543Z" } ``` - -#### Metrics - -The following cache metrics are available. - -|Metric|Type|Labels|Description| -|:---|:---|:---|:---| -|`gitaly_pack_objects_cache_enabled`|gauge|`dir`,`max_age`|Set to `1` when the cache is enabled via the Gitaly configuration file| -|`gitaly_pack_objects_cache_lookups_total`|counter|`result`|Hit/miss counter for cache lookups| -|`gitaly_pack_objects_generated_bytes_total`|counter||Number of bytes written into the cache| -|`gitaly_pack_objects_served_bytes_total`|counter||Number of bytes read from the cache| -|`gitaly_streamcache_filestore_disk_usage_bytes`|gauge|`dir`|Total size of cache files| -|`gitaly_streamcache_index_entries`|gauge|`dir`|Number of entries in the cache| - -Some of these metrics start with `gitaly_streamcache` -because they are generated by the "streamcache" internal library -package in Gitaly. - -Example: - -```plaintext -gitaly_pack_objects_cache_enabled{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache",max_age="300"} 1 -gitaly_pack_objects_cache_lookups_total{result="hit"} 2 -gitaly_pack_objects_cache_lookups_total{result="miss"} 1 -gitaly_pack_objects_generated_bytes_total 2.618649e+07 -gitaly_pack_objects_served_bytes_total 7.855947e+07 -gitaly_streamcache_filestore_disk_usage_bytes{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 2.6200152e+07 -gitaly_streamcache_filestore_removed_total{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 -gitaly_streamcache_index_entries{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 -``` diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 2403bbbba5f..86c0ce392f2 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -267,13 +267,7 @@ The primary node is chosen to serve the request if: - There are no up to date nodes. - Any other error occurs during node selection. -To track distribution of read operations, you can use the `gitaly_praefect_read_distribution` -Prometheus counter metric. It has two labels: - -- `virtual_storage`. -- `storage`. - -They reflect configuration defined for this instance of Praefect. +You can [monitor distribution of reads](#monitor-gitaly-cluster) using Prometheus. #### Strong consistency @@ -312,6 +306,137 @@ For configuration information, see [Configure replication factor](praefect.md#co For more information on configuring Gitaly Cluster, see [Configure Gitaly Cluster](praefect.md). +## Monitor Gitaly and Gitaly Cluster + +You can use the available logs and [Prometheus metrics](../monitoring/prometheus/index.md) to +monitor Gitaly and Gitaly Cluster (Praefect). + +Metric definitions are available: + +- Directly from Prometheus `/metrics` endpoint configured for Gitaly. +- Using [Grafana Explore](https://grafana.com/docs/grafana/latest/explore/) on a + Grafana instance configured against Prometheus. + +### Monitor Gitaly + +You can observe the behavior of [queued requests](configure_gitaly.md#limit-rpc-concurrency) using +the Gitaly logs and Prometheus: + +- In the [Gitaly logs](../logs.md#gitaly-logs), look for the string (or structured log field) + `acquire_ms`. Messages that have this field are reporting about the concurrency limiter. +- In Prometheus, look for the following metrics: + - `gitaly_rate_limiting_in_progress`. + - `gitaly_rate_limiting_queued`. + - `gitaly_rate_limiting_seconds`. + + Although the name of the Prometheus metric contains `rate_limiting`, it's a concurrency limiter, + not a rate limiter. If a Gitaly client makes 1,000 requests in a row very quickly, concurrency + doesn't exceed 1, and the concurrency limiter has no effect. + +The following [pack-objects cache](configure_gitaly.md#pack-objects-cache) metrics are available: + +- `gitaly_pack_objects_cache_enabled`, a gauge set to `1` when the cache is enabled. Available + labels: `dir` and `max_age`. +- `gitaly_pack_objects_cache_lookups_total`, a counter for cache lookups. Available label: `result`. +- `gitaly_pack_objects_generated_bytes_total`, a counter for the number of bytes written into the + cache. +- `gitaly_pack_objects_served_bytes_total`, a counter for the number of bytes read from the cache. +- `gitaly_streamcache_filestore_disk_usage_bytes`, a gauge for the total size of cache files. + Available label: `dir`. +- `gitaly_streamcache_index_entries`, a gauge for the number of entries in the cache. Available + label: `dir`. + +Some of these metrics start with `gitaly_streamcache` because they are generated by the +`streamcache` internal library package in Gitaly. + +Example: + +```plaintext +gitaly_pack_objects_cache_enabled{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache",max_age="300"} 1 +gitaly_pack_objects_cache_lookups_total{result="hit"} 2 +gitaly_pack_objects_cache_lookups_total{result="miss"} 1 +gitaly_pack_objects_generated_bytes_total 2.618649e+07 +gitaly_pack_objects_served_bytes_total 7.855947e+07 +gitaly_streamcache_filestore_disk_usage_bytes{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 2.6200152e+07 +gitaly_streamcache_filestore_removed_total{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 +gitaly_streamcache_index_entries{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 +``` + +#### Useful queries + +The following are useful queries for monitoring Gitaly: + +- Use the following Prometheus query to observe the + [type of connections](configure_gitaly.md#enable-tls-support) Gitaly is serving a production + environment: + + ```prometheus + sum(rate(gitaly_connections_total[5m])) by (type) + ``` + +- Use the following Prometheus query to monitor the + [authentication behavior](configure_gitaly.md#observe-type-of-gitaly-connections) of your GitLab + installation: + + ```prometheus + sum(rate(gitaly_authentications_total[5m])) by (enforced, status) + ``` + + In a system where authentication is configured correctly and where you have live traffic, you + see something like this: + + ```prometheus + {enforced="true",status="ok"} 4424.985419441742 + ``` + + There may also be other numbers with rate 0, but you only need to take note of the non-zero numbers. + + The only non-zero number should have `enforced="true",status="ok"`. If you have other non-zero + numbers, something is wrong in your configuration. + + The `status="ok"` number reflects your current request rate. In the example above, Gitaly is + handling about 4000 requests per second. + +- Use the following Prometheus query to observe the [Git protocol versions](../git_protocol.md) + being used in a production environment: + + ```prometheus + sum(rate(gitaly_git_protocol_requests_total[1m])) by (grpc_method,git_protocol,grpc_service) + ``` + +### Monitor Gitaly Cluster + +To monitor Gitaly Cluster (Praefect), you can use these Prometheus metrics: + +- `gitaly_praefect_read_distribution`, a counter to track [distribution of reads](#distributed-reads). + It has two labels: + + - `virtual_storage`. + - `storage`. + + They reflect configuration defined for this instance of Praefect. + +- `gitaly_praefect_replication_latency_bucket`, a histogram measuring the amount of time it takes + for replication to complete once the replication job starts. Available in GitLab 12.10 and later. +- `gitaly_praefect_replication_delay_bucket`, a histogram measuring how much time passes between + when the replication job is created and when it starts. Available in GitLab 12.10 and later. +- `gitaly_praefect_node_latency_bucket`, a histogram measuring the latency in Gitaly returning + health check information to Praefect. This indicates Praefect connection saturation. Available in + GitLab 12.10 and later. + +To monitor [strong consistency](#strong-consistency), you can use the following Prometheus metrics: + +- `gitaly_praefect_transactions_total`, the number of transactions created and voted on. +- `gitaly_praefect_subtransactions_per_transaction_total`, the number of times nodes cast a vote for + a single transaction. This can happen multiple times if multiple references are getting updated in + a single transaction. +- `gitaly_praefect_voters_per_transaction_total`: the number of Gitaly nodes taking part in a + transaction. +- `gitaly_praefect_transactions_delay_seconds`, the server-side delay introduced by waiting for the + transaction to be committed. +- `gitaly_hook_transaction_voting_delay_seconds`, the client-side delay introduced by waiting for + the transaction to be committed. + ## Do not bypass Gitaly GitLab doesn't advise directly accessing Gitaly repositories stored on disk with a Git client, diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index efe6cdf75ee..6a794dba4f9 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -1094,19 +1094,8 @@ Feature.enable(:gitaly_reference_transactions) Feature.disable(:gitaly_reference_transactions_primary_wins) ``` -To monitor strong consistency, you can use the following Prometheus metrics: - -- `gitaly_praefect_transactions_total`: Number of transactions created and - voted on. -- `gitaly_praefect_subtransactions_per_transaction_total`: Number of times - nodes cast a vote for a single transaction. This can happen multiple times if - multiple references are getting updated in a single transaction. -- `gitaly_praefect_voters_per_transaction_total`: Number of Gitaly nodes taking - part in a transaction. -- `gitaly_praefect_transactions_delay_seconds`: Server-side delay introduced by - waiting for the transaction to be committed. -- `gitaly_hook_transaction_voting_delay_seconds`: Client-side delay introduced - by waiting for the transaction to be committed. +For information on monitoring strong consistency, see the +[relevant documentation](index.md#monitor-gitaly-cluster). ## Configure replication factor diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index ec5a8d47ae2..9fe09be10a3 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -71,7 +71,7 @@ Remember to disable `transitioning` when you are done changing your token settings. All authentication attempts are counted in Prometheus under -the `gitaly_authentications_total` metric. +the [`gitaly_authentications_total` metric](index.md#useful-queries). ### TLS diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 10f0486a8c3..459eb259498 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w To enable the GitLab Prometheus metrics: -1. Log in to GitLab as a user with [administrator permissions](../../../user/permissions.md). +1. Log in to GitLab as a user with Administrator [role](../../../user/permissions.md). 1. On the top bar, select **Menu >** **{admin}** **Admin**. 1. On the left sidebar, select **Settings > Metrics and profiling**. 1. Find the **Metrics - Prometheus** section, and select **Add link to Prometheus**. @@ -153,15 +153,8 @@ The following metrics can be controlled by feature flags: ## Praefect metrics -You can [configure Praefect to report metrics](../../gitaly/praefect.md#praefect). -These are some of the Praefect metrics served from the `/metrics` path on the [configured port](index.md#changing-the-port-and-address-prometheus-listens-on) -(9652 by default). - -| Metric | Type | Since | Description | Labels | -| :----- | :--- | ----: | :---------- | :----- | -| `gitaly_praefect_replication_latency_bucket` | Histogram | 12.10 | The amount of time it takes for replication to complete once the replication job starts. | | -| `gitaly_praefect_replication_delay_bucket` | Histogram | 12.10 | A measure of how much time passes between when the replication job is created and when it starts. | | -| `gitaly_praefect_node_latency_bucket` | Histogram | 12.10 | The latency in Gitaly returning health check information to Praefect. This indicates Praefect connection saturation. | | +You can [configure Praefect](../../gitaly/praefect.md#praefect) to report metrics. For information +on available metrics, see the [relevant documentation](../../gitaly/index.md#monitor-gitaly-cluster). ## Sidekiq metrics diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 5b21e8027a4..dd81f71d418 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -8,18 +8,19 @@ info: To determine the technical writer assigned to the Stage/Group associated w [Prometheus](https://prometheus.io) is a powerful time-series monitoring service, providing a flexible platform for monitoring GitLab and other software products. + GitLab provides out-of-the-box monitoring with Prometheus, providing easy access to high quality time-series monitoring of GitLab services. -> **Notes:** -> -> - Prometheus and the various exporters listed in this page are bundled in the -> Omnibus GitLab package. Check each exporter's documentation for the timeline -> they got added. For installations from source you must install them -> yourself. Over subsequent releases additional GitLab metrics are captured. -> - Prometheus services are on by default with GitLab 9.0. -> - Prometheus and its exporters don't authenticate users, and are available -> to anyone who can access them. +Prometheus and the various exporters listed in this page are bundled in the +Omnibus GitLab package. Check each exporter's documentation for the timeline +they got added. For installations from source you must install them +yourself. Over subsequent releases additional GitLab metrics are captured. + +Prometheus services are on by default. + +Prometheus and its exporters don't authenticate users, and are available to anyone who can access +them. ## Overview @@ -33,7 +34,7 @@ dashboard tool like [Grafana](https://grafana.com). For installations from source, you must install and configure it yourself. -Prometheus and its exporters are on by default, starting with GitLab 9.0. +Prometheus and its exporters are on by default. Prometheus runs as the `gitlab-prometheus` user and listen on `http://localhost:9090`. By default, Prometheus is only accessible from the GitLab server itself. Each exporter is automatically set up as a diff --git a/doc/development/architecture.md b/doc/development/architecture.md index f39171b1e69..687d1dd5ffc 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -344,7 +344,7 @@ Component statuses are linked to configuration documentation for each component. | [Certificate Management](#certificate-management) | TLS Settings, Let's Encrypt | ✅ | ✅ | ✅ | ⚙ | ✅ | ⚙ | ⚙ | CE & EE | | [Consul](#consul) | Database node discovery, failover | ⚙ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | EE Only | | [Database Migrations](#database-migrations) | Database migrations | ✅ | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | -| [Elasticsearch](#elasticsearch) | Improved search within GitLab | ⤓ | ⚙ | ⤓ | ⤓ | ✅ | ⤓ | ⤓ | EE Only | +| [Elasticsearch](#elasticsearch) | Improved search within GitLab | ⤓ | ⚙ | ⤓ | ⤓ | ✅ | ⤓ | ⚙ | EE Only | | [Gitaly](#gitaly) | Git RPC service for handling all Git calls made by GitLab | ✅ | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | | [GitLab Exporter](#gitlab-exporter) | Generates a variety of GitLab metrics | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | CE & EE | | [GitLab Geo Node](#gitlab-geo) | Geographically distributed GitLab nodes | ⚙ | ⚙ | ❌ | ❌ | ✅ | ❌ | ⚙ | EE Only | @@ -352,18 +352,18 @@ Component statuses are linked to configuration documentation for each component. | [GitLab Pages](#gitlab-pages) | Hosts static websites | ⚙ | ⚙ | ❌ | ❌ | ✅ | ⚙ | ⚙ | CE & EE | | [GitLab Kubernetes Agent](#gitlab-kubernetes-agent) | Integrate Kubernetes clusters in a cloud-native way | ⚙ | ⚙ | ⚙ | ❌ | ❌ | ⤓ | ⚙ | EE Only | | [GitLab self-monitoring: Alertmanager](#alertmanager) | Deduplicates, groups, and routes alerts from Prometheus | ⚙ | ⚙ | ✅ | ⚙ | ✅ | ❌ | ❌ | CE & EE | -| [GitLab self-monitoring: Grafana](#grafana) | Metrics dashboard | ✅ | ✅ | ⚙ | ⤓ | ✅ | ❌ | ❌ | CE & EE | +| [GitLab self-monitoring: Grafana](#grafana) | Metrics dashboard | ✅ | ✅ | ⚙ | ⤓ | ✅ | ❌ | ⚙ | CE & EE | | [GitLab self-monitoring: Jaeger](#jaeger) | View traces generated by the GitLab instance | ❌ | ⚙ | ⚙ | ❌ | ❌ | ⤓ | ⚙ | CE & EE | -| [GitLab self-monitoring: Prometheus](#prometheus) | Time-series database, metrics collection, and query service | ✅ | ✅ | ✅ | ⚙ | ✅ | ❌ | ❌ | CE & EE | +| [GitLab self-monitoring: Prometheus](#prometheus) | Time-series database, metrics collection, and query service | ✅ | ✅ | ✅ | ⚙ | ✅ | ❌ | ⚙ | CE & EE | | [GitLab self-monitoring: Sentry](#sentry) | Track errors generated by the GitLab instance | ⤓ | ⤓ | ⤓ | ❌ | ✅ | ⤓ | ⤓ | CE & EE | | [GitLab Shell](#gitlab-shell) | Handles `git` over SSH sessions | ✅ | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | | [GitLab Workhorse](#gitlab-workhorse) | Smart reverse proxy, handles large HTTP requests | ✅ | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | | [Inbound email (SMTP)](#inbound-email) | Receive messages to update issues | ⤓ | ⤓ | ⚙ | ⤓ | ✅ | ⤓ | ⤓ | CE & EE | -| [Jaeger integration](#jaeger) | Distributed tracing for deployed apps | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | EE Only | -| [LDAP Authentication](#ldap-authentication) | Authenticate users against centralized LDAP directory | ⤓ | ⤓ | ⤓ | ⤓ | ❌ | ⤓ | ⤓ | CE & EE | -| [Mattermost](#mattermost) | Open-source Slack alternative | ⚙ | ⚙ | ⤓ | ⤓ | ⤓ | ❌ | ❌ | CE & EE | +| [Jaeger integration](#jaeger) | Distributed tracing for deployed apps | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | ⤓ | ⚙ | EE Only | +| [LDAP Authentication](#ldap-authentication) | Authenticate users against centralized LDAP directory | ⤓ | ⤓ | ⤓ | ⤓ | ❌ | ⤓ | ⚙ | CE & EE | +| [Mattermost](#mattermost) | Open-source Slack alternative | ⚙ | ⚙ | ⤓ | ⤓ | ⤓ | ❌ | ⚙ | CE & EE | | [MinIO](#minio) | Object storage service | ⤓ | ⤓ | ✅ | ✅ | ✅ | ❌ | ⚙ | CE & EE | -| [NGINX](#nginx) | Routes requests to appropriate components, terminates SSL | ✅ | ✅ | ✅ | ⚙ | ✅ | ⤓ | ❌ | CE & EE | +| [NGINX](#nginx) | Routes requests to appropriate components, terminates SSL | ✅ | ✅ | ✅ | ⚙ | ✅ | ⤓ | ⚙ | CE & EE | | [Node Exporter](#node-exporter) | Prometheus endpoint with system metrics | ✅ | ✅ | N/A | N/A | ✅ | ❌ | ❌ | CE & EE | | [Outbound email (SMTP)](#outbound-email) | Send email messages to users | ⤓ | ⤓ | ⚙ | ⤓ | ✅ | ⤓ | ⤓ | CE & EE | | [Patroni](#patroni) | Manage PostgreSQL HA cluster leader selection and replication | ⚙ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | EE Only | diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index 6c05101436c..f3b00d20fb1 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -37,7 +37,7 @@ documentation. ### The documentation includes all information Include problem-solving actions that may address rare cases or be considered -_risky_, but provide proper context through fully-detailed +risky, but provide proper context through fully detailed warnings and caveats. This kind of content should be included as it could be helpful to others and, when properly explained, its benefits outweigh the risks. If you think you have found an exception to this rule, contact the @@ -94,7 +94,7 @@ of truth and explain why it is important. ### Docs-first methodology -We employ a _documentation-first methodology_. This method ensures the documentation +We employ a documentation-first methodology. This method ensures the documentation remains a complete and trusted resource, and makes communicating about the use of GitLab more efficient. @@ -222,8 +222,8 @@ Put files for a specific product area into the related folder: When working with directories and files: 1. When you create a new directory, always start with an `index.md` file. - Don't use another filename and _do not_ create `README.md` files. -1. _Do not_ use special characters and spaces, or capital letters in file + Don't use another filename and do not create `README.md` files. +1. Do not use special characters and spaces, or capital letters in file names, directory names, branch names, and anything that generates a path. 1. When creating or renaming a file or directory and it has more than one word in its name, use underscores (`_`) instead of spaces or dashes. For example, @@ -347,11 +347,11 @@ npm. ### Fake user information You may need to include user information in entries such as a REST call or user profile. -_Do not_ use real user information or email addresses in GitLab documentation. For email -addresses and names, do use: +Do not use real user information or email addresses in GitLab documentation. For email +addresses and names, use: -- _Email addresses_: Use an email address ending in `example.com`. -- _Names_: Use strings like `example_username`. Alternatively, use diverse or +- Email addresses: Use an email address ending in `example.com`. +- Names: Use strings like `example_username`. Alternatively, use diverse or non-gendered names with common surnames, such as `Sidney Jones`, `Zhang Wei`, or `Alex Garcia`. @@ -452,14 +452,14 @@ Follow these guidelines for punctuation: | Rule | Example | |------------------------------------------------------------------|--------------------------------------------------------| -| Avoid semicolons. Use two sentences instead. | _That's the way that the world goes 'round. You're up one day and the next you're down._ -| Always end full sentences with a period. | _For a complete overview, read through this document._ | -| Always add a space after a period when beginning a new sentence. | _For a complete overview, check this doc. For other references, check out this guide._ | +| Avoid semicolons. Use two sentences instead. | That's the way that the world goes 'round. You're up one day and the next you're down. +| Always end full sentences with a period. | For a complete overview, read through this document. | +| Always add a space after a period when beginning a new sentence. | For a complete overview, check this doc. For other references, check out this guide. | | Do not use double spaces. (Tested in [`SentenceSpacing.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/SentenceSpacing.yml).) | --- | | Do not use tabs for indentation. Use spaces instead. You can configure your code editor to output spaces instead of tabs when pressing the tab key. | --- | -| Use serial commas (_Oxford commas_) before the final _and_ or _or_ in a list of three or more items. (Tested in [`OxfordComma.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/OxfordComma.yml).) | _You can create new issues, merge requests, and milestones._ | -| Always add a space before and after dashes when using it in a sentence (for replacing a comma, for example). | _You should try this - or not._ | -| When a colon is part of a sentence, always use lowercase after the colon. | _Linked issues: a way to create a relationship between issues._ | +| Use serial commas (Oxford commas) before the final **and** or **or** in a list of three or more items. (Tested in [`OxfordComma.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/OxfordComma.yml).) | You can create new issues, merge requests, and milestones. | +| Always add a space before and after dashes when using it in a sentence (for replacing a comma, for example). | You should try this - or not. | +| When a colon is part of a sentence, always use lowercase after the colon. | Linked issues: a way to create a relationship between issues. | @@ -740,7 +740,7 @@ For other punctuation rules, refer to the ## Headings -- Add _only one H1_ in each document, by adding `#` at the beginning of +- Add only one H1 in each document, by adding `#` at the beginning of it (when using Markdown). The `h1` becomes the document ``. - Start with an `h2` (`##`), and respect the order `h2` > `h3` > `h4` > `h5` > `h6`. Never skip the hierarchy level, such as `h2` > `h4` @@ -839,7 +839,7 @@ We include guidance for links in these categories: ### Basic link criteria - Use inline link Markdown markup `[Text](https://example.com)`. - It's easier to read, review, and maintain. _Do not_ use `[Text][identifier]` reference-style links. + It's easier to read, review, and maintain. Do not use `[Text][identifier]` reference-style links. - Use [meaningful anchor texts](https://www.futurehosting.com/blog/links-should-have-meaningful-anchor-text-heres-why/). For example, instead of writing something like `Read more about GitLab Issue Boards [here](LINK)`, @@ -848,7 +848,7 @@ We include guidance for links in these categories: ### Links to internal documentation NOTE: -_Internal_ refers to documentation in the same project. When linking to +**Internal** refers to documentation in the same project. When linking to documentation in separate projects (for example, linking to Omnibus documentation from GitLab documentation), you must use absolute URLs. @@ -980,8 +980,8 @@ the commit link ensures the user lands on the line you're referring to. The **Permalink** button, displayed when viewing a file in a project, provides a link to the most recent commit of that file. -- _Do_: `[link to line 3](https://gitlab.com/gitlab-org/gitlab/-/blob/11f17c56d8b7f0b752562d78a4298a3a95b5ce66/.gitlab/issue_templates/Feature%20proposal.md#L3)` -- _Don't_: `[link to line 3](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20proposal.md#L3).` +- Do: `[link to line 3](https://gitlab.com/gitlab-org/gitlab/-/blob/11f17c56d8b7f0b752562d78a4298a3a95b5ce66/.gitlab/issue_templates/Feature%20proposal.md#L3)` +- Don't: `[link to line 3](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20proposal.md#L3).` If that linked expression has changed line numbers due to additional commits, you can still search the file for that query. In this case, update the @@ -1042,13 +1042,13 @@ they need to interact with the application. When you take screenshots: -- _Capture the most relevant area of the page._ Don't include unnecessary white +- **Capture the most relevant area of the page.** Don't include unnecessary white space or areas of the page that don't help illustrate the point. The left sidebar of the GitLab user interface can change, so don't include the sidebar if it's not necessary. -- _Keep it small._ If you don't need to show the full width of the screen, don't. +- **Keep it small.** If you don't need to show the full width of the screen, don't. A value of 1000 pixels is a good maximum width for your screenshot image. -- _Be consistent._ Coordinate screenshots with the other screenshots already on +- **Be consistent.** Coordinate screenshots with the other screenshots already on a documentation page. For example, if other screenshots include the left sidebar, include the sidebar in all screenshots. @@ -1067,8 +1067,8 @@ When you take screenshots: - Consider using PNG images instead of JPEG. - [Compress all PNG images](#compress-images). - Compress GIFs with <https://ezgif.com/optimize> or similar tool. -- Images should be used (only when necessary) to _illustrate_ the description - of a process, not to _replace_ it. +- Images should be used (only when necessary) to illustrate the description + of a process, not to replace it. - Max image size: 100KB (GIFs included). - See also how to link and embed [videos](#videos) to illustrate the documentation. @@ -1083,7 +1083,7 @@ documentation site. For accessibility and SEO, use [descriptions](https://webaim that: - Are accurate, succinct, and unique. -- Don't use _image of…_ or _graphic of…_ to describe the image. +- Don't use **image of** or **graphic of** to describe the image. ### Compress images @@ -1156,7 +1156,7 @@ embedded videos take up a lot of space on the page and can be distracting to rea To embed a video: 1. Copy the code from this procedure and paste it into your Markdown file. Leave a - blank line above and below it. Do _not_ edit the code (don't remove or add any spaces). + blank line above and below it. Do not edit the code (don't remove or add any spaces). 1. In YouTube, visit the video URL you want to display. Copy the regular URL from your browser (`https://www.youtube.com/watch?v=VIDEO-ID`) and replace the video title and link in the line under `<div class="video-fallback">`. @@ -1451,18 +1451,18 @@ application: Use these verbs for specific uses with user interface elements: -| Recommended | Used for | Replaces | -|:------------------------|:--------------------------------------|:----------------------------| -| _select_ | buttons, links, menu items, dropdowns | "click, "press," "hit" | -| _select_ or _clear_ | checkboxes | "enable", "click", "press" | -| _expand_ | expandable sections | "open" | -| _turn on_ or _turn off_ | toggles | "flip", "enable", "disable" | +| Recommended | Used for | Replaces | +|:--------------------|:--------------------------------------|:----------------------| +| select | buttons, links, menu items, dropdowns | click, press, hit | +| select or clear | checkboxes | enable, click, press | +| expand | expandable sections | open | +| turn on or turn off | toggles | flip, enable, disable | ### Other Verbs | Recommended | Used for | Replaces | |:------------|:--------------------------------|:----------------------| -| _go to_ | making a browser go to location | "navigate to", "open" | +| go to | making a browser go to location | navigate to, open | ## GitLab versions @@ -1605,7 +1605,7 @@ like these can raise legal issues. Instead, say that an issue exists, for exampl "Support for this feature is tracked [in this issue]." Or, "Improvements to this functionality are tracked [in this issue]." -You _can_ say that we plan to remove a feature. +You can say that we plan to remove a feature. ### Removing versions after each major release @@ -1665,17 +1665,17 @@ badges and tooltips (`<span class="badge-trigger free">`). #### Available product tier badges -| Tier in which feature is available | Tier badge | -|:--------------------------------------------------------------------------|:----------------------| -| GitLab Free self-managed and SaaS, and higher tiers | `**(FREE)**` | -| GitLab Premium self-managed and SaaS, and their higher tiers | `**(PREMIUM)**` | -| GitLab Ultimate self-managed and SaaS | `**(ULTIMATE)**` | -| _Only_ GitLab Free self-managed and higher tiers (no SaaS-based tiers) | `**(FREE SELF)**` | -| _Only_ GitLab Premium self-managed and higher tiers (no SaaS-based tiers) | `**(PREMIUM SELF)**` | -| _Only_ GitLab Ultimate self-managed (no SaaS-based tiers) | `**(ULTIMATE SELF)**` | -| _Only_ GitLab Free SaaS and higher tiers (no self-managed instances) | `**(FREE SAAS)**` | -| _Only_ GitLab Premium SaaS and higher tiers (no self-managed instances) | `**(PREMIUM SAAS)**` | -| _Only_ GitLab Ultimate SaaS (no self-managed instances) | `**(ULTIMATE SAAS)**` | +| Tier in which feature is available | Tier badge | +|:------------------------------------------------------------------------|:----------------------| +| GitLab Free self-managed and SaaS, and higher tiers | `**(FREE)**` | +| GitLab Premium self-managed and SaaS, and their higher tiers | `**(PREMIUM)**` | +| GitLab Ultimate self-managed and SaaS | `**(ULTIMATE)**` | +| Only GitLab Free self-managed and higher tiers (no SaaS-based tiers) | `**(FREE SELF)**` | +| Only GitLab Premium self-managed and higher tiers (no SaaS-based tiers) | `**(PREMIUM SELF)**` | +| Only GitLab Ultimate self-managed (no SaaS-based tiers) | `**(ULTIMATE SELF)**` | +| Only GitLab Free SaaS and higher tiers (no self-managed instances) | `**(FREE SAAS)**` | +| Only GitLab Premium SaaS and higher tiers (no self-managed instances) | `**(PREMIUM SAAS)**` | +| Only GitLab Ultimate SaaS (no self-managed instances) | `**(ULTIMATE SAAS)**` | Topics that mention the `gitlab.rb` file are referring to self-managed instances of GitLab. To prevent confusion, include the relevant `TIER SELF` diff --git a/doc/user/permissions.md b/doc/user/permissions.md index e47c70ba2f2..e6071b1f87c 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -52,6 +52,23 @@ The following table lists project permissions available for each role: | [Analytics](analytics/index.md):<br>View [CI/CD analytics](analytics/ci_cd_analytics.md) | | ✓ | ✓ | ✓ | ✓ | | [Analytics](analytics/index.md):<br>View [code review analytics](analytics/code_review_analytics.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | | [Analytics](analytics/index.md):<br>View [repository analytics](analytics/repository_analytics.md) | | ✓ | ✓ | ✓ | ✓ | +| [Application security](application_security/index.md):<br>View licenses in [dependency list](application_security/dependency_list/index.md) **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | +| [Application security](application_security/index.md):<br>Create and run [on-demand DAST scans](application_security/dast/index.md#on-demand-scans) **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Application security](application_security/index.md):<br>Manage [security policy](application_security/policies/index.md) **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Application security](application_security/index.md):<br>View [dependency list](application_security/dependency_list/index.md) **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Application security](application_security/index.md):<br>View [threats list](application_security/threat_monitoring/index.md#threat-monitoring) **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Application security](application_security/index.md):<br>Create a [CVE ID Request](application_security/cve_id_request.md) **(ULTIMATE SAAS)** | | | | ✓ | ✓ | +| [Application security](application_security/index.md):<br>Create or assign [security policy project](application_security/policies/index.md) **(ULTIMATE)** | | | | | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>View Security reports **(ULTIMATE)** | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>Create issue from vulnerability finding **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>Create vulnerability from vulnerability finding **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>Dismiss vulnerability **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>Dismiss vulnerability finding **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>Resolve vulnerability **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>Revert vulnerability to detected state **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>Use security dashboard **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>View vulnerability **(ULTIMATE)** | | | ✓ | ✓ | ✓ | +| [Security dashboard](application_security/security_dashboard/index.md):<br>View vulnerability findings in [dependency list](application_security/dependency_list/index.md) **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Assign issues | ✓ (*16*)| ✓ | ✓ | ✓ | ✓ | | Create [confidential issue](project/issues/confidential_issues.md) | ✓ | ✓ | ✓ | ✓ | ✓ | | Create new issue | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -73,10 +90,8 @@ The following table lists project permissions available for each role: | View GitLab Pages protected by [access control](project/pages/introduction.md#gitlab-pages-access-control) | ✓ | ✓ | ✓ | ✓ | ✓ | | View Insights **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View License Compliance reports **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | -| View licenses in Dependency list **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View project code | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View requirements **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | -| View Security reports **(ULTIMATE)** | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | | View wiki pages | ✓ | ✓ | ✓ | ✓ | ✓ | | Archive [test case](../ci/test_cases/index.md) | | ✓ | ✓ | ✓ | ✓ | | Archive/reopen requirements **(ULTIMATE)** | | ✓ | ✓ | ✓ | ✓ | @@ -108,46 +123,33 @@ The following table lists project permissions available for each role: | Assign merge requests | | | ✓ | ✓ | ✓ | | Cancel and retry jobs | | | ✓ | ✓ | ✓ | | Create and edit wiki pages | | | ✓ | ✓ | ✓ | -| Create and run [on-demand DAST scans](application_security/dast/#on-demand-scans) | | | ✓ | ✓ | ✓ | -| Create issue from vulnerability finding **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Create new branches | | | ✓ | ✓ | ✓ | | Create new environments | | | ✓ | ✓ | ✓ | | Create new merge request | | | ✓ | ✓ | ✓ | | Create or update commit status | | | ✓ (*5*) | ✓ | ✓ | -| Create vulnerability from vulnerability finding **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Create/edit/delete [releases](project/releases/index.md)| | | ✓ (*13*) | ✓ (*13*) | ✓ (*13*) | | Create/edit/delete a Cleanup policy | | | ✓ | ✓ | ✓ | | Create/edit/delete metrics dashboard annotations | | | ✓ | ✓ | ✓ | | Create/edit/delete project milestones | | | ✓ | ✓ | ✓ | -| Dismiss vulnerability **(ULTIMATE)** | | | ✓ | ✓ | ✓ | -| Dismiss vulnerability finding **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Enable Review Apps | | | ✓ | ✓ | ✓ | | Force push to non-protected branches | | | ✓ | ✓ | ✓ | | Label merge requests | | | ✓ | ✓ | ✓ | | Lock merge request threads | | | ✓ | ✓ | ✓ | | Manage Feature Flags **(PREMIUM)** | | | ✓ | ✓ | ✓ | -| Manage security policy **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Manage/Accept merge requests | | | ✓ | ✓ | ✓ | | Publish [packages](packages/index.md) | | | ✓ | ✓ | ✓ | | Push to non-protected branches | | | ✓ | ✓ | ✓ | | Read Terraform state | | | ✓ | ✓ | ✓ | | Remove a container registry image | | | ✓ | ✓ | ✓ | | Remove non-protected branches | | | ✓ | ✓ | ✓ | -| Resolve vulnerability **(ULTIMATE)** | | | ✓ | ✓ | ✓ | -| Revert vulnerability to detected state **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Rewrite/remove Git tags | | | ✓ | ✓ | ✓ | | Run CI/CD pipeline against a protected branch | | | ✓ (*5*) | ✓ | ✓ | | See a job with [debug logging](../ci/variables/index.md#debug-logging) | | | ✓ | ✓ | ✓ | | Stop environments | | | ✓ | ✓ | ✓ | | Update a container registry | | | ✓ | ✓ | ✓ | | Upload [Design Management](project/issues/design_management.md) files | | | ✓ | ✓ | ✓ | -| Use security dashboard **(ULTIMATE)** | | | ✓ | ✓ | ✓ | -| View [Threats list](application_security/threat_monitoring/#threat-monitoring) **(ULTIMATE)** | | | ✓ | ✓ | ✓ | -| View Dependency list **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | View Pods logs | | | ✓ | ✓ | ✓ | | View project Audit Events | | | ✓ (*11*) | ✓ | ✓ | -| View vulnerability **(ULTIMATE)** | | | ✓ | ✓ | ✓ | -| View vulnerability findings in Dependency list **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Add deploy keys to project | | | | ✓ | ✓ | | Add new team members | | | | ✓ | ✓ | | Configure project hooks | | | | ✓ | ✓ | @@ -175,7 +177,6 @@ The following table lists project permissions available for each role: | Manage Terraform state | | | | ✓ | ✓ | | Push to protected branches | | | | ✓ | ✓ | | Remove GitLab Pages | | | | ✓ | ✓ | -| Request a CVE ID **(FREE SAAS)** | | | | ✓ | ✓ | | Run Web IDE's Interactive Web Terminals **(ULTIMATE SELF)** | | | | ✓ | ✓ | | Share (invite) projects with groups | | | | ✓ (*8*) | ✓ (*8*)| | Turn on/off protected branch push for developers | | | | ✓ | ✓ | @@ -184,7 +185,6 @@ The following table lists project permissions available for each role: | Administer project compliance frameworks | | | | | ✓ | | Archive project | | | | | ✓ | | Change project visibility level | | | | | ✓ | -| Create or assign security policy project **(ULTIMATE)** | | | | | ✓ | | Delete issues | | | | | ✓ | | Delete merge request | | | | | ✓ | | Delete pipelines | | | | | ✓ | diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index 97696180d6b..020a7b04f88 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -30,21 +30,33 @@ Adjust your project's name, description, avatar, [default branch](../repository/ ![general project settings](img/general_settings_v13_11.png) -The project description also partially supports [standard Markdown](../../markdown.md#features-extended-from-standard-markdown). You can use [emphasis](../../markdown.md#emphasis), [links](../../markdown.md#links), and [line-breaks](../../markdown.md#line-breaks) to add more context to the project description. +The project description also partially supports [standard Markdown](../../markdown.md#features-extended-from-standard-markdown). +You can use [emphasis](../../markdown.md#emphasis), [links](../../markdown.md#links), and +[line-breaks](../../markdown.md#line-breaks) to add more context to the project description. #### Compliance frameworks **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276221) in GitLab 13.9. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/287779) in GitLab 13.12. -You can create a framework label to identify that your project has certain compliance requirements or needs additional oversight. +You can create a framework label to identify that your project has certain compliance requirements +or needs additional oversight. -Group owners can create, edit and delete compliance frameworks by going to **Settings** > **General** and expanding the **Compliance frameworks** section. -Compliance frameworks created can then be assigned to any number of projects via the project settings page inside the group or subgroups. +Group owners can create, edit, and delete compliance frameworks: + +1. Go to the group's **Settings** > **General**. +1. Expand the **Compliance frameworks** section. + +Compliance frameworks created can then be assigned to any number of projects using: + +- The project settings page inside the group or subgroups. +- In [GitLab 14.2](https://gitlab.com/gitlab-org/gitlab/-/issues/333249) and later, using the + [GraphQL API](../../../api/graphql/reference/index.md#mutationprojectsetcomplianceframework). NOTE: -Attempting to create compliance frameworks on subgroups via GraphQL will cause the framework to be created on the root ancestor if the user has the correct permissions. -The web UI presents a read-only view to discourage this behavior. +Creating compliance frameworks on subgroups with GraphQL causes the framework to be +created on the root ancestor if the user has the correct permissions. The GitLab UI presents a +read-only view to discourage this behavior. #### Compliance pipeline configuration **(ULTIMATE)** @@ -296,7 +308,7 @@ available in project listings. Only project owners and administrators have the To find an archived project: -1. Sign in to GitLab as a user with project owner or administrator permissions. +1. Sign in to GitLab as the project owner or a user with the Administrator role. 1. If you: - Have the project's URL, open the project's page in your browser. - Don't have the project's URL: @@ -417,8 +429,10 @@ To immediately delete a project marked for deletion: 1. In the "Permanently delete project" section, select **Delete project**. 1. Confirm the action when asked to. -Your project, its repository, and all related resources, including issues and merge requests, -are deleted. +The following are deleted: + +- Your project and its repository. +- All related resources including issues and merge requests. #### Restore a project **(PREMIUM)** diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index 9a6acdfc710..db0731aa524 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -104,6 +104,8 @@ Group access tokens let you use a single token to: - Perform actions at the group level. - Manage the projects within the group. +- In [GitLab 14.2](https://gitlab.com/gitlab-org/gitlab/-/issues/330718) and later, authenticate + with Git over HTTPS. We don't support group access tokens in the GitLab UI, though GitLab self-managed administrators can create them using the [Rails console](../../../administration/operations/rails_console.md). diff --git a/spec/controllers/concerns/redis_tracking_spec.rb b/spec/controllers/concerns/redis_tracking_spec.rb index 4077f4f5cce..178684ae2d0 100644 --- a/spec/controllers/concerns/redis_tracking_spec.rb +++ b/spec/controllers/concerns/redis_tracking_spec.rb @@ -3,6 +3,8 @@ require "spec_helper" RSpec.describe RedisTracking do + include TrackingHelpers + let(:user) { create(:user) } controller(ApplicationController) do @@ -60,7 +62,7 @@ RSpec.describe RedisTracking do end it 'tracks the event if DNT is not enabled' do - request.headers['DNT'] = '0' + stub_do_not_track('0') expect_tracking @@ -68,7 +70,7 @@ RSpec.describe RedisTracking do end it 'does not track the event if DNT is enabled' do - request.headers['DNT'] = '1' + stub_do_not_track('1') expect_no_tracking diff --git a/spec/controllers/projects/merge_requests/diffs_controller_spec.rb b/spec/controllers/projects/merge_requests/diffs_controller_spec.rb index 71030b5b01c..3d7636b1f30 100644 --- a/spec/controllers/projects/merge_requests/diffs_controller_spec.rb +++ b/spec/controllers/projects/merge_requests/diffs_controller_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' RSpec.describe Projects::MergeRequests::DiffsController do include ProjectForksHelper + include TrackingHelpers shared_examples '404 for unexistent diffable' do context 'when diffable does not exists' do @@ -447,7 +448,7 @@ RSpec.describe Projects::MergeRequests::DiffsController do context 'when DNT is enabled' do before do - request.headers['DNT'] = '1' + stub_do_not_track('1') end it 'does not track any mr_diffs event' do diff --git a/spec/lib/gitlab/experimentation/controller_concern_spec.rb b/spec/lib/gitlab/experimentation/controller_concern_spec.rb index 7a619c9f155..8535d72a61f 100644 --- a/spec/lib/gitlab/experimentation/controller_concern_spec.rb +++ b/spec/lib/gitlab/experimentation/controller_concern_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do + include TrackingHelpers + before do stub_const('Gitlab::Experimentation::EXPERIMENTS', { backwards_compatible_test_experiment: { @@ -43,7 +45,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do let(:cookie_value) { nil } before do - request.headers['DNT'] = do_not_track if do_not_track.present? + stub_do_not_track(do_not_track) if do_not_track.present? request.cookies[:experimentation_subject_id] = cookie_value if cookie_value get :index @@ -242,7 +244,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'do not track is disabled' do before do - request.headers['DNT'] = '0' + stub_do_not_track('0') end it 'does track the event' do @@ -260,7 +262,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'do not track enabled' do before do - request.headers['DNT'] = '1' + stub_do_not_track('1') end it 'does not track the event' do @@ -396,7 +398,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'do not track disabled' do before do - request.headers['DNT'] = '0' + stub_do_not_track('0') end it 'pushes the right parameters to gon' do @@ -414,7 +416,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'do not track enabled' do before do - request.headers['DNT'] = '1' + stub_do_not_track('1') end it 'does not push data to gon' do @@ -525,7 +527,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'is disabled' do before do - request.headers['DNT'] = '0' + stub_do_not_track('0') stub_experiment_for_subject(test_experiment: false) end @@ -538,7 +540,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do context 'is enabled' do before do - request.headers['DNT'] = '1' + stub_do_not_track('1') end it 'does not call add_user on the Experiment model' do diff --git a/spec/migrations/associate_existing_dast_builds_with_variables_spec.rb b/spec/migrations/associate_existing_dast_builds_with_variables_spec.rb new file mode 100644 index 00000000000..ce0ab4223e8 --- /dev/null +++ b/spec/migrations/associate_existing_dast_builds_with_variables_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'migrate', '20210629031900_associate_existing_dast_builds_with_variables.rb') + +RSpec.describe AssociateExistingDastBuildsWithVariables do + subject(:migration) { described_class.new } + + let_it_be(:namespaces_table) { table(:namespaces) } + let_it_be(:projects_table) { table(:projects) } + let_it_be(:ci_pipelines_table) { table(:ci_pipelines) } + let_it_be(:ci_builds_table) { table(:ci_builds) } + let_it_be(:dast_sites_table) { table(:dast_sites) } + let_it_be(:dast_site_profiles_table) { table(:dast_site_profiles) } + let_it_be(:dast_scanner_profiles_table) { table(:dast_scanner_profiles) } + let_it_be(:dast_site_profiles_builds_table) { table(:dast_site_profiles_builds) } + let_it_be(:dast_profiles_table) { table(:dast_profiles) } + let_it_be(:dast_profiles_pipelines_table) { table(:dast_profiles_pipelines) } + + let!(:group) { namespaces_table.create!(type: 'Group', name: 'group', path: 'group') } + let!(:project) { projects_table.create!(name: 'project', path: 'project', namespace_id: group.id) } + + let!(:pipeline_0) { ci_pipelines_table.create!(project_id: project.id, source: 13) } + let!(:pipeline_1) { ci_pipelines_table.create!(project_id: project.id, source: 13) } + let!(:build_0) { ci_builds_table.create!(project_id: project.id, commit_id: pipeline_0.id, name: :dast, stage: :dast) } + let!(:build_1) { ci_builds_table.create!(project_id: project.id, commit_id: pipeline_0.id, name: :dast, stage: :dast) } + let!(:build_2) { ci_builds_table.create!(project_id: project.id, commit_id: pipeline_1.id, name: :dast, stage: :dast) } + let!(:build_3) { ci_builds_table.create!(project_id: project.id, commit_id: pipeline_1.id, name: :dast) } + let!(:build_4) { ci_builds_table.create!(project_id: project.id, commit_id: pipeline_1.id, stage: :dast) } + + let!(:dast_site) { dast_sites_table.create!(project_id: project.id, url: generate(:url)) } + let!(:dast_site_profile) { dast_site_profiles_table.create!(project_id: project.id, dast_site_id: dast_site.id, name: SecureRandom.hex) } + let!(:dast_scanner_profile) { dast_scanner_profiles_table.create!(project_id: project.id, name: SecureRandom.hex) } + + let!(:dast_profile) do + dast_profiles_table.create!( + project_id: project.id, + dast_site_profile_id: dast_site_profile.id, + dast_scanner_profile_id: dast_scanner_profile.id, + name: SecureRandom.hex, + description: SecureRandom.hex + ) + end + + let!(:dast_profiles_pipeline_0) { dast_profiles_pipelines_table.create!(dast_profile_id: dast_profile.id, ci_pipeline_id: pipeline_0.id) } + let!(:dast_profiles_pipeline_1) { dast_profiles_pipelines_table.create!(dast_profile_id: dast_profile.id, ci_pipeline_id: pipeline_1.id) } + + context 'when there are ci_pipelines with associated dast_profiles' do + describe 'migration up' do + it 'adds association of dast_site_profiles to ci_builds', :aggregate_failures do + expect(dast_site_profiles_builds_table.all).to be_empty + + migration.up + + expected_results = [ + [dast_site_profile.id, build_0.id], + [dast_site_profile.id, build_1.id], + [dast_site_profile.id, build_2.id] + ] + + expect(dast_site_profiles_builds_table.all.map { |assoc| [assoc.dast_site_profile_id, assoc.ci_build_id] }).to contain_exactly(*expected_results) + end + end + end + + describe 'migration down' do + it 'deletes all records in the dast_site_profiles_builds table', :aggregate_failures do + expect(dast_site_profiles_builds_table.all).to be_empty + + migration.up + migration.down + + expect(dast_site_profiles_builds_table.all).to be_empty + end + end +end diff --git a/spec/support/helpers/tracking_helpers.rb b/spec/support/helpers/tracking_helpers.rb new file mode 100644 index 00000000000..c0374578531 --- /dev/null +++ b/spec/support/helpers/tracking_helpers.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module TrackingHelpers + def stub_do_not_track(value) + request.headers['DNT'] = value + end +end diff --git a/spec/support/shared_examples/controllers/unique_visits_shared_examples.rb b/spec/support/shared_examples/controllers/unique_visits_shared_examples.rb index 3f97c031e27..30914e61df0 100644 --- a/spec/support/shared_examples/controllers/unique_visits_shared_examples.rb +++ b/spec/support/shared_examples/controllers/unique_visits_shared_examples.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true RSpec.shared_examples 'tracking unique visits' do |method| + include TrackingHelpers + let(:request_params) { {} } it 'tracks unique visit if the format is HTML' do @@ -14,14 +16,15 @@ RSpec.shared_examples 'tracking unique visits' do |method| expect(Gitlab::UsageDataCounters::HLLRedisCounter) .to receive(:track_event).with(target_id, values: kind_of(String)) - request.headers['DNT'] = '0' + stub_do_not_track('0') get method, params: request_params, format: :html end it 'does not track unique visit if DNT is enabled' do expect(Gitlab::UsageDataCounters::HLLRedisCounter).not_to receive(:track_event) - request.headers['DNT'] = '1' + + stub_do_not_track('1') get method, params: request_params, format: :html end