diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index 590593b9d75..218dc0a7859 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -10,4 +10,4 @@ lint-yaml: variables: LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates script: - - yamllint -f colored $LINT_PATHS + - yamllint --strict -f colored $LINT_PATHS diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index e93e7b77cb4..49b75157066 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -ba88310205fe3a79a1f1eade7e5cbff65a904e4d +68091bcb23719a26f8b638cc5d1d87ee4a48cd87 diff --git a/Gemfile b/Gemfile index f85c1f0e5ad..fad1abdd00b 100644 --- a/Gemfile +++ b/Gemfile @@ -50,7 +50,7 @@ gem 'omniauth-shibboleth', '~> 1.3.0' gem 'omniauth-twitter', '~> 1.4' gem 'omniauth_crowd', '~> 2.4.0' gem 'omniauth-authentiq', '~> 0.3.3' -gem 'gitlab-omniauth-openid-connect', '~> 0.8.0', require: 'omniauth_openid_connect' +gem 'gitlab-omniauth-openid-connect', '~> 0.9.0', require: 'omniauth_openid_connect' gem 'omniauth-salesforce', '~> 1.0.5' gem 'omniauth-atlassian-oauth2', '~> 0.2.0' gem 'rack-oauth2', '~> 1.16.0' diff --git a/Gemfile.lock b/Gemfile.lock index 3d7e0b7e540..896fcc165ff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -491,7 +491,7 @@ GEM gitlab-mail_room (0.0.9) gitlab-markup (1.8.0) gitlab-net-dns (0.9.1) - gitlab-omniauth-openid-connect (0.8.0) + gitlab-omniauth-openid-connect (0.9.1) addressable (~> 2.7) omniauth (~> 1.9) openid_connect (~> 1.2) @@ -887,7 +887,7 @@ GEM nokogiri (>= 1.4.4) omniauth (~> 1.0) open4 (1.3.4) - openid_connect (1.2.0) + openid_connect (1.3.0) activemodel attr_required (>= 1.0.0) json-jwt (>= 1.5.0) @@ -1251,7 +1251,7 @@ GEM unicode-display_width (>= 1.5, < 3.0) unicode_utils (~> 1.4) strings-ansi (0.2.0) - swd (1.2.0) + swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) httpclient (>= 2.4) @@ -1358,7 +1358,7 @@ GEM safety_net_attestation (~> 0.4.0) securecompare (~> 1.0) tpm-key_attestation (~> 0.9.0) - webfinger (1.1.0) + webfinger (1.2.0) activesupport httpclient (>= 2.4) webmock (3.9.1) @@ -1484,7 +1484,7 @@ DEPENDENCIES gitlab-mail_room (~> 0.0.9) gitlab-markup (~> 1.8.0) gitlab-net-dns (~> 0.9.1) - gitlab-omniauth-openid-connect (~> 0.8.0) + gitlab-omniauth-openid-connect (~> 0.9.0) gitlab-sidekiq-fetcher (= 0.8.0) gitlab-styles (~> 6.6.0) gitlab_chronic_duration (~> 0.10.6.2) diff --git a/app/assets/javascripts/tracking/index.js b/app/assets/javascripts/tracking/index.js index 7e99ecb4f4e..d60eb37a9a2 100644 --- a/app/assets/javascripts/tracking/index.js +++ b/app/assets/javascripts/tracking/index.js @@ -46,7 +46,10 @@ export function initDefaultTrackers() { // must be after enableActivityTracking const standardContext = getStandardContext(); const experimentContexts = getAllExperimentContexts(); - window.snowplow('trackPageView', null, [standardContext, ...experimentContexts]); + // To not expose personal identifying information, the page title is hardcoded as `GitLab` + // See: https://gitlab.com/gitlab-org/gitlab/-/issues/345243 + window.snowplow('trackPageView', 'GitLab', [standardContext, ...experimentContexts]); + window.snowplow('setDocumentTitle', 'GitLab'); if (window.snowplowOptions.formTracking) { Tracking.enableFormTracking(opts.formTrackingConfig); diff --git a/db/post_migrate/20211217120000_modify_kubernetes_resource_location_index_to_vulnerability_occurrences.rb b/db/post_migrate/20211217120000_modify_kubernetes_resource_location_index_to_vulnerability_occurrences.rb new file mode 100644 index 00000000000..310a49a667e --- /dev/null +++ b/db/post_migrate/20211217120000_modify_kubernetes_resource_location_index_to_vulnerability_occurrences.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class ModifyKubernetesResourceLocationIndexToVulnerabilityOccurrences < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + OLD_CLUSTER_ID_INDEX_NAME = 'index_vulnerability_occurrences_on_location_cluster_id' + OLD_AGENT_ID_INDEX_NAME = 'index_vulnerability_occurrences_on_location_agent_id' + + NEW_CLUSTER_ID_INDEX_NAME = 'index_vulnerability_occurrences_on_location_k8s_cluster_id' + NEW_AGENT_ID_INDEX_NAME = 'index_vulnerability_occurrences_on_location_k8s_agent_id' + + def up + add_concurrent_index :vulnerability_occurrences, "(location -> 'kubernetes_resource' -> 'cluster_id')", + using: 'GIN', + where: 'report_type = 7', + name: NEW_CLUSTER_ID_INDEX_NAME + + add_concurrent_index :vulnerability_occurrences, "(location -> 'kubernetes_resource' -> 'agent_id')", + using: 'GIN', + where: 'report_type = 7', + name: NEW_AGENT_ID_INDEX_NAME + + remove_concurrent_index_by_name :vulnerability_occurrences, OLD_CLUSTER_ID_INDEX_NAME + remove_concurrent_index_by_name :vulnerability_occurrences, OLD_AGENT_ID_INDEX_NAME + end + + def down + add_concurrent_index :vulnerability_occurrences, "(location -> 'cluster_id')", + using: 'GIN', + where: 'report_type = 7', + name: OLD_CLUSTER_ID_INDEX_NAME + + add_concurrent_index :vulnerability_occurrences, "(location -> 'agent_id')", + using: 'GIN', + where: 'report_type = 7', + name: OLD_AGENT_ID_INDEX_NAME + + remove_concurrent_index_by_name :vulnerability_occurrences, NEW_CLUSTER_ID_INDEX_NAME + remove_concurrent_index_by_name :vulnerability_occurrences, NEW_AGENT_ID_INDEX_NAME + end +end diff --git a/db/schema_migrations/20211217120000 b/db/schema_migrations/20211217120000 new file mode 100644 index 00000000000..d4efb66b985 --- /dev/null +++ b/db/schema_migrations/20211217120000 @@ -0,0 +1 @@ +d4360d6057602ec1f5e6e9d11c93cfbb16d878e9ecd4d5bfb1bed1c01e14c7a3 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index c53e99e41d0..f1deeb9db5e 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -27873,12 +27873,12 @@ CREATE INDEX index_vulnerability_occurrences_deduplication ON vulnerability_occu CREATE INDEX index_vulnerability_occurrences_for_issue_links_migration ON vulnerability_occurrences USING btree (project_id, report_type, encode(project_fingerprint, 'hex'::text)); -CREATE INDEX index_vulnerability_occurrences_on_location_agent_id ON vulnerability_occurrences USING gin (((location -> 'agent_id'::text))) WHERE (report_type = 7); - -CREATE INDEX index_vulnerability_occurrences_on_location_cluster_id ON vulnerability_occurrences USING gin (((location -> 'cluster_id'::text))) WHERE (report_type = 7); - CREATE INDEX index_vulnerability_occurrences_on_location_image ON vulnerability_occurrences USING gin (((location -> 'image'::text))) WHERE (report_type = ANY (ARRAY[2, 7])); +CREATE INDEX index_vulnerability_occurrences_on_location_k8s_agent_id ON vulnerability_occurrences USING gin ((((location -> 'kubernetes_resource'::text) -> 'agent_id'::text))) WHERE (report_type = 7); + +CREATE INDEX index_vulnerability_occurrences_on_location_k8s_cluster_id ON vulnerability_occurrences USING gin ((((location -> 'kubernetes_resource'::text) -> 'cluster_id'::text))) WHERE (report_type = 7); + CREATE INDEX index_vulnerability_occurrences_on_migrated_to_new_structure ON vulnerability_occurrences USING btree (migrated_to_new_structure, id); CREATE INDEX index_vulnerability_occurrences_on_primary_identifier_id ON vulnerability_occurrences USING btree (primary_identifier_id); diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index 240f282fa27..fba151fefe1 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -201,6 +201,84 @@ The LDAP check Rake task tests the bind DN and password credentials executed as part of the `gitlab:check` task, but can run independently. See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details. +## Verify database values can be decrypted using the current secrets + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20069) in GitLab 13.1. + +This task runs through all possible encrypted values in the +database, verifying that they are decryptable using the current +secrets file (`gitlab-secrets.json`). + +Automatic resolution is not yet implemented. If you have values that +cannot be decrypted, you can follow steps to reset them, see our +docs on what to do [when the secrets file is lost](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost). + +This can take a very long time, depending on the size of your +database, as it checks all rows in all tables. + +**Omnibus Installation** + +```shell +sudo gitlab-rake gitlab:doctor:secrets +``` + +**Source Installation** + +```shell +bundle exec rake gitlab:doctor:secrets RAILS_ENV=production +``` + +**Example output** + +```plaintext +I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database +I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0 +I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0 +[...] other models possibly containing encrypted data +I, [2020-06-11T17:18:14.938335 #27148] INFO -- : - Group failures: 1 +I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0 +I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0 +I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected +I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done! +``` + +### Verbose mode + +To get more detailed information about which rows and columns can't be +decrypted, you can pass a `VERBOSE` environment variable: + +**Omnibus Installation** + +```shell +sudo gitlab-rake gitlab:doctor:secrets VERBOSE=1 +``` + +**Source Installation** + +```shell +bundle exec rake gitlab:doctor:secrets RAILS_ENV=production VERBOSE=1 +``` + +**Example verbose output** + + + +```plaintext +I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database +I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0 +I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0 +[...] other models possibly containing encrypted data +D, [2020-06-11T17:19:53.224344 #27351] DEBUG -- : > Something went wrong for Group[10].runners_token: Validation failed: Route can't be blank +I, [2020-06-11T17:19:53.225178 #27351] INFO -- : - Group failures: 1 +D, [2020-06-11T17:19:53.225267 #27351] DEBUG -- : - Group[10]: runners_token +I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0 +I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0 +I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected +I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done! +``` + + + ## Troubleshooting The following are solutions to problems you might discover using the Rake tasks documented diff --git a/doc/administration/raketasks/doctor.md b/doc/administration/raketasks/doctor.md index 02d1557b6a4..bed3cdcbcfe 100644 --- a/doc/administration/raketasks/doctor.md +++ b/doc/administration/raketasks/doctor.md @@ -1,88 +1,9 @@ --- -stage: Enablement -group: Geo -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'check.md#verify-database-values-can-be-decrypted-using-the-current-secrets' +remove_date: '2022-03-04' --- -# Doctor Rake tasks **(FREE SELF)** +This document was moved to [another location](check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). -This is a collection of tasks to help investigate and repair -problems caused by data integrity issues. - -## Verify database values can be decrypted using the current secrets - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20069) in GitLab 13.1. - -This task runs through all possible encrypted values in the -database, verifying that they are decryptable using the current -secrets file (`gitlab-secrets.json`). - -Automatic resolution is not yet implemented. If you have values that -cannot be decrypted, you can follow steps to reset them, see our -docs on what to do [when the secrets file is lost](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost). - -This can take a very long time, depending on the size of your -database, as it checks all rows in all tables. - -**Omnibus Installation** - -```shell -sudo gitlab-rake gitlab:doctor:secrets -``` - -**Source Installation** - -```shell -bundle exec rake gitlab:doctor:secrets RAILS_ENV=production -``` - -**Example output** - -```plaintext -I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database -I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0 -I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0 -[...] other models possibly containing encrypted data -I, [2020-06-11T17:18:14.938335 #27148] INFO -- : - Group failures: 1 -I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0 -I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0 -I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected -I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done! -``` - -### Verbose mode - -To get more detailed information about which rows and columns can't be -decrypted, you can pass a `VERBOSE` environment variable: - -**Omnibus Installation** - -```shell -sudo gitlab-rake gitlab:doctor:secrets VERBOSE=1 -``` - -**Source Installation** - -```shell -bundle exec rake gitlab:doctor:secrets RAILS_ENV=production VERBOSE=1 -``` - -**Example verbose output** - - - -```plaintext -I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database -I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0 -I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0 -[...] other models possibly containing encrypted data -D, [2020-06-11T17:19:53.224344 #27351] DEBUG -- : > Something went wrong for Group[10].runners_token: Validation failed: Route can't be blank -I, [2020-06-11T17:19:53.225178 #27351] INFO -- : - Group failures: 1 -D, [2020-06-11T17:19:53.225267 #27351] DEBUG -- : - Group[10]: runners_token -I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0 -I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0 -I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected -I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done! -``` - - + + diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index bd42e528bf2..d66f3b1ed35 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -120,6 +120,8 @@ You may also have a look at our troubleshooting guides for: - [GitLab](../index.md#troubleshooting) - [Omnibus GitLab](https://docs.gitlab.com/omnibus/index.html#troubleshooting) +Additionally you should also [verify database values can be decrypted using the current secrets](check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). + To run `gitlab:check`, run: **Omnibus Installation** diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 0daee7879da..07542dd73d4 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -526,7 +526,7 @@ master f05321a5b5728bd8a89b7bf530aa44043c951dce...7d02e575fd790e76 ### Find mirrors with "bad decrypt" errors -This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md). +This content has been converted to a Rake task, see [verify database values can be decrypted using the current secrets](../raketasks/check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). ### Transfer mirror users and tokens to a single service account @@ -1073,7 +1073,7 @@ area on disk. It remains to be seen exactly how or whether the deletion is usefu ### Bad Decrypt Script (for encrypted variables) -This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md). +This content has been converted to a Rake task, see [verify database values can be decrypted using the current secrets](../raketasks/check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). As an example of repairing, if `ProjectImportData Bad count:` is detected and the decision is made to delete the encrypted credentials to allow manual reentry: @@ -1116,7 +1116,7 @@ gitlab-rails runner /tmp/encrypted-tokens.rb ### Decrypt Script for encrypted tokens -This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md). +This content has been converted to a Rake task, see [verify database values can be decrypted using the current secrets](../raketasks/check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). ## Geo diff --git a/doc/api/group_protected_environments.md b/doc/api/group_protected_environments.md index 0e1cd149c51..6ce4e1791b0 100644 --- a/doc/api/group_protected_environments.md +++ b/doc/api/group_protected_environments.md @@ -48,12 +48,13 @@ Example response: "name":"production", "deploy_access_levels":[ { - "access_level":40, - "access_level_description":"Maintainers", - "user_id":null, - "group_id":null + "access_level": 40, + "access_level_description": "Maintainers", + "user_id": null, + "group_id": null } - ] + ], + "required_approval_count": 0 } ] ``` @@ -87,7 +88,8 @@ Example response: "user_id":null, "group_id":null } - ] + ], + "required_approval_count": 0 } ``` @@ -104,6 +106,7 @@ POST /groups/:id/protected_environments | `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) maintained by the authenticated user. | | `name` | string | yes | The deployment tier of the protected environment. One of `production`, `staging`, `testing`, `development`, or `other`. Read more about [deployment tiers](../ci/environments/index.md#deployment-tier-of-environments).| | `deploy_access_levels` | array | yes | Array of access levels allowed to deploy, with each described by a hash. One of `user_id`, `group_id` or `access_level`. They take the form of `{user_id: integer}`, `{group_id: integer}` or `{access_level: integer}` respectively. | +| `required_approval_count` | integer | no | The number of approvals required to deploy to this environment. This is part of Deployment Approvals, which isn't yet available for use. For details, see [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/343864). | The assignable `user_id` are the users who belong to the given group with the Maintainer role (or above). The assignable `group_id` are the sub-groups under the given group. @@ -119,12 +122,13 @@ Example response: "name":"production", "deploy_access_levels":[ { - "access_level":40, - "access_level_description":"protected-access-group", - "user_id":null, - "group_id":9899826 + "access_level": 40, + "access_level_description": "protected-access-group", + "user_id": null, + "group_id": 9899826 } - ] + ], + "required_approval_count": 0 } ``` diff --git a/doc/api/protected_environments.md b/doc/api/protected_environments.md index c7de4c504a4..61587136a14 100644 --- a/doc/api/protected_environments.md +++ b/doc/api/protected_environments.md @@ -49,7 +49,8 @@ Example response: "user_id":null, "group_id":null } - ] + ], + "required_approval_count": 0 } ] ``` @@ -78,12 +79,13 @@ Example response: "name":"production", "deploy_access_levels":[ { - "access_level":40, - "access_level_description":"Maintainers", - "user_id":null, - "group_id":null + "access_level": 40, + "access_level_description": "Maintainers", + "user_id": null, + "group_id": null } - ] + ], + "required_approval_count": 0 } ``` @@ -107,6 +109,7 @@ curl --header 'Content-Type: application/json' --request POST \ | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | | `name` | string | yes | The name of the environment. | | `deploy_access_levels` | array | yes | Array of access levels allowed to deploy, with each described by a hash. | +| `required_approval_count` | integer | no | The number of approvals required to deploy to this environment. This is part of Deployment Approvals, which isn't yet available for use. For details, see [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/343864). | Elements in the `deploy_access_levels` array should be one of `user_id`, `group_id` or `access_level`, and take the form `{user_id: integer}`, `{group_id: integer}` or @@ -125,7 +128,8 @@ Example response: "user_id": null, "group_id": 9899826 } - ] + ], + "required_approval_count": 0 } ``` diff --git a/doc/development/snowplow/implementation.md b/doc/development/snowplow/implementation.md index 6da4896c7e7..439485c9e73 100644 --- a/doc/development/snowplow/implementation.md +++ b/doc/development/snowplow/implementation.md @@ -397,6 +397,7 @@ Before you test frontend events in development, you must: All URLs are pseudonymized. The entity identifier [replaces](https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/javascript-trackers/javascript-tracker/javascript-tracker-v2/tracker-setup/other-parameters-2/#Setting_a_custom_page_URL_and_referrer_URL) personally identifiable information (PII). PII includes usernames, group, and project names. +Page titles are hardcoded as `GitLab` for the same reason. #### Snowplow Analytics Debugger Chrome Extension diff --git a/doc/development/snowplow/schemas.md b/doc/development/snowplow/schemas.md index f66e0566a9c..6bda026b9a7 100644 --- a/doc/development/snowplow/schemas.md +++ b/doc/development/snowplow/schemas.md @@ -30,6 +30,7 @@ The [`StandardContext`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/g Frontend events include a [web-specific schema](https://docs.snowplowanalytics.com/docs/understanding-your-pipeline/canonical-event/#Web-specific_fields) provided by Snowplow. All URLs are pseudonymized. The entity identifier [replaces](https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/javascript-trackers/javascript-tracker/javascript-tracker-v2/tracker-setup/other-parameters-2/#Setting_a_custom_page_URL_and_referrer_URL) personally identifiable information (PII). PII includes usernames, group, and project names. +Page titles are hardcoded as `GitLab` for the same reason. | Field Name | Required | Type | Description | |--------------------------|---------------------|-----------|----------------------------------------------------------------------------------------------------------------------------------| @@ -105,7 +106,7 @@ information (PII). PII includes usernames, group, and project names. | `os_name` | **{dotted-circle}** | string | Name of operating system | | `os_timezone` | **{dotted-circle}** | string | Client operating system time zone | | `page_referrer` | **{dotted-circle}** | string | Referrer URL | -| `page_title` | **{dotted-circle}** | string | Page title | +| `page_title` | **{dotted-circle}** | string | To not expose personal identifying information, the page title is hardcoded as `GitLab` | | `page_url` | **{dotted-circle}** | string | Page URL | | `page_urlfragment` | **{dotted-circle}** | string | Fragment aka anchor | | `page_urlhost` | **{dotted-circle}** | string | Host aka domain | diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index c1f40afe814..891b0a0311c 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -990,7 +990,7 @@ sudo gitlab-ctl restart sudo gitlab-rake gitlab:check SANITIZE=true ``` -In GitLab 13.1 and later, check [database values can be decrypted](../administration/raketasks/doctor.md) +In GitLab 13.1 and later, check [database values can be decrypted](../administration/raketasks/check.md#verify-database-values-can-be-decrypted-using-the-current-secrets) especially if `/etc/gitlab/gitlab-secrets.json` was restored, or if a different server is the target for the restore. @@ -1359,8 +1359,8 @@ Use the information in the following sections at your own risk. #### Verify that all values can be decrypted -You can determine if your database contains values that can't be decrypted by using the -[Secrets Doctor Rake task](../administration/raketasks/doctor.md). +You can determine if your database contains values that can't be decrypted by using a +[Rake task](../administration/raketasks/check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). #### Take a backup diff --git a/doc/raketasks/index.md b/doc/raketasks/index.md index 6227731e807..96c31047d8d 100644 --- a/doc/raketasks/index.md +++ b/doc/raketasks/index.md @@ -26,7 +26,6 @@ The following Rake tasks are available for use with GitLab: | [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. | | [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. | | [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. | -| [Doctor tasks](../administration/raketasks/doctor.md) | Checks for data integrity issues. | | [Elasticsearch](../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) | Maintain Elasticsearch in a GitLab instance. | | [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. | | [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. | @@ -34,7 +33,7 @@ The following Rake tasks are available for use with GitLab: | [GitHub import](../administration/raketasks/github_import.md) | Retrieve and import repositories from GitHub. | | [Import repositories](import.md) | Import bare repositories into your GitLab instance. | | [Import large project exports](../development/import_project.md#importing-via-a-rake-task) | Import large GitLab [project exports](../user/project/settings/import_export.md). | -| [Integrity checks](../administration/raketasks/check.md) | Check the integrity of repositories, files, and LDAP. | +| [Integrity checks](../administration/raketasks/check.md) | Check the integrity of repositories, files, LDAP, and more. | | [LDAP maintenance](../administration/raketasks/ldap.md) | [LDAP](../administration/auth/ldap/index.md)-related tasks. | | [List repositories](list_repos.md) | List all GitLab-managed Git repositories on disk. | | [Migrate snippets to Git](migrate_snippets.md) | Migrate GitLab Snippets to Git repositories, and show the migration status. | diff --git a/doc/update/plan_your_upgrade.md b/doc/update/plan_your_upgrade.md index 98549cc136a..82b92d89d50 100644 --- a/doc/update/plan_your_upgrade.md +++ b/doc/update/plan_your_upgrade.md @@ -35,7 +35,7 @@ to ensure the major components of GitLab are working: sudo gitlab-rake gitlab:check ``` -1. Confirm that encrypted database values [can be decrypted](../administration/raketasks/doctor.md#verify-database-values-can-be-decrypted-using-the-current-secrets): +1. Confirm that encrypted database values [can be decrypted](../administration/raketasks/check.md#verify-database-values-can-be-decrypted-using-the-current-secrets): ```shell sudo gitlab-rake gitlab:doctor:secrets diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index bb5959cb930..ba0802b3b7a 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -110,13 +110,13 @@ You can combine the filter options. For example, to list only public projects wi #### Projects pending deletion **(PREMIUM SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37014) in GitLab 13.3. -> - [Tab renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/347468) from **Deleted projects** in GitLab 14.7. +> - [Tab renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/347468) from **Deleted projects** in GitLab 14.6. When delayed project deletion is [enabled for a group](../group/index.md#enable-delayed-project-deletion), projects within that group are not deleted immediately, but only after a delay. To access a list of all projects that are pending deletion: 1. On the top bar, select **Menu > Projects > Explore projects**. -1. Select the **Pending deletion** tab (in GitLab 14.7 and later) or the **Deleted projects** tab (GitLab 14.6 and earlier). +1. Select the **Pending deletion** tab (in GitLab 14.6 and later) or the **Deleted projects** tab (GitLab 14.5 and earlier). Listed for each project is: diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 3ce2e50c548..84c01cf4baf 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -33,18 +33,33 @@ module Gitlab # changes. This set may also contain commits which are not referenced by # any of the new revisions. def commits + allow_quarantine = true + newrevs = @changes.map do |change| + oldrev = change[:oldrev] newrev = change[:newrev] - newrev unless newrev.blank? || Gitlab::Git.blank_ref?(newrev) + + next if blank_rev?(newrev) + + # In case any of the old revisions is blank, then we cannot reliably + # detect which commits are new for a given change when enumerating + # objects via the object quarantine directory given that the client + # may have pushed too many commits, and we don't know when to + # terminate the walk. We thus fall back to using `git rev-list --not + # --all`, which is a lot less efficient but at least can only ever + # returns commits which really are new. + allow_quarantine = false if allow_quarantine && blank_rev?(oldrev) + + newrev end.compact return [] if newrevs.empty? - @commits ||= project.repository.new_commits(newrevs, allow_quarantine: true) + @commits ||= project.repository.new_commits(newrevs, allow_quarantine: allow_quarantine) end # All commits which have been newly introduced via the given revision. - def commits_for(newrev) + def commits_for(oldrev, newrev) commits_by_id = commits.index_by(&:id) result = [] @@ -65,9 +80,11 @@ module Gitlab # Only add the parent ID to the pending set if we actually know its # commit to guards us against readding an ID which we have already - # queued up before. + # queued up before. Furthermore, we stop walking as soon as we hit + # `oldrev` such that we do not include any commits in our checks + # which have been "over-pushed" by the client. commit.parent_ids.each do |parent_id| - pending.add(parent_id) if commits_by_id.has_key?(parent_id) + pending.add(parent_id) if commits_by_id.has_key?(parent_id) && parent_id != oldrev end result << commit @@ -80,10 +97,10 @@ module Gitlab @single_changes_accesses ||= changes.map do |change| commits = - if change[:newrev].blank? || Gitlab::Git.blank_ref?(change[:newrev]) + if blank_rev?(change[:newrev]) [] else - Gitlab::Lazy.new { commits_for(change[:newrev]) } + Gitlab::Lazy.new { commits_for(change[:oldrev], change[:newrev]) } end Checks::SingleChangeAccess.new( @@ -109,6 +126,10 @@ module Gitlab def bulk_access_checks! Gitlab::Checks::LfsCheck.new(self).validate! end + + def blank_rev?(rev) + rev.blank? || Gitlab::Git.blank_ref?(rev) + end end end end diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake index 123a4775605..e7ece9e0fdd 100644 --- a/lib/tasks/gitlab/docs/redirect.rake +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -51,7 +51,7 @@ namespace :gitlab do post.puts "remove_date: '#{date}'" post.puts '---' post.puts - post.puts "This file was moved to [another location](#{new_path})." + post.puts "This document was moved to [another location](#{new_path})." post.puts post.puts "" post.puts "" diff --git a/spec/frontend/tracking/tracking_initialization_spec.js b/spec/frontend/tracking/tracking_initialization_spec.js index 2b70aacc4cb..f1628ad9793 100644 --- a/spec/frontend/tracking/tracking_initialization_spec.js +++ b/spec/frontend/tracking/tracking_initialization_spec.js @@ -81,7 +81,8 @@ describe('Tracking', () => { it('should activate features based on what has been enabled', () => { initDefaultTrackers(); expect(snowplowSpy).toHaveBeenCalledWith('enableActivityTracking', 30, 30); - expect(snowplowSpy).toHaveBeenCalledWith('trackPageView', null, [standardContext]); + expect(snowplowSpy).toHaveBeenCalledWith('trackPageView', 'GitLab', [standardContext]); + expect(snowplowSpy).toHaveBeenCalledWith('setDocumentTitle', 'GitLab'); expect(snowplowSpy).not.toHaveBeenCalledWith('enableFormTracking'); expect(snowplowSpy).not.toHaveBeenCalledWith('enableLinkClickTracking'); @@ -130,7 +131,7 @@ describe('Tracking', () => { it('includes those contexts alongside the standard context', () => { initDefaultTrackers(); - expect(snowplowSpy).toHaveBeenCalledWith('trackPageView', null, [ + expect(snowplowSpy).toHaveBeenCalledWith('trackPageView', 'GitLab', [ standardContext, ...experimentContexts, ]); diff --git a/spec/lib/gitlab/checks/changes_access_spec.rb b/spec/lib/gitlab/checks/changes_access_spec.rb index 633c4baa931..1cb4edd7337 100644 --- a/spec/lib/gitlab/checks/changes_access_spec.rb +++ b/spec/lib/gitlab/checks/changes_access_spec.rb @@ -44,16 +44,30 @@ RSpec.describe Gitlab::Checks::ChangesAccess do it 'calls #new_commits' do expect(project.repository).to receive(:new_commits).and_call_original - expect(subject.commits).to eq([]) + expect(subject.commits).to match_array([]) end context 'when changes contain empty revisions' do - let(:changes) { [{ newrev: newrev }, { newrev: '' }, { newrev: Gitlab::Git::BLANK_SHA }] } let(:expected_commit) { instance_double(Commit) } - it 'returns only commits with non empty revisions' do - expect(project.repository).to receive(:new_commits).with([newrev], { allow_quarantine: true }) { [expected_commit] } - expect(subject.commits).to eq([expected_commit]) + shared_examples 'returns only commits with non empty revisions' do + specify do + expect(project.repository).to receive(:new_commits).with([newrev], { allow_quarantine: allow_quarantine }) { [expected_commit] } + expect(subject.commits).to match_array([expected_commit]) + end + end + + it_behaves_like 'returns only commits with non empty revisions' do + let(:changes) { [{ oldrev: oldrev, newrev: newrev }, { newrev: '' }, { newrev: Gitlab::Git::BLANK_SHA }] } + let(:allow_quarantine) { true } + end + + context 'without oldrev' do + it_behaves_like 'returns only commits with non empty revisions' do + let(:changes) { [{ newrev: newrev }, { newrev: '' }, { newrev: Gitlab::Git::BLANK_SHA }] } + # The quarantine directory should not be used because we're lacking oldrev. + let(:allow_quarantine) { false } + end end end end @@ -61,12 +75,13 @@ RSpec.describe Gitlab::Checks::ChangesAccess do describe '#commits_for' do let(:new_commits) { [] } let(:expected_commits) { [] } + let(:oldrev) { Gitlab::Git::BLANK_SHA } shared_examples 'a listing of new commits' do it 'returns expected commits' do expect(subject).to receive(:commits).and_return(new_commits) - expect(subject.commits_for(newrev)).to eq(expected_commits) + expect(subject.commits_for(oldrev, newrev)).to eq(expected_commits) end end @@ -172,6 +187,31 @@ RSpec.describe Gitlab::Checks::ChangesAccess do it_behaves_like 'a listing of new commits' end + + context 'with over-push' do + let(:newrev) { '1' } + let(:oldrev) { '3' } + + # `#new_commits` returns too many commits, where some commits are not + # part of the current change. + let(:new_commits) do + [ + create_commit('1', %w[2]), + create_commit('2', %w[3]), + create_commit('3', %w[4]), + create_commit('4', %w[]) + ] + end + + let(:expected_commits) do + [ + create_commit('1', %w[2]), + create_commit('2', %w[3]) + ] + end + + it_behaves_like 'a listing of new commits' + end end describe '#single_change_accesses' do @@ -180,10 +220,10 @@ RSpec.describe Gitlab::Checks::ChangesAccess do shared_examples '#single_change_access' do before do - commits_for.each do |id, commits| + commits_for.each do |oldrev, newrev, commits| expect(subject) .to receive(:commits_for) - .with(id) + .with(oldrev, newrev) .and_return(commits) end end @@ -205,7 +245,12 @@ RSpec.describe Gitlab::Checks::ChangesAccess do end context 'with a single change and no new commits' do - let(:commits_for) { { 'new' => [] } } + let(:commits_for) do + [ + ['old', 'new', []] + ] + end + let(:changes) do [ { oldrev: 'old', newrev: 'new', ref: 'refs/heads/branch' } @@ -222,7 +267,12 @@ RSpec.describe Gitlab::Checks::ChangesAccess do end context 'with a single change and new commits' do - let(:commits_for) { { 'new' => [create_commit('new', [])] } } + let(:commits_for) do + [ + ['old', 'new', [create_commit('new', [])]] + ] + end + let(:changes) do [ { oldrev: 'old', newrev: 'new', ref: 'refs/heads/branch' } @@ -240,11 +290,11 @@ RSpec.describe Gitlab::Checks::ChangesAccess do context 'with multiple changes' do let(:commits_for) do - { - 'a' => [create_commit('a', [])], - 'c' => [create_commit('c', [])], - 'd' => [] - } + [ + [nil, 'a', [create_commit('a', [])]], + ['a', 'c', [create_commit('c', [])]], + [nil, 'd', []] + ] end let(:changes) do