diff --git a/app/models/user.rb b/app/models/user.rb index dc2f7ff5b5e..16ed3205d93 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -214,14 +214,15 @@ class User < ApplicationRecord has_many :spam_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :builds, class_name: 'Ci::Build' has_many :pipelines, class_name: 'Ci::Pipeline' - has_many :todos + has_many :todos, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :authored_todos, class_name: 'Todo', dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :notification_settings has_many :award_emoji, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :triggers, class_name: 'Ci::Trigger', foreign_key: :owner_id has_many :issue_assignees, inverse_of: :assignee - has_many :merge_request_assignees, inverse_of: :assignee - has_many :merge_request_reviewers, inverse_of: :reviewer + has_many :merge_request_assignees, inverse_of: :assignee, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :merge_request_reviewers, inverse_of: :reviewer, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, class_name: "MergeRequest", through: :merge_request_assignees, source: :merge_request has_many :created_custom_emoji, class_name: 'CustomEmoji', inverse_of: :creator @@ -254,6 +255,8 @@ class User < ApplicationRecord has_many :timelogs has_many :resource_label_events, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent + has_many :resource_state_events, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent + has_many :authored_events, class_name: 'Event', dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent # # Validations diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 904905de75c..d77ffb97293 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -212,5 +212,5 @@ successfully, you must replicate their data using some other means. |[Alert Metric Images](../../../operations/incident_management/alerts.md#metrics-tab) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/362564) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362564) | No | No | | |[Server-side Git hooks](../../server_hooks.md) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | N/A | N/A | Not planned because of current implementation complexity, low customer interest, and availability of alternatives to hooks. | |[Elasticsearch integration](../../../integration/advanced_search/elasticsearch.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | No | Not planned because further product discovery is required and Elasticsearch (ES) clusters can be rebuilt. Secondaries use the same ES cluster as the primary. | -|[Dependency proxy images](../../../user/packages/dependency_proxy/index.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/259694) | No | No | No | Blocked by [Geo: Secondary Mimicry](https://gitlab.com/groups/gitlab-org/-/epics/1528). Replication of this cache is not needed for disaster recovery purposes because it can be recreated from external sources. | +|[Dependency proxy images](../../../user/packages/dependency_proxy/index.md) | [Planned](https://gitlab.com/groups/gitlab-org/-/epics/8833) | No | No | No | Blocked by [Geo: Secondary Mimicry](https://gitlab.com/groups/gitlab-org/-/epics/1528). Replication of this cache is not needed for disaster recovery purposes because it can be recreated from external sources. | |[Vulnerability Export](../../../user/application_security/vulnerability_report/index.md#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | No | No | Not planned because they are ephemeral and sensitive information. They can be regenerated on demand. | diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 8ea7dc866e4..3f16c1552ad 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -311,6 +311,51 @@ sudo gitlab-rake gitlab:geo:check When performing a PostgreSQL major version (9 > 10) update this is expected. Follow the [initiate-the-replication-process](../setup/database.md#step-3-initiate-the-replication-process). +### Repository verification failures + +[Start a Rails console session](../../../administration/operations/rails_console.md#starting-a-rails-console-session) +to gather the following, basic troubleshooting information. + +WARNING: +Any command that changes data directly could be damaging if not run correctly, or under the right conditions. We highly recommend running them in a test environment with a backup of the instance ready to be restored, just in case. + +#### Get the number of verification failed repositories + +```ruby +Geo::ProjectRegistry.verification_failed('repository').count +``` + +#### Find the verification failed repositories + +```ruby +Geo::ProjectRegistry.verification_failed('repository') +``` + +#### Find repositories that failed to sync + +```ruby +Geo::ProjectRegistry.sync_failed('repository') +``` + +### Resync repositories + +[Start a Rails console session](../../../administration/operations/rails_console.md#starting-a-rails-console-session) +to enact the following, basic troubleshooting steps. + +#### Queue up all repositories for resync. Sidekiq handles each sync + +```ruby +Geo::ProjectRegistry.update_all(resync_repository: true, resync_wiki: true) +``` + +#### Sync individual repository now + +```ruby +project = Project.find_by_full_path('') + +Geo::RepositorySyncService.new(project).execute +``` + ## Fixing replication errors The following sections outline troubleshooting steps for fixing replication @@ -779,7 +824,7 @@ This behavior affects only the following data types through GitLab 14.6: | Data type | From version | | ------------------------ | ------------ | | Package Registry | 13.10 | -| Pipeline Artifacts | 13.11 | +| CI Pipeline Artifacts | 13.11 | | Terraform State Versions | 13.12 | | Infrastructure Registry | 14.0 | | External MR diffs | 14.6 | @@ -792,6 +837,120 @@ This behavior affects only the following data types through GitLab 14.6: to make Geo visibly surface data loss risks. The sync/verification loop is therefore short-circuited. `last_sync_failure` is now set to `The file is missing on the Geo primary site`. +### Blob types + +- `Ci::JobArtifact` +- `Ci::PipelineArtifact` +- `Ci::SecureFile` +- `LfsObject` +- `MergeRequestDiff` +- `Packages::PackageFile` +- `PagesDeployment` +- `Terraform::StateVersion` +- `Upload` + +`Packages::PackageFile` is used in the following +[Rails console](../../../administration/operations/rails_console.md#starting-a-rails-console-session) +examples, but things generally work the same for the other types. + +WARNING: +Any command that changes data directly could be damaging if not run correctly, or under the right conditions. We highly recommend running them in a test environment with a backup of the instance ready to be restored, just in case. + +#### The Replicator + +The main kinds of classes are Registry, Model, and Replicator. If you have an instance of one of these classes, you can get the others. The Registry and Model mostly manage PostgreSQL DB state. The Replicator knows how to replicate/verify (or it can call a service to do it): + +```ruby +model_record = Packages::PackageFile.last +model_record.replicator.registry.replicator.model_record # just showing that these methods exist +``` + +#### Replicate a package file, synchronously, given an ID + +```ruby +model_record = Packages::PackageFile.find(id) +model_record.replicator.send(:download) +``` + +#### Replicate a package file, synchronously, given a registry ID + +```ruby +registry = Geo::PackageFileRegistry.find(registry_id) +registry.replicator.send(:download) +``` + +#### Verify package files on the secondary manually + +This iterates over all package files on the secondary, looking at the +`verification_checksum` stored in the database (which came from the primary) +and then calculate this value on the secondary to check if they match. This +does not change anything in the UI: + +```ruby +# Run on secondary +status = {} + +Packages::PackageFile.find_each do |package_file| + primary_checksum = package_file.verification_checksum + secondary_checksum = Packages::PackageFile.hexdigest(package_file.file.path) + verification_status = (primary_checksum == secondary_checksum) + + status[verification_status.to_s] ||= [] + status[verification_status.to_s] << package_file.id +end + +# Count how many of each value we get +status.keys.each {|key| puts "#{key} count: #{status[key].count}"} + +# See the output in its entirety +status +``` + +### Repository types newer than project/wiki repositories + +- `SnippetRepository` +- `GroupWikiRepository` + +`SnippetRepository` is used in the examples below, but things generally work the same for the other Repository types. + +#### The Replicator + +The main kinds of classes are Registry, Model, and Replicator. If you have an instance of one of these classes, you can get the others. The Registry and Model mostly manage PostgreSQL DB state. The Replicator knows how to replicate/verify (or it can call a service to do it). + +```ruby +model_record = SnippetRepository.last +model_record.replicator.registry.replicator.model_record # just showing that these methods exist +``` + +#### Replicate a snippet repository, synchronously, given an ID + +```ruby +model_record = SnippetRepository.find(id) +model_record.replicator.send(:sync_repository) +``` + +#### Replicate a snippet repository, synchronously, given a registry ID + +```ruby +registry = Geo::SnippetRepositoryRegistry.find(registry_id) +registry.replicator.send(:sync_repository) +``` + +### Find failed artifacts + +[Start a Rails console session](../../../administration/operations/rails_console.md#starting-a-rails-console-session) +to run the following commands: + +```ruby +Geo::JobArtifactRegistry.failed +``` + +#### Find `ID` of synced artifacts that are missing on primary + +```ruby +Geo::JobArtifactRegistry.synced.missing_on_primary.pluck(:artifact_id) +``` + #### Failed syncs with GitLab-managed object storage replication There is [an issue in GitLab 14.2 through 14.7](https://gitlab.com/gitlab-org/gitlab/-/issues/299819#note_822629467) diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 8e761b3ef74..922f9a27aad 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -243,7 +243,6 @@ control over how the Pages daemon runs and serves content in your environment. | `artifacts_server_url` | API URL to proxy artifact requests to. Defaults to GitLab `external URL` + `/api/v4`, for example `https://gitlab.com/api/v4`. When running a [separate Pages server](#running-gitlab-pages-on-a-separate-server), this URL must point to the main GitLab server's API. | | `auth_redirect_uri` | Callback URL for authenticating with GitLab. Defaults to project's subdomain of `pages_external_url` + `/auth`. | | `auth_secret` | Secret key for signing authentication requests. Leave blank to pull automatically from GitLab during OAuth registration. | -| `client_cert_key_pairs` | Client certificates and keys used for mutual TLS with the GitLab API. See [Support mutual TLS when calling the GitLab API](#support-mutual-tls-when-calling-the-gitlab-api) for details. [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/548) in GitLab 14.8. | | `dir` | Working directory for configuration and secrets files. | | `enable` | Enable or disable GitLab Pages on the current system. | | `external_http` | Configure Pages to bind to one or more secondary IP addresses, serving HTTP requests. Multiple addresses can be given as an array, along with exact ports, for example `['1.2.3.4', '1.2.3.5:8063']`. Sets value for `listen_http`. | @@ -525,22 +524,6 @@ Authority (CA) in the system certificate store. For Omnibus, this is fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). -### Support mutual TLS when calling the GitLab API - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/548) in GitLab 14.8. - -If GitLab has been [configured to require mutual TLS](https://docs.gitlab.com/omnibus/settings/ssl.html#enable-2-way-ssl-client-authentication), you need to add the client certificates to Pages: - -1. Configure in `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_pages['client_cert_key_pairs'] = [':'] - ``` - - Where `` and `` are the file paths to the client certificate and its respective key file. - Both of these files must be encoded in PEM format. -1. To configure Pages to validate the server certificates, [add the root CA to the system trust store](#using-a-custom-certificate-authority-ca). - ### ZIP serving and cache configuration > [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/392) in GitLab 13.7. @@ -754,7 +737,7 @@ To set the maximum number of GitLab Pages custom domains for a project: ## Running GitLab Pages on a separate server You can run the GitLab Pages daemon on a separate server to decrease the load on -your main application server. +your main application server. This configuration does not support mutual TLS (mTLS). See the [corresponding feature proposal](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/548) for more information. To configure GitLab Pages on a separate server: diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index bdfde61e778..6893f8f1a39 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -46,7 +46,7 @@ The following Cloud Native Hybrid reference architectures, where select recommen A GitLab [Premium or Ultimate](https://about.gitlab.com/pricing/#self-managed) license is required to get assistance from Support with troubleshooting the [2,000 users](2k_users.md) and higher reference architectures. -[Read more about our definition of scaled architectures](https://about.gitlab.com/support/#definition-of-scaled-architecture). +[Read more about our definition of scaled architectures](https://about.gitlab.com/support/definitions/#definition-of-scaled-architecture). ## Deciding which architecture to use diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index f1775768aac..4358af585df 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -725,152 +725,19 @@ There is an [issue to implement this functionality in the Admin UI](https://gitl ### Artifacts -#### Find failed artifacts - -```ruby -Geo::JobArtifactRegistry.failed -``` - -#### Get a count of the synced artifacts - -```ruby -Geo::JobArtifactRegistry.synced.count -``` - -#### Find `ID` of synced artifacts that are missing on primary - -```ruby -Geo::JobArtifactRegistry.synced.missing_on_primary.pluck(:artifact_id) -``` +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#find-failed-artifacts). ### Repository verification failures -#### Get the number of verification failed repositories - -```ruby -Geo::ProjectRegistry.verification_failed('repository').count -``` - -#### Find the verification failed repositories - -```ruby -Geo::ProjectRegistry.verification_failed('repository') -``` - -### Find repositories that failed to sync - -```ruby -Geo::ProjectRegistry.sync_failed('repository') -``` +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#repository-verification-failures). ### Resync repositories -#### Queue up all repositories for resync. Sidekiq handles each sync - -```ruby -Geo::ProjectRegistry.update_all(resync_repository: true, resync_wiki: true) -``` - -#### Sync individual repository now - -```ruby -project = Project.find_by_full_path('') - -Geo::RepositorySyncService.new(project).execute -``` +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#resync-repositories). ### Blob types -- `Ci::JobArtifact` -- `Ci::PipelineArtifact` -- `LfsObject` -- `MergeRequestDiff` -- `Packages::PackageFile` -- `PagesDeployment` -- `Terraform::StateVersion` -- `Upload` - -`Packages::PackageFile` is used in the following examples, but things generally work the same for the other Blob types. - -#### The Replicator - -The main kinds of classes are Registry, Model, and Replicator. If you have an instance of one of these classes, you can get the others. The Registry and Model mostly manage PostgreSQL DB state. The Replicator knows how to replicate/verify (or it can call a service to do it): - -```ruby -model_record = Packages::PackageFile.last -model_record.replicator.registry.replicator.model_record # just showing that these methods exist -``` - -#### Replicate a package file, synchronously, given an ID - -```ruby -model_record = Packages::PackageFile.find(id) -model_record.replicator.send(:download) -``` - -#### Replicate a package file, synchronously, given a registry ID - -```ruby -registry = Geo::PackageFileRegistry.find(registry_id) -registry.replicator.send(:download) -``` - -#### Verify package files on the secondary manually - -This iterates over all package files on the secondary, looking at the -`verification_checksum` stored in the database (which came from the primary) -and then calculate this value on the secondary to check if they match. This -does not change anything in the UI: - -```ruby -# Run on secondary -status = {} - -Packages::PackageFile.find_each do |package_file| - primary_checksum = package_file.verification_checksum - secondary_checksum = Packages::PackageFile.hexdigest(package_file.file.path) - verification_status = (primary_checksum == secondary_checksum) - - status[verification_status.to_s] ||= [] - status[verification_status.to_s] << package_file.id -end - -# Count how many of each value we get -status.keys.each {|key| puts "#{key} count: #{status[key].count}"} - -# See the output in its entirety -status -``` - -### Repository types newer than project/wiki repositories - -- `SnippetRepository` -- `GroupWikiRepository` - -`SnippetRepository` is used in the examples below, but things generally work the same for the other Repository types. - -#### The Replicator - -The main kinds of classes are Registry, Model, and Replicator. If you have an instance of one of these classes, you can get the others. The Registry and Model mostly manage PostgreSQL DB state. The Replicator knows how to replicate/verify (or it can call a service to do it). - -```ruby -model_record = SnippetRepository.last -model_record.replicator.registry.replicator.model_record # just showing that these methods exist -``` - -#### Replicate a snippet repository, synchronously, given an ID - -```ruby -model_record = SnippetRepository.find(id) -model_record.replicator.send(:sync_repository) -``` - -#### Replicate a snippet repository, synchronously, given a registry ID - -```ruby -registry = Geo::SnippetRepositoryRegistry.find(registry_id) -registry.replicator.send(:sync_repository) -``` +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#blob-types). ## Generate Service Ping diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index bceb7167645..e6c3c20d50b 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -55,7 +55,7 @@ change feature flags or you do not have access. ### Enabling a feature for pre-production testing As a first step in a feature rollout, you should enable the feature on -[`about.staging.gitlab.com`](https://about.staging.gitlab.com) +[`staging.gitlab.com`](https://staging.gitlab.com) and [`dev.gitlab.org`](https://dev.gitlab.org). These two environments have different scopes. diff --git a/doc/operations/incident_management/linked_resources.md b/doc/operations/incident_management/linked_resources.md index 70ee9816b73..4c3ef34f634 100644 --- a/doc/operations/incident_management/linked_resources.md +++ b/doc/operations/incident_management/linked_resources.md @@ -50,6 +50,24 @@ To add a linked resource: 1. Complete the required fields. 1. Select **Add**. +### Using a quick action **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/374964) in GitLab 15.5. + +To add multiple links to an incident, use the `/link` +[quick action](../../user/project/quick_actions.md): + +```plaintext +/link https://example.link.us/j/123456789 +``` + +You can also submit a short description with the link. +The description shows instead of the URL in the **Linked resources** section of the incident: + +```plaintext +/link https://example.link.us/j/123456789, multiple alerts firing +``` + ### Link Zoom meetings from an incident **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230853) in GitLab 15.4. diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index b77401a1098..3471123f8b5 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -80,6 +80,7 @@ threads. Some quick actions might not be available to all subscription tiers. | `/iteration *iteration:"iteration name"` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set iteration. For example, to set the `Late in July` iteration: `/iteration *iteration:"Late in July"` ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196795) in GitLab 13.1). | | `/label ~label1 ~label2` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Add one or more labels. Label names can also start without a tilde (`~`), but mixed syntax is not supported. | | `/lock` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Lock the discussions. | +| `/link` | **{check-circle}** Yes | ****{dotted-circle}** No | **{dotted-circle}** No | Add a link and description to [linked resources](../../operations/incident_management/linked_resources.md) in an incident ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/374964) in GitLab 15.5). | | `/merge` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Merge changes. Depending on the project setting, this may be [when the pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md), or adding to a [Merge Train](../../ci/pipelines/merge_trains.md). | | `/milestone %milestone` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Set milestone. | | `/move ` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Move this issue to another project. Be careful when moving an issue to a project with different access rules. Before moving the issue, make sure it does not contain sensitive data. | diff --git a/doc/user/project/repository/managing_large_repositories.md b/doc/user/project/repository/managing_large_repositories.md index 0a91fada6f8..e5c1e4a6614 100644 --- a/doc/user/project/repository/managing_large_repositories.md +++ b/doc/user/project/repository/managing_large_repositories.md @@ -14,11 +14,11 @@ In the following sections, we detail several best practices for improving perfor ## Large File System (LFS) -It's *strongly* recommended in any Git system that binary or blob files (for example, packages, audio, video, or graphics) are stored as Large File Storage (LFS) objects. In such setup, the Objects are stored elsewhere, such as in Object Storage, and this can reduce the repository size significantly, thus improving performance. +It's *strongly* recommended in any Git system that binary or blob files (for example, packages, audio, video, or graphics) are stored as Large File Storage (LFS) objects. With LFS, the objects are stored externally, such as in Object Storage, which reduces the number and size of objects in the repository. Storing objects in external Object Storage can improve performance. -To analyze if the repository has these sorts of objects, it's recommended to run a tool like [`git-sizer`](https://github.com/github/git-sizer) to get a detailed analysis. These tools can show in detail what makes up the repository as well as highlights any areas of concern. If any large objects are found, it's then recommended removing them with tools such as [`git filter-repo`](reducing_the_repo_size_using_git.md). +To analyze if a repository has large objects, you can use a tool like [`git-sizer`](https://github.com/github/git-sizer) for detailed analysis. This tool shows details about what makes up the repository, and highlights any areas of concern. If any large objects are found, you can then remove them with a tool such as [`git filter-repo`](reducing_the_repo_size_using_git.md). -Refer to the [Git LFS documentation for more information](../../../topics/git/lfs/index.md). +For more information, refer to the [Git LFS documentation](../../../topics/git/lfs/index.md). ## Gitaly Pack Objects Cache diff --git a/qa/Gemfile b/qa/Gemfile index 16413642eb9..4dd02c7b3d8 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -10,14 +10,14 @@ gem 'capybara-screenshot', '~> 1.0.26' gem 'rake', '~> 13' gem 'rspec', '~> 3.10' gem 'selenium-webdriver', '~> 4.5' -gem 'airborne', '~> 0.3.4', require: false # airborne is messing with rspec sandboxed mode so not requiring by default +gem 'airborne', '~> 0.3.7', require: false # airborne is messing with rspec sandboxed mode so not requiring by default gem 'rest-client', '~> 2.1.0' gem 'rspec-retry', '~> 0.6.1', require: 'rspec/retry' -gem 'rspec_junit_formatter', '~> 0.4.1' +gem 'rspec_junit_formatter', '~> 0.6.0' gem 'faker', '~> 2.19', '>= 2.19.0' gem 'knapsack', '~> 4.0' gem 'parallel_tests', '~> 2.32' -gem 'rotp', '~> 3.1.0' +gem 'rotp', '~> 6.2.0' gem 'timecop', '~> 0.9.5' gem 'parallel', '~> 1.19' gem 'rainbow', '~> 3.0.0' @@ -35,7 +35,7 @@ gem "warning", "~> 1.3" gem 'confiner', '~> 0.3' -gem 'chemlab', '~> 0.9' +gem 'chemlab', '~> 0.10' gem 'chemlab-library-www-gitlab-com', '~> 0.1' # dependencies for jenkins client diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 99bfea9a760..280d0a72c29 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -9,7 +9,7 @@ GEM zeitwerk (~> 2.3) addressable (2.8.1) public_suffix (>= 2.0.2, < 6.0) - airborne (0.3.4) + airborne (0.3.7) activesupport rack rack-test (>= 1.1.0, < 2.0) @@ -38,7 +38,7 @@ GEM capybara-screenshot (1.0.26) capybara (>= 1.0, < 4) launchy - chemlab (0.9.2) + chemlab (0.10.0) colorize (~> 0.8) i18n (~> 1.8) rake (>= 12, < 14) @@ -219,7 +219,7 @@ GEM netrc (~> 0.8) retriable (3.1.2) rexml (3.2.5) - rotp (3.1.0) + rotp (6.2.0) rspec (3.10.0) rspec-core (~> 3.10.0) rspec-expectations (~> 3.10.0) @@ -241,7 +241,7 @@ GEM rspec-retry (0.6.1) rspec-core (> 3.3) rspec-support (3.10.2) - rspec_junit_formatter (0.4.1) + rspec_junit_formatter (0.6.0) rspec-core (>= 2, < 4, != 2.12.0) ruby-debug-ide (0.7.3) rake (>= 0.8.1) @@ -280,9 +280,9 @@ GEM uuid (2.3.9) macaddr (~> 1.0) warning (1.3.0) - watir (6.19.1) + watir (7.1.0) regexp_parser (>= 1.2, < 3) - selenium-webdriver (>= 3.142.7) + selenium-webdriver (~> 4.0) webdrivers (5.2.0) nokogiri (~> 1.6) rubyzip (>= 1.3.0) @@ -298,11 +298,11 @@ PLATFORMS DEPENDENCIES activesupport (~> 6.1.4.7) - airborne (~> 0.3.4) + airborne (~> 0.3.7) allure-rspec (~> 2.16.0) capybara (~> 3.35.0) capybara-screenshot (~> 1.0.26) - chemlab (~> 0.9) + chemlab (~> 0.10) chemlab-library-www-gitlab-com (~> 0.1) confiner (~> 0.3) deprecation_toolkit (~> 2.0.0) @@ -321,11 +321,11 @@ DEPENDENCIES rainbow (~> 3.0.0) rake (~> 13) rest-client (~> 2.1.0) - rotp (~> 3.1.0) + rotp (~> 6.2.0) rspec (~> 3.10) rspec-parameterized (~> 0.5.2) rspec-retry (~> 0.6.1) - rspec_junit_formatter (~> 0.4.1) + rspec_junit_formatter (~> 0.6.0) ruby-debug-ide (~> 0.7.3) selenium-webdriver (~> 4.5) slack-notifier (~> 2.4) diff --git a/spec/lib/gitlab/regex_requires_app_spec.rb b/spec/lib/gitlab/regex_requires_app_spec.rb index 5808033dc4c..780184cdfd2 100644 --- a/spec/lib/gitlab/regex_requires_app_spec.rb +++ b/spec/lib/gitlab/regex_requires_app_spec.rb @@ -30,6 +30,8 @@ RSpec.describe Gitlab::Regex do it { is_expected.not_to match('AMD64') } it { is_expected.not_to match('Amd64') } it { is_expected.not_to match('aMD64') } + + it_behaves_like 'regex rejecting path traversal' end describe '.npm_package_name_regex' do @@ -73,6 +75,8 @@ RSpec.describe Gitlab::Regex do # Do not allow Unicode it { is_expected.not_to match('hé') } + + it_behaves_like 'regex rejecting path traversal' end describe '.debian_component_regex' do @@ -86,5 +90,7 @@ RSpec.describe Gitlab::Regex do # Do not allow Unicode it { is_expected.not_to match('hé') } + + it_behaves_like 'regex rejecting path traversal' end end diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index d8f182d903d..89ef76d246e 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -3,6 +3,7 @@ require 'fast_spec_helper' require_relative '../../../lib/gitlab/regex' +require_relative '../../support/shared_examples/lib/gitlab/regex_shared_examples' # All specs that can be run with fast_spec_helper only # See regex_requires_app_spec for tests that require the full spec_helper @@ -543,6 +544,8 @@ RSpec.describe Gitlab::Regex do it { is_expected.not_to match('aA') } # No underscore it { is_expected.not_to match('a_b') } + + it_behaves_like 'regex rejecting path traversal' end describe '.debian_version_regex' do @@ -596,6 +599,13 @@ RSpec.describe Gitlab::Regex do it { is_expected.to match('1-2-3-4-5-6-7-8-9-10-11-12-13-14-15') } it { is_expected.not_to match('1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16') } end + + context 'path traversals' do + it { is_expected.not_to match('1../0') } + it { is_expected.not_to match('1..%2f0') } + it { is_expected.not_to match('1%2e%2e%2f0') } + it { is_expected.not_to match('1%2e%2e/0') } + end end describe '.helm_channel_regex' do diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb index efaac06c1f2..03e1811c8a5 100644 --- a/spec/services/users/destroy_service_spec.rb +++ b/spec/services/users/destroy_service_spec.rb @@ -388,42 +388,95 @@ RSpec.describe Users::DestroyService do context 'batched nullify' do let(:other_user) { create(:user) } + # rubocop:disable Layout/LineLength + def nullify_in_batches_regexp(table, column, user, batch_size: 100) + %r{^UPDATE "#{table}" SET "#{column}" = NULL WHERE "#{table}"."id" IN \(SELECT "#{table}"."id" FROM "#{table}" WHERE "#{table}"."#{column}" = #{user.id} LIMIT #{batch_size}\)} + end + + def delete_in_batches_regexps(table, column, user, items, batch_size: 1000) + select_query = %r{^SELECT "#{table}".* FROM "#{table}" WHERE "#{table}"."#{column}" = #{user.id}.*ORDER BY "#{table}"."id" ASC LIMIT #{batch_size}} + + [select_query] + items.map { |item| %r{^DELETE FROM "#{table}" WHERE "#{table}"."id" = #{item.id}} } + end + # rubocop:enable Layout/LineLength + it 'nullifies related associations in batches' do expect(other_user).to receive(:nullify_dependent_associations_in_batches).and_call_original described_class.new(user).execute(other_user, skip_authorization: true) end - it 'nullifies last_updated_issues, closed_issues, resource_label_events' do + it 'nullifies issues and resource associations', :aggregate_failures do issue = create(:issue, closed_by: other_user, updated_by: other_user) resource_label_event = create(:resource_label_event, user: other_user) + resource_state_event = create(:resource_state_event, user: other_user) + todos = create_list(:todo, 2, project: issue.project, user: other_user, author: other_user, target: issue) + event = create(:event, project: issue.project, author: other_user) - described_class.new(user).execute(other_user, skip_authorization: true) + query_recorder = ActiveRecord::QueryRecorder.new do + described_class.new(user).execute(other_user, skip_authorization: true) + end issue.reload resource_label_event.reload + resource_state_event.reload expect(issue.closed_by).to be_nil expect(issue.updated_by).to be_nil expect(resource_label_event.user).to be_nil + expect(resource_state_event.user).to be_nil + expect(other_user.authored_todos).to be_empty + expect(other_user.todos).to be_empty + expect(other_user.authored_events).to be_empty + + expected_queries = [ + nullify_in_batches_regexp(:issues, :updated_by_id, other_user), + nullify_in_batches_regexp(:issues, :closed_by_id, other_user), + nullify_in_batches_regexp(:resource_label_events, :user_id, other_user), + nullify_in_batches_regexp(:resource_state_events, :user_id, other_user) + ] + + expected_queries += delete_in_batches_regexps(:todos, :user_id, other_user, todos) + expected_queries += delete_in_batches_regexps(:todos, :author_id, other_user, todos) + expected_queries += delete_in_batches_regexps(:events, :author_id, other_user, [event]) + + expect(query_recorder.log).to include(*expected_queries) end - it 'nullifies merge request associations' do + it 'nullifies merge request associations', :aggregate_failures do merge_request = create(:merge_request, source_project: project, target_project: project, assignee: other_user, updated_by: other_user, merge_user: other_user) merge_request.metrics.update!(merged_by: other_user, latest_closed_by: other_user) + merge_request.reviewers = [other_user] + merge_request.assignees = [other_user] - described_class.new(user).execute(other_user, skip_authorization: true) + query_recorder = ActiveRecord::QueryRecorder.new do + described_class.new(user).execute(other_user, skip_authorization: true) + end merge_request.reload - aggregate_failures do - expect(merge_request.updated_by).to be_nil - expect(merge_request.assignee).to be_nil - expect(merge_request.assignee_id).to be_nil - expect(merge_request.metrics.merged_by).to be_nil - expect(merge_request.metrics.latest_closed_by).to be_nil - end + expect(merge_request.updated_by).to be_nil + expect(merge_request.assignee).to be_nil + expect(merge_request.assignee_id).to be_nil + expect(merge_request.metrics.merged_by).to be_nil + expect(merge_request.metrics.latest_closed_by).to be_nil + expect(merge_request.reviewers).to be_empty + expect(merge_request.assignees).to be_empty + + expected_queries = [ + nullify_in_batches_regexp(:merge_requests, :updated_by_id, other_user), + nullify_in_batches_regexp(:merge_requests, :assignee_id, other_user), + nullify_in_batches_regexp(:merge_request_metrics, :merged_by_id, other_user), + nullify_in_batches_regexp(:merge_request_metrics, :latest_closed_by_id, other_user) + ] + + expected_queries += delete_in_batches_regexps(:merge_request_assignees, :user_id, other_user, + merge_request.assignees) + expected_queries += delete_in_batches_regexps(:merge_request_reviewers, :user_id, other_user, + merge_request.reviewers) + + expect(query_recorder.log).to include(*expected_queries) end end end diff --git a/spec/support/shared_examples/lib/gitlab/regex_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/regex_shared_examples.rb new file mode 100644 index 00000000000..150741c6344 --- /dev/null +++ b/spec/support/shared_examples/lib/gitlab/regex_shared_examples.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'regex rejecting path traversal' do + it { is_expected.not_to match('a../b') } + it { is_expected.not_to match('a..%2fb') } + it { is_expected.not_to match('a%2e%2e%2fb') } + it { is_expected.not_to match('a%2e%2e/b') } +end diff --git a/spec/tooling/danger/specs_spec.rb b/spec/tooling/danger/specs_spec.rb index 607cbf5f376..d6aed86e7dc 100644 --- a/spec/tooling/danger/specs_spec.rb +++ b/spec/tooling/danger/specs_spec.rb @@ -167,9 +167,6 @@ RSpec.describe Tooling::Danger::Specs do " str = 'let(:project) { create(:project) }'", " let(:project) { create(:project_empty_repo) }", " let(:project) { create(:forked_project_with_submodules) }", - " let(:project) { create(:redmine_project) }", - " let(:project) { create(:jira_project) }", - " let(:project) { create(:prometheus_project) }", " let(:project) { create(:project_with_design) }", " let(:authorization) { create(:project_authorization) }" ] @@ -185,9 +182,6 @@ RSpec.describe Tooling::Danger::Specs do "+ let(:project) { create(:project, :repository) }", "+ let(:project) { create(:project_empty_repo) }", "+ let(:project) { create(:forked_project_with_submodules) }", - "+ let(:project) { create(:redmine_project) }", - "+ let(:project) { create(:jira_project) }", - "+ let(:project) { create(:prometheus_project) }", "+ let(:project) { create(:project_with_design) }" ] end @@ -214,10 +208,7 @@ RSpec.describe Tooling::Danger::Specs do { suggested_line: " let_it_be(:project) { create(:project, :repository) }", number: 15 }, { suggested_line: " let_it_be(:project) { create(:project_empty_repo) }", number: 17 }, { suggested_line: " let_it_be(:project) { create(:forked_project_with_submodules) }", number: 18 }, - { suggested_line: " let_it_be(:project) { create(:redmine_project) }", number: 19 }, - { suggested_line: " let_it_be(:project) { create(:jira_project) }", number: 20 }, - { suggested_line: " let_it_be(:project) { create(:prometheus_project) }", number: 21 }, - { suggested_line: " let_it_be(:project) { create(:project_with_design) }", number: 22 } + { suggested_line: " let_it_be(:project) { create(:project_with_design) }", number: 19 } ].each do |test_case| comment = format(template, suggested_line: test_case[:suggested_line]) expect(specs).to receive(:markdown).with(comment, file: filename, line: test_case[:number]) diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb index 78319d6baeb..6832f7d10d1 100644 --- a/tooling/danger/specs.rb +++ b/tooling/danger/specs.rb @@ -12,9 +12,6 @@ module Tooling :project :project_empty_repo :forked_project_with_submodules - :redmine_project - :jira_project - :prometheus_project :project_with_design ].freeze