diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 008b62f6a0f..bf097ae660c 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -600,9 +600,13 @@ - <<: *if-stable-branch-refs when: never - <<: *if-merge-request-labels-as-if-jh + allow_failure: true - <<: *if-merge-request-labels-run-all-rspec + allow_failure: true - changes: *code-backstage-qa-patterns + allow_failure: true - changes: *startup-css-patterns + allow_failure: true .frontend:rules:default-frontend-jobs: rules: @@ -637,11 +641,15 @@ - <<: *if-stable-branch-refs when: never - <<: *if-merge-request-labels-as-if-jh + allow_failure: true - <<: *if-merge-request-labels-run-all-rspec + allow_failure: true - <<: *if-merge-request changes: *startup-css-patterns + allow_failure: true - <<: *if-merge-request changes: *ci-patterns + allow_failure: true .frontend:rules:jest: rules: @@ -1798,9 +1806,13 @@ - <<: *if-stable-branch-refs when: never - <<: *if-merge-request-labels-as-if-jh + allow_failure: true - <<: *if-merge-request-labels-run-all-rspec + allow_failure: true - changes: *code-backstage-qa-patterns + allow_failure: true - changes: *startup-css-patterns + allow_failure: true ####################### # Test metadata rules # diff --git a/.markdownlint.yml b/.markdownlint.yml index 77da3b22fbd..e1e2b246314 100644 --- a/.markdownlint.yml +++ b/.markdownlint.yml @@ -92,7 +92,7 @@ proper-names: "markdownlint", "Mattermost", "Microsoft", - "Minikube", + "minikube", "MinIO", "ModSecurity", "NGINX Ingress", diff --git a/app/assets/javascripts/diffs/components/compare_versions.vue b/app/assets/javascripts/diffs/components/compare_versions.vue index da918947cc5..442807587d5 100644 --- a/app/assets/javascripts/diffs/components/compare_versions.vue +++ b/app/assets/javascripts/diffs/components/compare_versions.vue @@ -182,7 +182,7 @@ export default { class="gl-mr-3" @click="expandAllFiles" > - {{ __('Expand all') }} + {{ __('Expand all files') }} diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 307aaf8f8b8..98254c2259b 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -366,7 +366,7 @@ Microsoft middleware middlewares migratus -Minikube +minikube MinIO misconfiguration misconfigurations diff --git a/doc/administration/compliance.md b/doc/administration/compliance.md index 0bdaeaed971..843d5d0930a 100644 --- a/doc/administration/compliance.md +++ b/doc/administration/compliance.md @@ -6,37 +6,49 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Compliance features **(FREE)** -These GitLab features can help ensure that your GitLab instance meets common compliance standards. For more information -about compliance management, see the compliance management [solutions page](https://about.gitlab.com/solutions/compliance/). +These GitLab features can help ensure that your GitLab instance meets common +compliance standards. For more information about compliance management, see the +compliance management [solutions page](https://about.gitlab.com/solutions/compliance/). -The [security features](../security/index.md) in GitLab may also help you meet relevant compliance standards. +The [security features](../security/index.md) in GitLab may also help you meet +relevant compliance standards. ## Policy management -Organizations have unique policy requirements, either due to organizational standards or mandates from regulatory bodies. -The following features help you define rules and policies to adhere to workflow requirements, separation of duties, and -secure supply chain best practices: +Organizations have unique policy requirements, either due to organizational +standards or mandates from regulatory bodies. The following features help you +define rules and policies to adhere to workflow requirements, separation of duties, +and secure supply chain best practices: -- [**Credentials inventory**](../user/admin_area/credentials_inventory.md) (for instances): With a credentials inventory, - GitLab administrators can keep track of the credentials used by all of the users in their GitLab instance. -- [**Granular user roles and flexible permissions**](../user/permissions.md) (for instances, groups, and projects): Manage - access and permissions with five different user roles and settings for external users. Set permissions according to people's - role, rather than either read or write access to a repository. Don't share the source code with people that only need - access to the issue tracker. -- [**Merge request approvals**](../user/project/merge_requests/approvals/index.md) (for instances, groups, and projects): - Configure approvals required for merge requests. -- [**Push rules**](../push_rules/push_rules.md) (for instances, groups, and projects): Control pushes to your - repositories. +- [**Credentials inventory**](../user/admin_area/credentials_inventory.md) (for + instances): With a credentials inventory, GitLab administrators can keep track + of the credentials used by all of the users in their GitLab instance. +- [**Granular user roles and flexible permissions**](../user/permissions.md) + (for instances, groups, and projects): Manage access and permissions with five + different user roles and settings for external users. Set permissions according + to people's role, rather than either read or write access to a repository. Don't + share the source code with people that only need access to the issue tracker. +- [**Merge request approvals**](../user/project/merge_requests/approvals/index.md) + (for instances, groups, and projects): Configure approvals required for + merge requests. +- [**Push rules**](../push_rules/push_rules.md) (for instances, groups, and + projects): Control pushes to your repositories. - Separation of duties using [**protected branches**](../user/project/protected_branches.md#require-code-owner-approval-on-a-protected-branch) - and [**custom CI/CD configuration paths**](../ci/pipelines/settings.md#specify-a-custom-cicd-configuration-file) (for projects): - Users can leverage the GitLab cross-project YAML configurations to define deployers of code and developers of code. - See how to use this setup to define these roles in: + and [**custom CI/CD configuration paths**](../ci/pipelines/settings.md#specify-a-custom-cicd-configuration-file) (for projects): Users can leverage the GitLab cross-project YAML configurations + to define deployers of code and developers of code. See how to use this setup + to define these roles in: - The [Separation of Duties deploy project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md). - The [Separation of Duties project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md). ## Compliant workflow automation -It is important for compliance teams to be confident that their controls and requirements are set up correctly, but also that they _stay_ set up correctly. One way of doing this is manually checking settings periodically, but this is error prone and time consuming. A better approach is to use single-source-of-truth settings and automation to ensure that whatever a compliance team has configured, stays configured and working correctly. These features can help you automate compliance: +It is important for compliance teams to be confident that their controls and +requirements are set up correctly, but also that they _stay_ set up correctly. +One way of doing this is manually checking settings periodically, but this is +error prone and time consuming. A better approach is to use single-source-of-truth +settings and automation to ensure that whatever a compliance team has configured, +stays configured and working correctly. These features can help you automate +compliance: - [**Compliance frameworks**](../user/project/settings/index.md#compliance-frameworks) (for groups): Create a custom compliance framework at the group level to describe the type of compliance requirements any child project needs to follow. @@ -45,46 +57,59 @@ It is important for compliance teams to be confident that their controls and req ## Audit management -An important part of any compliance program is being able to go back and understand what happened, when -it happened, and who was responsible. This is useful in audit situations as well as for understanding -the root cause of issues when they occur. It is useful to have both low-level, raw lists of audit data -as well as high-level, summary lists of audit data. Between these two, compliance teams can quickly -identify if problems exist and then drill down into the specifics of those issues. These features can help provide visibility into GitLab and audit what is happening: +An important part of any compliance program is being able to go back and understand +what happened, when it happened, and who was responsible. This is useful in audit +situations as well as for understanding the root cause of issues when they occur. +It is useful to have both low-level, raw lists of audit data as well as high-level, +summary lists of audit data. Between these two, compliance teams can quickly +identify if problems exist and then drill down into the specifics of those issues. +These features can help provide visibility into GitLab and audit what is happening: -- [**Audit events**](audit_events.md) (for instances, groups, and projects): To maintain the integrity of your code, - audit events give administrators the ability to view any modifications made within the GitLab - server in an advanced audit events system, so you can control, analyze, and track every change. -- [**Audit reports**](audit_reports.md) (for instances, groups, and projects): Create and access reports based on the - audit events that have occurred. Use pre-built GitLab reports or the API to build your own. -- [**Auditor users**](auditor_users.md) (for instances): Auditor users are users who are given read-only access to all - projects, groups, and other resources on the GitLab instance. -- [**Compliance report**](../user/compliance/compliance_report/index.md) (for groups): Quickly get visibility into the - compliance posture of your organization. +- [**Audit events**](audit_events.md) (for instances, groups, and projects): To + maintain the integrity of your code, audit events give administrators the + ability to view any modifications made within the GitLab server in an advanced + audit events system, so you can control, analyze, and track every change. +- [**Audit reports**](audit_reports.md) (for instances, groups, and projects): + Create and access reports based on the audit events that have occurred. Use + pre-built GitLab reports or the API to build your own. +- [**Auditor users**](auditor_users.md) (for instances): Auditor users are users + who are given read-only access to all projects, groups, and other resources on + the GitLab instance. +- [**Compliance report**](../user/compliance/compliance_report/index.md) (for + groups): Quickly get visibility into the compliance posture of your organization. ## Other compliance features These features can also help with compliance requirements: -- [**Email all users of a project, group, or entire server**](../tools/email.md) (for instances): An administrator can - email groups of users based on project or group membership, or email everyone using the GitLab instance. These emails +- [**Email all users of a project, group, or entire server**](../tools/email.md) + (for instances): An administrator can email groups of users based on project + or group membership, or email everyone using the GitLab instance. These emails are great for scheduled maintenance or upgrades. -- [**Enforce ToS acceptance**](../user/admin_area/settings/terms.md) (for instances): Enforce your users accepting new - terms of service by blocking GitLab traffic. -- [**External Status Checks**](../user/project/merge_requests/status_checks.md) (for projects): Interface with third-party - systems you already use during development to ensure you remain compliant. -- [**Generate reports on permission levels of users**](../user/admin_area/index.md#user-permission-export) (for - instances): Administrators can generate a report listing all users' access permissions for groups and projects in the - instance. -- [**License compliance**](../user/compliance/license_compliance/index.md) (for projects): Search dependencies for their - licenses. This lets you determine if the licenses of your project's dependencies are compatible with your project's license. -- [**Lock project membership to group**](../user/group/index.md#prevent-members-from-being-added-to-projects-in-a-group) (for - groups): Group owners can prevent new members from being added to projects within a group. -- [**LDAP group sync**](auth/ldap/ldap_synchronization.md#group-sync) (for instances): Gives administrators the ability - to automatically sync groups and manage SSH keys, permissions, and authentication, so you can focus on building your - product, not configuring your tools. -- [**LDAP group sync filters**](auth/ldap/ldap_synchronization.md#group-sync) (for instances): Gives more flexibility to - synchronize with LDAP based on filters, meaning you can leverage LDAP attributes to map GitLab permissions. +- [**Enforce ToS acceptance**](../user/admin_area/settings/terms.md) (for + instances): Enforce your users accepting new terms of service by blocking GitLab + traffic. +- [**External Status Checks**](../user/project/merge_requests/status_checks.md) + (for projects): Interface with third-party systems you already use during + development to ensure you remain compliant. +- [**Generate reports on permission levels of users**](../user/admin_area/index.md#user-permission-export) + (for instances): Administrators can generate a report listing all users' access + permissions for groups and projects in the instance. +- [**License compliance**](../user/compliance/license_compliance/index.md) (for + projects): Search dependencies for their licenses. This lets you determine if + the licenses of your project's dependencies are compatible with your project's + license. +- [**Lock project membership to group**](../user/group/index.md#prevent-members-from-being-added-to-projects-in-a-group) + (for groups): Group owners can prevent new members from being added to projects + within a group. +- [**LDAP group sync**](auth/ldap/ldap_synchronization.md#group-sync) (for + instances): Gives administrators the ability to automatically sync groups and + manage SSH keys, permissions, and authentication, so you can focus on building + your product, not configuring your tools. +- [**LDAP group sync filters**](auth/ldap/ldap_synchronization.md#group-sync) + (for instances): Gives more flexibility to synchronize with LDAP based on + filters, meaning you can leverage LDAP attributes to map GitLab permissions. - [**Omnibus GitLab package supports log forwarding**](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-forwarding) (for instances): Forward your logs to a central system. -- [**Restrict SSH Keys**](../security/ssh_keys_restrictions.md) (for instances): Control the technology and key length - of SSH keys used to access GitLab. +- [**Restrict SSH Keys**](../security/ssh_keys_restrictions.md) (for instances): + Control the technology and key length of SSH keys used to access GitLab. diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index dbd8962414b..0fa08dcc9e0 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -188,7 +188,7 @@ successfully, you must replicate their data using some other means. |[Project repository](../../../user/project/repository/) | **Yes** (10.2) | **Yes** (10.7) | No | | |[Project wiki repository](../../../user/project/wiki/) | **Yes** (10.2) | **Yes** (10.7) | No | | |[Group wiki repository](../../../user/project/wiki/group.md) | [**Yes** (13.10)](https://gitlab.com/gitlab-org/gitlab/-/issues/208147) | No | No | Behind feature flag `geo_group_wiki_repository_replication`, enabled by default. | -|[Uploads](../../uploads.md) | **Yes** (10.2) | **Yes** (14.6) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_upload_replication`, enabled by default. Verification is behind the feature flag `geo_upload_verification` introduced and enabled by default in 14.6. | +|[Uploads](../../uploads.md) | **Yes** (10.2) | **Yes** (14.6) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_upload_replication`, enabled by default. Verification was behind the feature flag `geo_upload_verification`, removed in 14.8. | |[LFS objects](../../lfs/index.md) | **Yes** (10.2) | **Yes** (14.6) | Via Object Storage provider if supported. Native Geo support (Beta). | GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696).

Replication is behind the feature flag `geo_lfs_object_replication`, enabled by default. Verification was behind the feature flag `geo_lfs_object_verification`, removed in 14.7. | |[Personal snippets](../../../user/snippets.md) | **Yes** (10.2) | **Yes** (10.2) | No | | |[Project snippets](../../../user/snippets.md) | **Yes** (10.2) | **Yes** (10.2) | No | | diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index b48f015a9ca..4d60832720b 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -533,12 +533,14 @@ You can also monitor the [Praefect logs](../logs.md#praefect-logs). #### Database metrics `/db_metrics` endpoint +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/3286) in GitLab 14.5. + The following metrics are available from the `/db_metrics` endpoint: - `gitaly_praefect_unavailable_repositories`, the number of repositories that have no healthy, up to date replicas. - `gitaly_praefect_read_only_repositories`, the number of repositories in read-only mode within a virtual storage. - This is an older metric that is still available for backwards compatibility reasons. `gitaly_praefect_unavailable_repositories` - is a more accurate. + This metric is available for backwards compatibility reasons. `gitaly_praefect_unavailable_repositories` is more + accurate. - `gitaly_praefect_replication_queue_depth`, the number of jobs in the replication queue. ## Recover from failure diff --git a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md index 8c756260f05..290d6d9f21d 100644 --- a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md +++ b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md @@ -221,9 +221,9 @@ all Kubernetes resources and dependent charts: helm get manifest ``` -## Installation of minimal GitLab configuration via Minikube on macOS +## Installation of minimal GitLab configuration via minikube on macOS -This section is based on [Developing for Kubernetes with Minikube](https://docs.gitlab.com/charts/development/minikube/index.html) +This section is based on [Developing for Kubernetes with minikube](https://docs.gitlab.com/charts/development/minikube/index.html) and [Helm](https://docs.gitlab.com/charts/installation/tools.html#helm). Refer to those documents for details. @@ -233,13 +233,13 @@ to those documents for details. brew install kubernetes-cli ``` -- Install Minikube via Homebrew: +- Install minikube via Homebrew: ```shell brew cask install minikube ``` -- Start Minikube and configure it. If Minikube cannot start, try running `minikube delete && minikube start` +- Start minikube and configure it. If minikube cannot start, try running `minikube delete && minikube start` and repeat the steps: ```shell @@ -253,7 +253,7 @@ to those documents for details. brew install helm ``` -- Copy the [Minikube minimum values YAML file](https://gitlab.com/gitlab-org/charts/gitlab/raw/master/examples/values-minikube-minimum.yaml) +- Copy the [minikube minimum values YAML file](https://gitlab.com/gitlab-org/charts/gitlab/raw/master/examples/values-minikube-minimum.yaml) to your workstation: ```shell diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 078eb5dd0de..36d3655ba93 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -339,7 +339,7 @@ Component statuses are linked to configuration documentation for each component. ### Component list -| Component | Description | [Omnibus GitLab](https://docs.gitlab.com/omnibus/) | [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/quality/gitlab-environment-toolkit) | [GitLab chart](https://docs.gitlab.com/charts/) | [Minikube Minimal](https://docs.gitlab.com/charts/development/minikube/#deploying-gitlab-with-minimal-settings) | [GitLab.com](https://gitlab.com) | [Source](../install/installation.md) | [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) | [CE/EE](https://about.gitlab.com/install/ce-or-ee/) | +| Component | Description | [Omnibus GitLab](https://docs.gitlab.com/omnibus/) | [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/quality/gitlab-environment-toolkit) | [GitLab chart](https://docs.gitlab.com/charts/) | [minikube Minimal](https://docs.gitlab.com/charts/development/minikube/#deploying-gitlab-with-minimal-settings) | [GitLab.com](https://gitlab.com) | [Source](../install/installation.md) | [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) | [CE/EE](https://about.gitlab.com/install/ce-or-ee/) | |-------------------------------------------------------|----------------------------------------------------------------------|:--------------:|:--------------:|:------------:|:----------------:|:----------:|:------:|:---:|:-------:| | [Certificate Management](#certificate-management) | TLS Settings, Let's Encrypt | ✅ | ✅ | ✅ | ⚙ | ✅ | ⚙ | ⚙ | CE & EE | | [Consul](#consul) | Database node discovery, failover | ⚙ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | EE Only | @@ -797,7 +797,7 @@ For monitoring deployed apps, see the [Sentry integration docs](../operations/er - Configuration: - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template) - [Charts](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/) - - [Minikube Minimal](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/index.html) + - [minikube Minimal](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/index.html) - [Source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/gitlab.yml.example) - [GDK](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/gitlab.yml.example) - Layer: Core Service (Processor) diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index a8c5bf6b579..dc3dcba0c95 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -25,6 +25,13 @@ Try to avoid **`@mention`**. Say **mention** instead, and consider linking to th [mentions topic](../../../user/discussions/index.md#mentions). Don't use backticks. +## 2FA, two-factor authentication + +Spell out **two-factor authentication** in sentence case for the first use and in section headings, and **2FA** +thereafter. If the first word in a sentence, do not capitalize `factor` or `authentication`. For example: + +- Two-factor authentication (2FA) helps secure your account. Set up 2FA when you first log in. + ## above Try to avoid using **above** when referring to an example or table in a documentation page. If required, use **previous** instead. For example: @@ -846,6 +853,10 @@ You **turn on** or **turn off** a toggle. For example: - Turn on the **blah** toggle. +## TFA, two-factor authentication + +Use [**2FA** and **two-factor authentication**](#2fa-two-factor-authentication) instead. + ## type Do not use **type** if you can avoid it. Use **enter** instead. diff --git a/doc/security/rate_limits.md b/doc/security/rate_limits.md index 73cc222f843..14fc526ca7e 100644 --- a/doc/security/rate_limits.md +++ b/doc/security/rate_limits.md @@ -87,6 +87,33 @@ There is a rate limit for [testing webhooks](../user/project/integrations/webhoo The **rate limit** is 5 requests per minute per user. +### Users sign up + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77835) in GitLab 14.7. + +There is a rate limit per IP address on the `/users/sign_up` endpoint. This is to mitigate attempts to misuse the endpoint. For example, to mass +discover usernames or email addresses in use. + +The **rate limit** is 20 calls per minute per IP address. + +### Update username + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77221) in GitLab 14.7. + +There is a rate limit on the update username action. This is enforced to mitigate misuse of the feature. For example, to mass discover +which usernames are in use. + +The **rate limit** is 10 calls per minute per signed-in user. + +### Username exists + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77119) in GitLab 14.7. + +There is a rate limit for the internal endpoint `/users/:username/exists`, used by registration to perform a client-side validation for +uniqueness of the chosen username. This is to mitigate the risk of misuses, such as mass discovery of usernames in use. + +The **rate limit** is 20 calls per minute per IP address. + ## Troubleshooting ### Rack Attack is denylisting the load balancer diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 9c1908f3d25..f880e603133 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -443,7 +443,7 @@ You can check the recommended variables for each cluster type in the official do - [Google GKE](https://docs.cilium.io/en/v1.8/gettingstarted/k8s-install-gke/#deploy-cilium) - [AWS EKS](https://docs.cilium.io/en/v1.8/gettingstarted/k8s-install-eks/#deploy-cilium) -Do not use `clusterType` for sandbox environments like [Minikube](https://minikube.sigs.k8s.io/docs/). +Do not use `clusterType` for sandbox environments like [minikube](https://minikube.sigs.k8s.io/docs/). You can customize Cilium's Helm variables by defining the `.gitlab/managed-apps/cilium/values.yaml` file in your cluster diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/cilium.md b/doc/user/infrastructure/clusters/manage/management_project_applications/cilium.md index b5959624954..5d704a2c6df 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/cilium.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/cilium.md @@ -37,7 +37,7 @@ You can check the recommended variables for each cluster type in the official do - [Google GKE](https://docs.cilium.io/en/v1.8/gettingstarted/k8s-install-gke/#deploy-cilium) - [AWS EKS](https://docs.cilium.io/en/v1.8/gettingstarted/k8s-install-eks/#deploy-cilium) -Do not use `clusterType` for sandbox environments like [Minikube](https://minikube.sigs.k8s.io/docs/). +Do not use `clusterType` for sandbox environments like [minikube](https://minikube.sigs.k8s.io/docs/). You can customize Cilium's Helm variables by defining the `applications/cilium/values.yaml` file in your cluster diff --git a/lib/gitlab/background_migration/base_job.rb b/lib/gitlab/background_migration/base_job.rb new file mode 100644 index 00000000000..e21e7e0e4a3 --- /dev/null +++ b/lib/gitlab/background_migration/base_job.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Simple base class for background migration job classes which are executed through the sidekiq queue. + # + # Any job class that inherits from the base class will have connection to the tracking database set on + # initialization. + class BaseJob + def initialize(connection:) + @connection = connection + end + + def perform(*arguments) + raise NotImplementedError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + private + + attr_reader :connection + end + end +end diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index 08802ad7f7f..5dc77f935e3 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -81,7 +81,7 @@ module Gitlab def perform(class_name, arguments) with_shared_connection do - migration_class_for(class_name).new.perform(*arguments) + migration_instance_for(class_name).perform(*arguments) end end @@ -115,6 +115,16 @@ module Gitlab enqueued_job?([retry_set], migration_class) end + def migration_instance_for(class_name) + migration_class = migration_class_for(class_name) + + if migration_class < Gitlab::BackgroundMigration::BaseJob + migration_class.new(connection: connection) + else + migration_class.new + end + end + def migration_class_for(class_name) Gitlab::BackgroundMigration.const_get(class_name, false) end diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb index c046571a111..c0e3016fd3d 100644 --- a/lib/gitlab/database/background_migration_job.rb +++ b/lib/gitlab/database/background_migration_job.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class BackgroundMigrationJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + class BackgroundMigrationJob < SharedModel include EachBatch include BulkInsertSafe diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb index 17a42d997e6..f551fa06cad 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb @@ -4,7 +4,7 @@ module Gitlab module Database module PartitioningMigrationHelpers # Class that will generically copy data from a given table into its corresponding partitioned table - class BackfillPartitionedTable + class BackfillPartitionedTable < ::Gitlab::BackgroundMigration::BaseJob include ::Gitlab::Database::DynamicModelHelpers SUB_BATCH_SIZE = 2_500 @@ -21,7 +21,7 @@ module Gitlab return end - bulk_copy = BulkCopy.new(source_table, partitioned_table, source_column) + bulk_copy = BulkCopy.new(source_table, partitioned_table, source_column, connection: connection) parent_batch_relation = relation_scoped_to_range(source_table, source_column, start_id, stop_id) parent_batch_relation.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| @@ -36,10 +36,6 @@ module Gitlab private - def connection - ActiveRecord::Base.connection - end - def transaction_open? connection.transaction_open? end @@ -53,7 +49,8 @@ module Gitlab end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table).where(source_key_column => start_id..stop_id) + define_batchable_model(source_table) + .where(source_key_column => start_id..stop_id) end def mark_jobs_as_succeeded(*arguments) @@ -64,12 +61,13 @@ module Gitlab class BulkCopy DELIMITER = ', ' - attr_reader :source_table, :destination_table, :source_column + attr_reader :source_table, :destination_table, :source_column, :connection - def initialize(source_table, destination_table, source_column) + def initialize(source_table, destination_table, source_column, connection:) @source_table = source_table @destination_table = destination_table @source_column = source_column + @connection = connection end def copy_between(start_id, stop_id) @@ -85,10 +83,6 @@ module Gitlab private - def connection - @connection ||= ActiveRecord::Base.connection - end - def column_listing @column_listing ||= connection.columns(source_table).map(&:name).join(DELIMITER) end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index c382d2f0715..984c708aa48 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -406,7 +406,8 @@ module Gitlab end def copy_missed_records(source_table_name, partitioned_table_name, source_column) - backfill_table = BackfillPartitionedTable.new + backfill_table = BackfillPartitionedTable.new(connection: connection) + relation = ::Gitlab::Database::BackgroundMigrationJob.pending .for_partitioning_migration(MIGRATION_CLASS_NAME, source_table_name) diff --git a/spec/lib/gitlab/background_migration/base_job_spec.rb b/spec/lib/gitlab/background_migration/base_job_spec.rb new file mode 100644 index 00000000000..86abe4257e4 --- /dev/null +++ b/spec/lib/gitlab/background_migration/base_job_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BaseJob, '#perform' do + let(:connection) { double(:connection) } + + let(:test_job_class) { Class.new(described_class) } + let(:test_job) { test_job_class.new(connection: connection) } + + describe '#perform' do + it 'raises an error if not overridden by a subclass' do + expect { test_job.perform }.to raise_error(NotImplementedError, /must implement perform/) + end + end +end diff --git a/spec/lib/gitlab/background_migration/job_coordinator_spec.rb b/spec/lib/gitlab/background_migration/job_coordinator_spec.rb index 7a524d1489a..43d41408e66 100644 --- a/spec/lib/gitlab/background_migration/job_coordinator_spec.rb +++ b/spec/lib/gitlab/background_migration/job_coordinator_spec.rb @@ -202,23 +202,50 @@ RSpec.describe Gitlab::BackgroundMigration::JobCoordinator do end describe '#perform' do - let(:migration) { spy(:migration) } - let(:connection) { double('connection') } + let(:connection) { double(:connection) } before do - stub_const('Gitlab::BackgroundMigration::Foo', migration) - allow(coordinator).to receive(:connection).and_return(connection) end - it 'performs a background migration with the configured shared connection' do - expect(coordinator).to receive(:with_shared_connection).and_call_original + context 'when the background migration does not inherit from BaseJob' do + let(:migration_class) { Class.new } - expect(migration).to receive(:perform).with(10, 20).once do - expect(Gitlab::Database::SharedModel.connection).to be(connection) + before do + stub_const('Gitlab::BackgroundMigration::Foo', migration_class) end - coordinator.perform('Foo', [10, 20]) + it 'performs a background migration with the configured shared connection' do + expect(coordinator).to receive(:with_shared_connection).and_call_original + + expect_next_instance_of(migration_class) do |migration| + expect(migration).to receive(:perform).with(10, 20).once do + expect(Gitlab::Database::SharedModel.connection).to be(connection) + end + end + + coordinator.perform('Foo', [10, 20]) + end + end + + context 'when the background migration inherits from BaseJob' do + let(:migration_class) { Class.new(::Gitlab::BackgroundMigration::BaseJob) } + let(:migration) { double(:migration) } + + before do + stub_const('Gitlab::BackgroundMigration::Foo', migration_class) + end + + it 'passes the correct connection when constructing the migration' do + expect(coordinator).to receive(:with_shared_connection).and_call_original + + expect(migration_class).to receive(:new).with(connection: connection).and_return(migration) + expect(migration).to receive(:perform).with(10, 20).once do + expect(Gitlab::Database::SharedModel.connection).to be(connection) + end + + coordinator.perform('Foo', [10, 20]) + end end end diff --git a/spec/lib/gitlab/database/background_migration_job_spec.rb b/spec/lib/gitlab/database/background_migration_job_spec.rb index 42695925a1c..1117c17c84a 100644 --- a/spec/lib/gitlab/database/background_migration_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration_job_spec.rb @@ -5,6 +5,8 @@ require 'spec_helper' RSpec.describe Gitlab::Database::BackgroundMigrationJob do it_behaves_like 'having unique enum values' + it { is_expected.to be_a Gitlab::Database::SharedModel } + describe '.for_migration_execution' do let!(:job1) { create(:background_migration_job) } let!(:job2) { create(:background_migration_job, arguments: ['hi', 2]) } diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb index c43b51e10a0..3072c413246 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb @@ -3,14 +3,15 @@ require 'spec_helper' RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable, '#perform' do - subject { described_class.new } + subject(:backfill_job) { described_class.new(connection: connection) } + let(:connection) { ActiveRecord::Base.connection } let(:source_table) { '_test_partitioning_backfills' } let(:destination_table) { "#{source_table}_part" } let(:unique_key) { 'id' } before do - allow(subject).to receive(:transaction_open?).and_return(false) + allow(backfill_job).to receive(:transaction_open?).and_return(false) end context 'when the destination table exists' do @@ -50,10 +51,9 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition stub_const("#{described_class}::SUB_BATCH_SIZE", 2) stub_const("#{described_class}::PAUSE_SECONDS", pause_seconds) - allow(subject).to receive(:sleep) + allow(backfill_job).to receive(:sleep) end - let(:connection) { ActiveRecord::Base.connection } let(:source_model) { Class.new(ActiveRecord::Base) } let(:destination_model) { Class.new(ActiveRecord::Base) } let(:timestamp) { Time.utc(2020, 1, 2).round } @@ -66,7 +66,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition it 'copies data into the destination table idempotently' do expect(destination_model.count).to eq(0) - subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + backfill_job.perform(source1.id, source3.id, source_table, destination_table, unique_key) expect(destination_model.count).to eq(3) @@ -76,7 +76,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition expect(destination_record.attributes).to eq(source_record.attributes) end - subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + backfill_job.perform(source1.id, source3.id, source_table, destination_table, unique_key) expect(destination_model.count).to eq(3) end @@ -87,13 +87,13 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id) end - subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + backfill_job.perform(source1.id, source3.id, source_table, destination_table, unique_key) end it 'pauses after copying each sub-batch' do - expect(subject).to receive(:sleep).with(pause_seconds).twice + expect(backfill_job).to receive(:sleep).with(pause_seconds).twice - subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + backfill_job.perform(source1.id, source3.id, source_table, destination_table, unique_key) end it 'marks each job record as succeeded after processing' do @@ -103,7 +103,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition expect(::Gitlab::Database::BackgroundMigrationJob).to receive(:mark_all_as_succeeded).and_call_original expect do - subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + backfill_job.perform(source1.id, source3.id, source_table, destination_table, unique_key) end.to change { ::Gitlab::Database::BackgroundMigrationJob.succeeded.count }.from(0).to(1) end @@ -111,24 +111,24 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition create(:background_migration_job, class_name: "::#{described_class.name}", arguments: [source1.id, source3.id, source_table, destination_table, unique_key]) - jobs_updated = subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + jobs_updated = backfill_job.perform(source1.id, source3.id, source_table, destination_table, unique_key) expect(jobs_updated).to eq(1) end context 'when the job is run within an explicit transaction block' do + subject(:backfill_job) { described_class.new(connection: mock_connection) } + let(:mock_connection) { double('connection') } - before do - allow(subject).to receive(:connection).and_return(mock_connection) - allow(subject).to receive(:transaction_open?).and_return(true) - end - it 'raises an error before copying data' do + expect(backfill_job).to receive(:transaction_open?).and_call_original + + expect(mock_connection).to receive(:transaction_open?).and_return(true) expect(mock_connection).not_to receive(:execute) expect do - subject.perform(1, 100, source_table, destination_table, unique_key) + backfill_job.perform(1, 100, source_table, destination_table, unique_key) end.to raise_error(/Aborting job to backfill partitioned #{source_table}/) expect(destination_model.count).to eq(0) @@ -137,24 +137,25 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition end context 'when the destination table does not exist' do + subject(:backfill_job) { described_class.new(connection: mock_connection) } + let(:mock_connection) { double('connection') } let(:mock_logger) { double('logger') } before do - allow(subject).to receive(:connection).and_return(mock_connection) - allow(subject).to receive(:logger).and_return(mock_logger) - - expect(mock_connection).to receive(:table_exists?).with(destination_table).and_return(false) + allow(backfill_job).to receive(:logger).and_return(mock_logger) allow(mock_logger).to receive(:warn) end it 'exits without attempting to copy data' do + expect(mock_connection).to receive(:table_exists?).with(destination_table).and_return(false) expect(mock_connection).not_to receive(:execute) subject.perform(1, 100, source_table, destination_table, unique_key) end it 'logs a warning message that the job was skipped' do + expect(mock_connection).to receive(:table_exists?).with(destination_table).and_return(false) expect(mock_logger).to receive(:warn).with(/#{destination_table} does not exist/) subject.perform(1, 100, source_table, destination_table, unique_key)