Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
parent
bf57aa7662
commit
c68ee79c33
|
@ -11,6 +11,8 @@ class Projects::Analytics::CycleAnalytics::StagesController < Projects::Applicat
|
||||||
before_action :authorize_read_cycle_analytics!
|
before_action :authorize_read_cycle_analytics!
|
||||||
before_action :only_default_value_stream_is_allowed!
|
before_action :only_default_value_stream_is_allowed!
|
||||||
|
|
||||||
|
urgency :low
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
override :parent
|
override :parent
|
||||||
|
|
|
@ -9,6 +9,8 @@ class Projects::Analytics::CycleAnalytics::SummaryController < Projects::Applica
|
||||||
|
|
||||||
before_action :authorize_read_cycle_analytics!
|
before_action :authorize_read_cycle_analytics!
|
||||||
|
|
||||||
|
urgency :low
|
||||||
|
|
||||||
def show
|
def show
|
||||||
render json: project_level.summary
|
render json: project_level.summary
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
module Mutations
|
||||||
|
module Clusters
|
||||||
|
module AgentTokens
|
||||||
|
class Revoke < BaseMutation
|
||||||
|
graphql_name 'ClusterAgentTokenRevoke'
|
||||||
|
|
||||||
|
authorize :admin_cluster
|
||||||
|
|
||||||
|
TokenID = ::Types::GlobalIDType[::Clusters::AgentToken]
|
||||||
|
|
||||||
|
argument :id, TokenID,
|
||||||
|
required: true,
|
||||||
|
description: 'Global ID of the agent token that will be revoked.'
|
||||||
|
|
||||||
|
def resolve(id:)
|
||||||
|
token = authorized_find!(id: id)
|
||||||
|
token.update(status: token.class.statuses[:revoked])
|
||||||
|
|
||||||
|
{ errors: errors_on_object(token) }
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def find_object(id:)
|
||||||
|
# TODO: remove this line when the compatibility layer is removed
|
||||||
|
# See: https://gitlab.com/gitlab-org/gitlab/-/issues/257883
|
||||||
|
id = TokenID.coerce_isolated_input(id)
|
||||||
|
GitlabSchema.find_by_gid(id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -36,6 +36,7 @@ module Types
|
||||||
mount_mutation Mutations::Clusters::Agents::Delete
|
mount_mutation Mutations::Clusters::Agents::Delete
|
||||||
mount_mutation Mutations::Clusters::AgentTokens::Create
|
mount_mutation Mutations::Clusters::AgentTokens::Create
|
||||||
mount_mutation Mutations::Clusters::AgentTokens::Delete
|
mount_mutation Mutations::Clusters::AgentTokens::Delete
|
||||||
|
mount_mutation Mutations::Clusters::AgentTokens::Revoke
|
||||||
mount_mutation Mutations::Commits::Create, calls_gitaly: true
|
mount_mutation Mutations::Commits::Create, calls_gitaly: true
|
||||||
mount_mutation Mutations::CustomEmoji::Create, feature_flag: :custom_emoji
|
mount_mutation Mutations::CustomEmoji::Create, feature_flag: :custom_emoji
|
||||||
mount_mutation Mutations::CustomEmoji::Destroy, feature_flag: :custom_emoji
|
mount_mutation Mutations::CustomEmoji::Destroy, feature_flag: :custom_emoji
|
||||||
|
|
|
@ -15,7 +15,7 @@ module ImportState
|
||||||
def refresh_jid_expiration
|
def refresh_jid_expiration
|
||||||
return unless jid
|
return unless jid
|
||||||
|
|
||||||
Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2)
|
Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION)
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.jid_by(project_id:, status:)
|
def self.jid_by(project_id:, status:)
|
||||||
|
|
|
@ -107,10 +107,7 @@ class Deployment < ApplicationRecord
|
||||||
deployment.run_after_commit do
|
deployment.run_after_commit do
|
||||||
Deployments::UpdateEnvironmentWorker.perform_async(id)
|
Deployments::UpdateEnvironmentWorker.perform_async(id)
|
||||||
Deployments::LinkMergeRequestWorker.perform_async(id)
|
Deployments::LinkMergeRequestWorker.perform_async(id)
|
||||||
|
Deployments::ArchiveInProjectWorker.perform_async(deployment.project_id)
|
||||||
if ::Feature.enabled?(:deployments_archive, deployment.project, default_enabled: :yaml)
|
|
||||||
Deployments::ArchiveInProjectWorker.perform_async(deployment.project_id)
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -7,10 +7,6 @@ module Deployments
|
||||||
BATCH_SIZE = 100
|
BATCH_SIZE = 100
|
||||||
|
|
||||||
def execute
|
def execute
|
||||||
unless ::Feature.enabled?(:deployments_archive, project, default_enabled: :yaml)
|
|
||||||
return error('Feature flag is not enabled')
|
|
||||||
end
|
|
||||||
|
|
||||||
deployments = Deployment.archivables_in(project, limit: BATCH_SIZE)
|
deployments = Deployment.archivables_in(project, limit: BATCH_SIZE)
|
||||||
|
|
||||||
return success(result: :empty) if deployments.empty?
|
return success(result: :empty) if deployments.empty?
|
||||||
|
|
|
@ -8,13 +8,11 @@
|
||||||
%li
|
%li
|
||||||
%a{ href: "#tab-queries", data: { toggle: "tab" } }
|
%a{ href: "#tab-queries", data: { toggle: "tab" } }
|
||||||
= t('sherlock.queries')
|
= t('sherlock.queries')
|
||||||
%span.badge.badge-pill
|
= gl_badge_tag @transaction.queries.length.to_s
|
||||||
#{@transaction.queries.length}
|
|
||||||
%li
|
%li
|
||||||
%a{ href: "#tab-file-samples", data: { toggle: "tab" } }
|
%a{ href: "#tab-file-samples", data: { toggle: "tab" } }
|
||||||
= t('sherlock.file_samples')
|
= t('sherlock.file_samples')
|
||||||
%span.badge.badge-pill
|
= gl_badge_tag @transaction.file_samples.length.to_s
|
||||||
#{@transaction.file_samples.length}
|
|
||||||
|
|
||||||
.row-content-block
|
.row-content-block
|
||||||
.float-right
|
.float-right
|
||||||
|
|
|
@ -1,8 +0,0 @@
|
||||||
---
|
|
||||||
name: deployments_archive
|
|
||||||
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73628
|
|
||||||
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/345027
|
|
||||||
milestone: '14.5'
|
|
||||||
type: development
|
|
||||||
group: group::release
|
|
||||||
default_enabled: true
|
|
|
@ -1,8 +1,8 @@
|
||||||
---
|
---
|
||||||
name: log_implicit_sidekiq_status_calls
|
name: opt_in_sidekiq_status
|
||||||
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74815
|
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77349
|
||||||
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/343964
|
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/343964
|
||||||
milestone: '14.6'
|
milestone: '14.7'
|
||||||
type: development
|
type: development
|
||||||
group: group::scalability
|
group: group::scalability
|
||||||
default_enabled: false
|
default_enabled: false
|
|
@ -6,18 +6,37 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
||||||
|
|
||||||
# How to self-host the docs site **(FREE SELF)**
|
# How to self-host the docs site **(FREE SELF)**
|
||||||
|
|
||||||
The following guide describes how to use a local instance of the docs site with
|
If you have a self-managed instance of GitLab, you may not be able to access the
|
||||||
a self-managed GitLab instance.
|
product documentation as hosted on `docs.gitlab.com` from your GitLab instance.
|
||||||
|
|
||||||
## Run the docs site
|
Be aware of the following items if you self-host the product documentation:
|
||||||
|
|
||||||
The easiest way to run the docs site locally it to pick up one of the existing
|
- You must host the product documentation site under a subdirectory that matches
|
||||||
Docker images that contain the HTML files.
|
your installed GitLab version (for example, `14.5/`). The
|
||||||
|
[Docker images](https://gitlab.com/gitlab-org/gitlab-docs/container_registry/631635)
|
||||||
|
hosted by the GitLab Docs team provide this by default. We use a
|
||||||
|
[script](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/2995d1378175803b22fb8806ba77adf63e79f32c/scripts/normalize-links.sh#L28-82)
|
||||||
|
to normalize the links and prefix them with the respective version.
|
||||||
|
- The version dropdown will display additional versions that don't exist, selecting
|
||||||
|
those versions will display a 404 Not Found page.
|
||||||
|
- Results when using the search box will display results from `docs.gitlab.com`
|
||||||
|
and not the local documentation.
|
||||||
|
- When you use the Docker images to serve the product documentation site, by
|
||||||
|
default the landing page redirects to the respective version (for example, `/14.5/`),
|
||||||
|
which causes the landing page <https://docs.gitlab.com> to not be displayed.
|
||||||
|
|
||||||
Pick the version that matches your GitLab version and run it, in the following
|
## Documentation self-hosting options
|
||||||
examples 14.5.
|
|
||||||
|
|
||||||
### Host the docs site using Docker
|
You can self-host the GitLab product documentation locally using one of these
|
||||||
|
methods:
|
||||||
|
|
||||||
|
- Docker
|
||||||
|
- GitLab Pages
|
||||||
|
- From your own webserver
|
||||||
|
|
||||||
|
The examples on this page are based on GitLab 14.5.
|
||||||
|
|
||||||
|
### Self-host the product documentation with Docker
|
||||||
|
|
||||||
The Docker images use a built-in webserver listening on port `4000`, so you need
|
The Docker images use a built-in webserver listening on port `4000`, so you need
|
||||||
to expose that.
|
to expose that.
|
||||||
|
@ -42,9 +61,9 @@ services:
|
||||||
- '4000:4000'
|
- '4000:4000'
|
||||||
```
|
```
|
||||||
|
|
||||||
### Host the docs site using GitLab Pages
|
### Self-host the product documentation with GitLab Pages
|
||||||
|
|
||||||
You can also host the docs site with GitLab Pages.
|
You use GitLab Pages to host the GitLab product documentation locally.
|
||||||
|
|
||||||
Prerequisite:
|
Prerequisite:
|
||||||
|
|
||||||
|
@ -53,11 +72,11 @@ Prerequisite:
|
||||||
main domain or subdomain. For example, URLs like `https://example.com/docs/`
|
main domain or subdomain. For example, URLs like `https://example.com/docs/`
|
||||||
are not supported.
|
are not supported.
|
||||||
|
|
||||||
To host the docs site with GitLab Pages:
|
To host the product documentation site with GitLab Pages:
|
||||||
|
|
||||||
1. [Create a new blank project](../user/project/working_with_projects.md#create-a-blank-project).
|
1. [Create a new blank project](../user/project/working_with_projects.md#create-a-blank-project).
|
||||||
1. Create a new or edit your existing `.gitlab-ci.yml` file and add the following
|
1. Create a new or edit your existing `.gitlab-ci.yml` file, and add the following
|
||||||
`pages` job. Make sure the version is the same as your GitLab installation:
|
`pages` job, while ensuring the version is the same as your GitLab installation:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
image: registry.gitlab.com/gitlab-org/gitlab-docs:14.5
|
image: registry.gitlab.com/gitlab-org/gitlab-docs:14.5
|
||||||
|
@ -70,20 +89,22 @@ To host the docs site with GitLab Pages:
|
||||||
- public
|
- public
|
||||||
```
|
```
|
||||||
|
|
||||||
1. (Optional) Set the Pages domain name. Depending on the type of the Pages website,
|
1. Optional. Set the GitLab Pages domain name. Depending on the type of the
|
||||||
you have two options:
|
GitLab Pages website, you have two options:
|
||||||
|
|
||||||
| Type of website | [Default domain](../user/project/pages/getting_started_part_one.md#gitlab-pages-default-domain-names) | [Custom domain](../user/project/pages/custom_domains_ssl_tls_certification/index.md) |
|
| Type of website | [Default domain](../user/project/pages/getting_started_part_one.md#gitlab-pages-default-domain-names) | [Custom domain](../user/project/pages/custom_domains_ssl_tls_certification/index.md) |
|
||||||
| --------------- | -------------- | ------------- |
|
|-------------------------|----------------|---------------|
|
||||||
| [Project website](../user/project/pages/getting_started_part_one.md#project-website-examples) | Not supported | Supported |
|
| [Project website](../user/project/pages/getting_started_part_one.md#project-website-examples) | Not supported | Supported |
|
||||||
| [User or group website](../user/project/pages/getting_started_part_one.md#user-and-group-website-examples) | Supported | Supported |
|
| [User or group website](../user/project/pages/getting_started_part_one.md#user-and-group-website-examples) | Supported | Supported |
|
||||||
|
|
||||||
### Host the docs site on your own webserver
|
### Self-host the product documentation on your own webserver
|
||||||
|
|
||||||
Since the docs site is static, you can grab the directory from the container
|
Because the product documentation site is static, you can grab the directory from
|
||||||
(under `/usr/share/nginx/html`) and use your own web server to host
|
the container (in `/usr/share/nginx/html`) and use your own web server to host
|
||||||
it wherever you want. Replace `<destination>` with the directory where the
|
it wherever you want.
|
||||||
docs will be copied to:
|
|
||||||
|
Use the following commands, and replace `<destination>` with the directory where the
|
||||||
|
documentation files will be copied to:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
docker create -it --name gitlab-docs registry.gitlab.com/gitlab-org/gitlab-docs:14.5
|
docker create -it --name gitlab-docs registry.gitlab.com/gitlab-org/gitlab-docs:14.5
|
||||||
|
@ -93,32 +114,18 @@ docker rm -f gitlab-docs
|
||||||
|
|
||||||
## Redirect the `/help` links to the new docs page
|
## Redirect the `/help` links to the new docs page
|
||||||
|
|
||||||
When the docs site is up and running:
|
After your local product documentation site is running, [redirect the help
|
||||||
|
links](../user/admin_area/settings/help_page.md#redirect-help-pages) in the GitLab
|
||||||
|
application to your local site.
|
||||||
|
|
||||||
1. [Enable the help page redirects](../user/admin_area/settings/help_page.md#redirect-help-pages).
|
Be sure to use the fully qualified domain name as the docs URL. For example, if you
|
||||||
Use the Fully Qualified Domain Name as the docs URL. For example, if you
|
used the [Docker method](#self-host-the-product-documentation-with-docker), enter `http://0.0.0.0:4000`.
|
||||||
used the [Docker method](#host-the-docs-site-using-docker) , enter `http://0.0.0.0:4000`.
|
|
||||||
You don't need to append the version, it is detected automatically.
|
|
||||||
1. Test that everything works by selecting the **Learn more** link on the page
|
|
||||||
you're on. Your GitLab version is automatically detected and appended to the docs URL
|
|
||||||
you set in the admin area. In this example, if your GitLab version is 14.5,
|
|
||||||
`https://<instance_url>/` becomes `http://0.0.0.0:4000/14.5/`.
|
|
||||||
The link inside GitLab link shows as
|
|
||||||
`<instance_url>/help/user/admin_area/settings/help_page#destination-requirements`,
|
|
||||||
but when you select it, you are redirected to
|
|
||||||
`http://0.0.0.0:4000/14.5/ee/user/admin_area/settings/help_page/#destination-requirements`.
|
|
||||||
|
|
||||||
## Caveats
|
You don't need to append the version, as GitLab will detect it and append it to
|
||||||
|
any documentation URL requests, as needed. For example, if your GitLab version is
|
||||||
|
14.5, the GitLab Docs URL becomes `http://0.0.0.0:4000/14.5/`. The link
|
||||||
|
inside GitLab displays as `<instance_url>/help/user/admin_area/settings/help_page#destination-requirements`,
|
||||||
|
but when you select it, you are redirected to
|
||||||
|
`http://0.0.0.0:4000/14.5/ee/user/admin_area/settings/help_page/#destination-requirements`.
|
||||||
|
|
||||||
- You need to host the docs site under a subdirectory matching your GitLab version,
|
To test the setting, select a **Learn more** link within the GitLab application.
|
||||||
in the example of this guide `14.5/`. The
|
|
||||||
[Docker images](https://gitlab.com/gitlab-org/gitlab-docs/container_registry/631635)
|
|
||||||
hosted by the Docs team provide this by default. We use a
|
|
||||||
[script](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/2995d1378175803b22fb8806ba77adf63e79f32c/scripts/normalize-links.sh#L28-82)
|
|
||||||
to normalize the links and prefix them with the respective version.
|
|
||||||
- The version dropdown will show more versions which do not exist and will lead
|
|
||||||
to 404 if selected.
|
|
||||||
- The search results point to `docs.gitlab.com` and not the local docs.
|
|
||||||
- When you use the Docker images to serve the docs site, the landing page redirects
|
|
||||||
by default to the respective version, for example `/14.5/`, so you don't
|
|
||||||
see the landing page as seen at <https://docs.gitlab.com>.
|
|
||||||
|
|
|
@ -970,6 +970,24 @@ Input type: `ClusterAgentTokenDeleteInput`
|
||||||
| <a id="mutationclusteragenttokendeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
|
| <a id="mutationclusteragenttokendeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
|
||||||
| <a id="mutationclusteragenttokendeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
|
| <a id="mutationclusteragenttokendeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
|
||||||
|
|
||||||
|
### `Mutation.clusterAgentTokenRevoke`
|
||||||
|
|
||||||
|
Input type: `ClusterAgentTokenRevokeInput`
|
||||||
|
|
||||||
|
#### Arguments
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| ---- | ---- | ----------- |
|
||||||
|
| <a id="mutationclusteragenttokenrevokeclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
|
||||||
|
| <a id="mutationclusteragenttokenrevokeid"></a>`id` | [`ClustersAgentTokenID!`](#clustersagenttokenid) | Global ID of the agent token that will be revoked. |
|
||||||
|
|
||||||
|
#### Fields
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| ---- | ---- | ----------- |
|
||||||
|
| <a id="mutationclusteragenttokenrevokeclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
|
||||||
|
| <a id="mutationclusteragenttokenrevokeerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
|
||||||
|
|
||||||
### `Mutation.commitCreate`
|
### `Mutation.commitCreate`
|
||||||
|
|
||||||
Input type: `CommitCreateInput`
|
Input type: `CommitCreateInput`
|
||||||
|
|
|
@ -0,0 +1,255 @@
|
||||||
|
---
|
||||||
|
stage: none
|
||||||
|
group: unassigned
|
||||||
|
comments: false
|
||||||
|
description: 'CI/CD data time decay'
|
||||||
|
---
|
||||||
|
|
||||||
|
# CI/CD data time decay
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
GitLab CI/CD is one of the most data and compute intensive components of GitLab.
|
||||||
|
Since its [initial release in November 2012](https://about.gitlab.com/blog/2012/11/13/continuous-integration-server-from-gitlab/),
|
||||||
|
the CI/CD subsystem has evolved significantly. It was [integrated into GitLab in September 2015](https://about.gitlab.com/releases/2015/09/22/gitlab-8-0-released/)
|
||||||
|
and has become [one of the most beloved CI/CD solutions](https://about.gitlab.com/blog/2017/09/27/gitlab-leader-continuous-integration-forrester-wave/).
|
||||||
|
|
||||||
|
On February 1st, 2021, GitLab.com surpassed 1 billion CI/CD builds, and the number of
|
||||||
|
builds [continues to grow exponentially](../ci_scale/index.md).
|
||||||
|
|
||||||
|
GitLab CI/CD has come a long way since the initial release, but the design of
|
||||||
|
the data storage for pipeline builds remains almost the same since 2012. In
|
||||||
|
2021 we started working on database decomposition and extracting CI/CD data to
|
||||||
|
ia separate database. Now we want to improve the architecture of GitLab CI/CD
|
||||||
|
product to enable further scaling.
|
||||||
|
|
||||||
|
*Disclaimer: The following contain information related to upcoming products,
|
||||||
|
features, and functionality.
|
||||||
|
|
||||||
|
It is important to note that the information presented is for informational
|
||||||
|
purposes only. Please do not rely on this information for purchasing or
|
||||||
|
planning purposes.
|
||||||
|
|
||||||
|
As with all projects, the items mentioned in this document and linked pages are
|
||||||
|
subject to change or delay. The development, release and timing of any
|
||||||
|
products, features, or functionality remain at the sole discretion of GitLab
|
||||||
|
Inc.*
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
**Implement a new architecture of CI/CD data storage to enable scaling.**
|
||||||
|
|
||||||
|
## Challenges
|
||||||
|
|
||||||
|
There are more than two billion rows describing CI/CD builds in GitLab.com's
|
||||||
|
database. This data represents a sizable portion of the whole data stored in
|
||||||
|
PostgreSQL database running on GitLab.com.
|
||||||
|
|
||||||
|
This volume contributes to significant performance problems, development
|
||||||
|
challenges and is often related to production incidents.
|
||||||
|
|
||||||
|
We also expect a [significant growth in the number of builds executed on
|
||||||
|
GitLab.com](../ci_scale/index.md) in the upcoming years.
|
||||||
|
|
||||||
|
## Opportunity
|
||||||
|
|
||||||
|
CI/CD data is subject to
|
||||||
|
[time-decay](https://about.gitlab.com/company/team/structure/working-groups/database-scalability/time-decay.html)
|
||||||
|
because, usually, pipelines that are a few months old are not frequently
|
||||||
|
accessed or are even not relevant anymore. Restricting access to processing
|
||||||
|
pipelines that are older than a few months might help us to move this data out
|
||||||
|
of the primary database, to a different storage, that is more performant and
|
||||||
|
cost effective.
|
||||||
|
|
||||||
|
It is already possible to prevent processing builds [that have been
|
||||||
|
archived](../../../user/admin_area/settings/continuous_integration.md#archive-jobs).
|
||||||
|
When a build gets archived it will not be possible to retry it, but we still do
|
||||||
|
keep all the processing metadata in the database, and it consumes resources
|
||||||
|
that are scarce in the primary database.
|
||||||
|
|
||||||
|
In order to improve performance and make it easier to scale CI/CD data storage
|
||||||
|
we might want to follow these three tracks described below.
|
||||||
|
|
||||||
|
![pipeline data time decay](pipeline_data_time_decay.png)
|
||||||
|
|
||||||
|
<!-- markdownlint-disable MD029 -->
|
||||||
|
|
||||||
|
1. Partition builds queuing tables
|
||||||
|
2. Archive CI/CD data into partitioned database schema
|
||||||
|
3. Migrate archived builds metadata out of primary database
|
||||||
|
|
||||||
|
<!-- markdownlint-enable MD029 -->
|
||||||
|
|
||||||
|
### Migrate archived builds metadata out of primary database
|
||||||
|
|
||||||
|
Once a build (or a pipeline) gets archived, it is no longer possible to resume
|
||||||
|
pipeline processing in such pipeline. It means that all the metadata, we store
|
||||||
|
in PostgreSQL, that is needed to efficiently and reliably process builds can be
|
||||||
|
safely moved to a different data store.
|
||||||
|
|
||||||
|
Currently, storing pipeline processing data is expensive as this kind of CI/CD
|
||||||
|
data represents a significant portion of data stored in CI/CD tables. Once we
|
||||||
|
restrict access to processing archived pipelines, we can move this metadata to
|
||||||
|
a different place - preferably object storage - and make it accessible on
|
||||||
|
demand, when it is really needed again (for example for compliance or auditing purposes).
|
||||||
|
|
||||||
|
We need to evaluate whether moving data is the most optimal solution. We might
|
||||||
|
be able to use de-duplication of metadata entries and other normalization
|
||||||
|
strategies to consume less storage while retaining ability to query this
|
||||||
|
dataset. Technical evaluation will be required to find the best solution here.
|
||||||
|
|
||||||
|
Epic: [Migrate archived builds metadata out of primary database](https://gitlab.com/groups/gitlab-org/-/epics/7216).
|
||||||
|
|
||||||
|
### Archive CI/CD data into partitioned database schema
|
||||||
|
|
||||||
|
After we move CI/CD metadata to a different store, the problem of having
|
||||||
|
billions of rows describing pipelines, builds and artifacts, remains. We still
|
||||||
|
need to keep reference to the metadata we store in object storage and we still
|
||||||
|
do need to be able to retrieve this information reliably in bulk (or search
|
||||||
|
through it).
|
||||||
|
|
||||||
|
It means that by moving data to object storage we might not be able to reduce
|
||||||
|
the number of rows in CI/CD tables. Moving data to object storage should help
|
||||||
|
with reducing the data size, but not the quantity of entries describing this
|
||||||
|
data. Because of this limitation, we still want to partition CI/CD data to
|
||||||
|
reduce the impact on the database (indices size, auto-vacuum time and
|
||||||
|
frequency).
|
||||||
|
|
||||||
|
Our intent here is not to move this data out of our primary database elsewhere.
|
||||||
|
What want to divide very large database tables, that store CI/CD data, into
|
||||||
|
multiple smaller ones, using PostgreSQL partitioning features.
|
||||||
|
|
||||||
|
There are a few approaches we can take to partition CI/CD data. A promising one
|
||||||
|
is using list-based partitioning where a partition number is assigned a
|
||||||
|
pipeline, and gets propagated to all resources that are related to this
|
||||||
|
pipeline. We assign the partition number based on when the pipeline was created
|
||||||
|
or when we observed the last processing activity in it. This is very flexible
|
||||||
|
because we can extend this partitioning strategy at will; for example with this
|
||||||
|
strategy we can assign an arbitrary partition number based on multiple
|
||||||
|
partitioning keys, combining time-decay-based partitioning with tenant-based
|
||||||
|
partitioning on the application level.
|
||||||
|
|
||||||
|
Partitioning rarely accessed data should also follow the policy defined for
|
||||||
|
builds archival, to make it consistent and reliable.
|
||||||
|
|
||||||
|
Epic: [Archive CI/CD data into partitioned database schema](https://gitlab.com/groups/gitlab-org/-/epics/5417).
|
||||||
|
|
||||||
|
### Partition builds queuing tables
|
||||||
|
|
||||||
|
While working on the [CI/CD Scale](../ci_scale/index.md) blueprint, we have
|
||||||
|
introduced a [new architecture for queuing CI/CD builds](https://gitlab.com/groups/gitlab-org/-/epics/5909#note_680407908)
|
||||||
|
for execution.
|
||||||
|
|
||||||
|
This allowed us to significantly improve performance. We still consider the new
|
||||||
|
solution as an intermediate mechanism, needed before we start working on the
|
||||||
|
next iteration. The following iteration that should improve the architecture of
|
||||||
|
builds queuing even more (it might require moving off the PostgreSQL fully or
|
||||||
|
partially).
|
||||||
|
|
||||||
|
In the meantime we want to ship another iteration, an intermediate step towards
|
||||||
|
more flexible and reliable solution. We want to partition the new queuing
|
||||||
|
tables, to reduce the impact on the database, to improve reliability and
|
||||||
|
database health.
|
||||||
|
|
||||||
|
Partitioning of CI/CD queuing tables does not need to follow the policy defined
|
||||||
|
for builds archival. Instead we should leverage a long-standing policy saying
|
||||||
|
that builds created more 24 hours ago need to be removed from the queue. This
|
||||||
|
business rule is present in the product since the inception of GitLab CI.
|
||||||
|
|
||||||
|
Epic: [Partition builds queuing tables](https://gitlab.com/gitlab-org/gitlab/-/issues/347027).
|
||||||
|
|
||||||
|
## Principles
|
||||||
|
|
||||||
|
All the three tracks we will use to implement CI/CD time decay pattern are
|
||||||
|
associated with some challenges. As we progress with the implementation we will
|
||||||
|
need to solve many problems and devise many implementation details to make this
|
||||||
|
successful.
|
||||||
|
|
||||||
|
Below, we documented a few foundational principles to make it easier for
|
||||||
|
everyone to understand the vision described in this architectural blueprint.
|
||||||
|
|
||||||
|
### Removing pipeline data
|
||||||
|
|
||||||
|
While it might be tempting to simply remove old or archived data from our
|
||||||
|
databases this should be avoided. It is usually not desired to permanently
|
||||||
|
remove user data unless consent is given to do so. We can, however, move data
|
||||||
|
to a different data store, like object storage.
|
||||||
|
|
||||||
|
Archived data can still be needed sometimes (for example for compliance or
|
||||||
|
auditing reasons). We want to be able to retrieve this data if needed, as long
|
||||||
|
as permanent removal has not been requested or approved by a user.
|
||||||
|
|
||||||
|
### Accessing pipeline data in the UI
|
||||||
|
|
||||||
|
Implementing CI/CD data time-decay through partitioning might be challenging
|
||||||
|
when we still want to make it possible for users to access data stored in many
|
||||||
|
partitions.
|
||||||
|
|
||||||
|
We want to retain simplicity of accessing pipeline data in the UI. It will
|
||||||
|
require some backstage changes in how we reference pipeline data from other
|
||||||
|
resources, but we don't want to make it more difficult for users to find their
|
||||||
|
pipelines in the UI.
|
||||||
|
|
||||||
|
We may need to add "Archived" tab on the pipelines / builds list pages, but we
|
||||||
|
should be able to avoid additional steps / clicks when someone wants to view
|
||||||
|
pipeline status or builds associated with a merge request or a deployment.
|
||||||
|
|
||||||
|
We also may need to disable search in the "Archived" tab on pipelines / builds
|
||||||
|
list pages.
|
||||||
|
|
||||||
|
### Accessing pipeline data through the API
|
||||||
|
|
||||||
|
We accept the possible necessity of building a separate API endpoint /
|
||||||
|
endpoints needed to access pipeline data through the API.
|
||||||
|
|
||||||
|
In the new API users might need to provide a time range in which the data has
|
||||||
|
been created to search through their pipelines / builds. In order to make it
|
||||||
|
efficient it might be necessary to restrict access to querying data residing in
|
||||||
|
more than two partitions at once. We can do that by supporting time ranges
|
||||||
|
spanning the duration that equals to the builds archival policy.
|
||||||
|
|
||||||
|
It is possible to still allow users to use the old API to access archived
|
||||||
|
pipelines data, although a user provided partition identifier may be required.
|
||||||
|
|
||||||
|
## Iterations
|
||||||
|
|
||||||
|
All three tracks can be worked on in parallel:
|
||||||
|
|
||||||
|
1. [Migrate archived build metadata to object storage](https://gitlab.com/groups/gitlab-org/-/epics/7216).
|
||||||
|
1. [Partition CI/CD data that have been archived](https://gitlab.com/groups/gitlab-org/-/epics/5417).
|
||||||
|
1. [Partition CI/CD queuing tables using list partitioning](https://gitlab.com/gitlab-org/gitlab/-/issues/347027)
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
In progress.
|
||||||
|
|
||||||
|
## Who
|
||||||
|
|
||||||
|
Proposal:
|
||||||
|
|
||||||
|
<!-- vale gitlab.Spelling = NO -->
|
||||||
|
|
||||||
|
| Role | Who
|
||||||
|
|------------------------------|-------------------------|
|
||||||
|
| Author | Grzegorz Bizon |
|
||||||
|
| Engineering Leader | Cheryl Li |
|
||||||
|
| Product Manager | Jackie Porter |
|
||||||
|
| Architecture Evolution Coach | Kamil Trzciński |
|
||||||
|
|
||||||
|
DRIs:
|
||||||
|
|
||||||
|
| Role | Who
|
||||||
|
|------------------------------|------------------------|
|
||||||
|
| Leadership | Cheryl Li |
|
||||||
|
| Product | Jackie Porter |
|
||||||
|
| Engineering | Grzegorz Bizon |
|
||||||
|
|
||||||
|
Domain experts:
|
||||||
|
|
||||||
|
| Area | Who
|
||||||
|
|------------------------------|------------------------|
|
||||||
|
| Verify / Pipeline execution | Fabio Pitino |
|
||||||
|
| Verify / Pipeline execution | Marius Bobin |
|
||||||
|
| PostgreSQL Database | Andreas Brandl |
|
||||||
|
|
||||||
|
<!-- vale gitlab.Spelling = YES -->
|
Binary file not shown.
After Width: | Height: | Size: 38 KiB |
|
@ -774,10 +774,8 @@ fetch = +refs/environments/*:refs/remotes/origin/environments/*
|
||||||
### Archive Old Deployments
|
### Archive Old Deployments
|
||||||
|
|
||||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73628) in GitLab 14.5.
|
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73628) in GitLab 14.5.
|
||||||
> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.6.
|
> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345027) in GitLab 14.6.
|
||||||
|
> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73628) in GitLab 14.0. [Feature flag `deployments_archive`](https://gitlab.com/gitlab-org/gitlab/-/issues/345027) removed.
|
||||||
FLAG:
|
|
||||||
On self-managed GitLab, by default this feature is available. To hide the feature per project or for your entire instance, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `deployments_archive`. On GitLab.com, this feature is available.
|
|
||||||
|
|
||||||
When a new deployment happens in your project,
|
When a new deployment happens in your project,
|
||||||
GitLab creates [a special Git-ref to the deployment](#check-out-deployments-locally).
|
GitLab creates [a special Git-ref to the deployment](#check-out-deployments-locally).
|
||||||
|
|
|
@ -83,23 +83,11 @@ replacing the class name and arguments with whatever values are necessary for
|
||||||
your migration:
|
your migration:
|
||||||
|
|
||||||
```ruby
|
```ruby
|
||||||
migrate_async('BackgroundMigrationClassName', [arg1, arg2, ...])
|
migrate_in('BackgroundMigrationClassName', [arg1, arg2, ...])
|
||||||
```
|
```
|
||||||
|
|
||||||
Usually it's better to enqueue jobs in bulk, for this you can use
|
You can use the function `queue_background_migration_jobs_by_range_at_intervals`
|
||||||
`bulk_migrate_async`:
|
to automatically split the job into batches:
|
||||||
|
|
||||||
```ruby
|
|
||||||
bulk_migrate_async(
|
|
||||||
[['BackgroundMigrationClassName', [1]],
|
|
||||||
['BackgroundMigrationClassName', [2]]]
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
Note that this will queue a Sidekiq job immediately: if you have a large number
|
|
||||||
of records, this may not be what you want. You can use the function
|
|
||||||
`queue_background_migration_jobs_by_range_at_intervals` to split the job into
|
|
||||||
batches:
|
|
||||||
|
|
||||||
```ruby
|
```ruby
|
||||||
queue_background_migration_jobs_by_range_at_intervals(
|
queue_background_migration_jobs_by_range_at_intervals(
|
||||||
|
@ -117,16 +105,6 @@ consuming migrations it's best to schedule a background job using an
|
||||||
updates. Removals in turn can be handled by simply defining foreign keys with
|
updates. Removals in turn can be handled by simply defining foreign keys with
|
||||||
cascading deletes.
|
cascading deletes.
|
||||||
|
|
||||||
If you would like to schedule jobs in bulk with a delay, you can use
|
|
||||||
`BackgroundMigrationWorker.bulk_perform_in`:
|
|
||||||
|
|
||||||
```ruby
|
|
||||||
jobs = [['BackgroundMigrationClassName', [1]],
|
|
||||||
['BackgroundMigrationClassName', [2]]]
|
|
||||||
|
|
||||||
bulk_migrate_in(5.minutes, jobs)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Rescheduling background migrations
|
### Rescheduling background migrations
|
||||||
|
|
||||||
If one of the background migrations contains a bug that is fixed in a patch
|
If one of the background migrations contains a bug that is fixed in a patch
|
||||||
|
@ -197,53 +175,47 @@ the new format.
|
||||||
|
|
||||||
## Example
|
## Example
|
||||||
|
|
||||||
To explain all this, let's use the following example: the table `services` has a
|
To explain all this, let's use the following example: the table `integrations` has a
|
||||||
field called `properties` which is stored in JSON. For all rows you want to
|
field called `properties` which is stored in JSON. For all rows you want to
|
||||||
extract the `url` key from this JSON object and store it in the `services.url`
|
extract the `url` key from this JSON object and store it in the `integrations.url`
|
||||||
column. There are millions of services and parsing JSON is slow, thus you can't
|
column. There are millions of integrations and parsing JSON is slow, thus you can't
|
||||||
do this in a regular migration.
|
do this in a regular migration.
|
||||||
|
|
||||||
To do this using a background migration we'll start with defining our migration
|
To do this using a background migration we'll start with defining our migration
|
||||||
class:
|
class:
|
||||||
|
|
||||||
```ruby
|
```ruby
|
||||||
class Gitlab::BackgroundMigration::ExtractServicesUrl
|
class Gitlab::BackgroundMigration::ExtractIntegrationsUrl
|
||||||
class Service < ActiveRecord::Base
|
class Integration < ActiveRecord::Base
|
||||||
self.table_name = 'services'
|
self.table_name = 'integrations'
|
||||||
end
|
end
|
||||||
|
|
||||||
def perform(service_id)
|
def perform(start_id, end_id)
|
||||||
# A row may be removed between scheduling and starting of a job, thus we
|
Integration.where(id: start_id..end_id).each do |integration|
|
||||||
# need to make sure the data is still present before doing any work.
|
json = JSON.load(integration.properties)
|
||||||
service = Service.select(:properties).find_by(id: service_id)
|
|
||||||
|
|
||||||
return unless service
|
integration.update(url: json['url']) if json['url']
|
||||||
|
|
||||||
begin
|
|
||||||
json = JSON.load(service.properties)
|
|
||||||
rescue JSON::ParserError
|
rescue JSON::ParserError
|
||||||
# If the JSON is invalid we don't want to keep the job around forever,
|
# If the JSON is invalid we don't want to keep the job around forever,
|
||||||
# instead we'll just leave the "url" field to whatever the default value
|
# instead we'll just leave the "url" field to whatever the default value
|
||||||
# is.
|
# is.
|
||||||
return
|
next
|
||||||
end
|
end
|
||||||
|
|
||||||
service.update(url: json['url']) if json['url']
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
```
|
```
|
||||||
|
|
||||||
Next we'll need to adjust our code so we schedule the above migration for newly
|
Next we'll need to adjust our code so we schedule the above migration for newly
|
||||||
created and updated services. We can do this using something along the lines of
|
created and updated integrations. We can do this using something along the lines of
|
||||||
the following:
|
the following:
|
||||||
|
|
||||||
```ruby
|
```ruby
|
||||||
class Service < ActiveRecord::Base
|
class Integration < ActiveRecord::Base
|
||||||
after_commit :schedule_service_migration, on: :update
|
after_commit :schedule_integration_migration, on: :update
|
||||||
after_commit :schedule_service_migration, on: :create
|
after_commit :schedule_integration_migration, on: :create
|
||||||
|
|
||||||
def schedule_service_migration
|
def schedule_integration_migration
|
||||||
BackgroundMigrationWorker.perform_async('ExtractServicesUrl', [id])
|
BackgroundMigrationWorker.perform_async('ExtractIntegrationsUrl', [id, id])
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
```
|
```
|
||||||
|
@ -253,21 +225,20 @@ before the transaction completes as doing so can lead to race conditions where
|
||||||
the changes are not yet visible to the worker.
|
the changes are not yet visible to the worker.
|
||||||
|
|
||||||
Next we'll need a post-deployment migration that schedules the migration for
|
Next we'll need a post-deployment migration that schedules the migration for
|
||||||
existing data. Since we're dealing with a lot of rows we'll schedule jobs in
|
existing data.
|
||||||
batches instead of doing this one by one:
|
|
||||||
|
|
||||||
```ruby
|
```ruby
|
||||||
class ScheduleExtractServicesUrl < Gitlab::Database::Migration[1.0]
|
class ScheduleExtractIntegrationsUrl < Gitlab::Database::Migration[1.0]
|
||||||
disable_ddl_transaction!
|
disable_ddl_transaction!
|
||||||
|
|
||||||
def up
|
MIGRATION = 'ExtractIntegrationsUrl'
|
||||||
define_batchable_model('services').select(:id).in_batches do |relation|
|
DELAY_INTERVAL = 2.minutes
|
||||||
jobs = relation.pluck(:id).map do |id|
|
|
||||||
['ExtractServicesUrl', [id]]
|
|
||||||
end
|
|
||||||
|
|
||||||
BackgroundMigrationWorker.bulk_perform_async(jobs)
|
def up
|
||||||
end
|
queue_background_migration_jobs_by_range_at_intervals(
|
||||||
|
define_batchable_model('integrations'),
|
||||||
|
MIGRATION,
|
||||||
|
DELAY_INTERVAL)
|
||||||
end
|
end
|
||||||
|
|
||||||
def down
|
def down
|
||||||
|
@ -284,18 +255,18 @@ jobs and manually run on any un-migrated rows. Such a migration would look like
|
||||||
this:
|
this:
|
||||||
|
|
||||||
```ruby
|
```ruby
|
||||||
class ConsumeRemainingExtractServicesUrlJobs < Gitlab::Database::Migration[1.0]
|
class ConsumeRemainingExtractIntegrationsUrlJobs < Gitlab::Database::Migration[1.0]
|
||||||
disable_ddl_transaction!
|
disable_ddl_transaction!
|
||||||
|
|
||||||
def up
|
def up
|
||||||
# This must be included
|
# This must be included
|
||||||
Gitlab::BackgroundMigration.steal('ExtractServicesUrl')
|
Gitlab::BackgroundMigration.steal('ExtractIntegrationsUrl')
|
||||||
|
|
||||||
# This should be included, but can be skipped - see below
|
# This should be included, but can be skipped - see below
|
||||||
define_batchable_model('services').where(url: nil).each_batch(of: 50) do |batch|
|
define_batchable_model('integrations').where(url: nil).each_batch(of: 50) do |batch|
|
||||||
range = batch.pluck('MIN(id)', 'MAX(id)').first
|
range = batch.pluck('MIN(id)', 'MAX(id)').first
|
||||||
|
|
||||||
Gitlab::BackgroundMigration::ExtractServicesUrl.new.perform(*range)
|
Gitlab::BackgroundMigration::ExtractIntegrationsUrl.new.perform(*range)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -313,9 +284,9 @@ If the application does not depend on the data being 100% migrated (for
|
||||||
instance, the data is advisory, and not mission-critical), then this final step
|
instance, the data is advisory, and not mission-critical), then this final step
|
||||||
can be skipped.
|
can be skipped.
|
||||||
|
|
||||||
This migration will then process any jobs for the ExtractServicesUrl migration
|
This migration will then process any jobs for the ExtractIntegrationsUrl migration
|
||||||
and continue once all jobs have been processed. Once done you can safely remove
|
and continue once all jobs have been processed. Once done you can safely remove
|
||||||
the `services.properties` column.
|
the `integrations.properties` column.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
|
|
|
@ -12,9 +12,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
||||||
> - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) from GitLab Premium to GitLab Free in 14.5.
|
> - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) from GitLab Premium to GitLab Free in 14.5.
|
||||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0, the `resource_inclusions` and `resource_exclusions` attributes were removed and `reconcile_timeout`, `dry_run_strategy`, `prune`, `prune_timeout`, `prune_propagation_policy`, and `inventory_policy` attributes were added.
|
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0, the `resource_inclusions` and `resource_exclusions` attributes were removed and `reconcile_timeout`, `dry_run_strategy`, `prune`, `prune_timeout`, `prune_propagation_policy`, and `inventory_policy` attributes were added.
|
||||||
|
|
||||||
WARNING:
|
|
||||||
This feature might not be available to you. Check the **version history** note above for details.
|
|
||||||
|
|
||||||
The [GitLab Agent](index.md) supports hosting your configuration for
|
The [GitLab Agent](index.md) supports hosting your configuration for
|
||||||
multiple agents in a single repository. These agents can be running
|
multiple agents in a single repository. These agents can be running
|
||||||
in the same cluster or in multiple clusters, and potentially with more than one agent per cluster.
|
in the same cluster or in multiple clusters, and potentially with more than one agent per cluster.
|
||||||
|
|
|
@ -43,27 +43,27 @@ module Gitlab
|
||||||
TRANSLATION_LEVELS = {
|
TRANSLATION_LEVELS = {
|
||||||
'bg' => 0,
|
'bg' => 0,
|
||||||
'cs_CZ' => 0,
|
'cs_CZ' => 0,
|
||||||
'da_DK' => 51,
|
'da_DK' => 49,
|
||||||
'de' => 15,
|
'de' => 15,
|
||||||
'en' => 100,
|
'en' => 100,
|
||||||
'eo' => 0,
|
'eo' => 0,
|
||||||
'es' => 39,
|
'es' => 38,
|
||||||
'fil_PH' => 0,
|
'fil_PH' => 0,
|
||||||
'fr' => 12,
|
'fr' => 11,
|
||||||
'gl_ES' => 0,
|
'gl_ES' => 0,
|
||||||
'id_ID' => 0,
|
'id_ID' => 0,
|
||||||
'it' => 2,
|
'it' => 2,
|
||||||
'ja' => 35,
|
'ja' => 36,
|
||||||
'ko' => 11,
|
'ko' => 12,
|
||||||
'nb_NO' => 33,
|
'nb_NO' => 32,
|
||||||
'nl_NL' => 0,
|
'nl_NL' => 0,
|
||||||
'pl_PL' => 5,
|
'pl_PL' => 5,
|
||||||
'pt_BR' => 49,
|
'pt_BR' => 50,
|
||||||
'ro_RO' => 23,
|
'ro_RO' => 22,
|
||||||
'ru' => 25,
|
'ru' => 26,
|
||||||
'tr_TR' => 15,
|
'tr_TR' => 14,
|
||||||
'uk' => 45,
|
'uk' => 45,
|
||||||
'zh_CN' => 95,
|
'zh_CN' => 98,
|
||||||
'zh_HK' => 2,
|
'zh_HK' => 2,
|
||||||
'zh_TW' => 3
|
'zh_TW' => 3
|
||||||
}.freeze
|
}.freeze
|
||||||
|
|
|
@ -13,7 +13,7 @@ module Gitlab
|
||||||
def self.set_jid(import_state)
|
def self.set_jid(import_state)
|
||||||
jid = generate_jid(import_state)
|
jid = generate_jid(import_state)
|
||||||
|
|
||||||
Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2)
|
Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION)
|
||||||
|
|
||||||
import_state.update_column(:jid, jid)
|
import_state.update_column(:jid, jid)
|
||||||
end
|
end
|
||||||
|
|
|
@ -29,16 +29,15 @@ module Gitlab
|
||||||
# for most jobs.
|
# for most jobs.
|
||||||
DEFAULT_EXPIRATION = 30.minutes.to_i
|
DEFAULT_EXPIRATION = 30.minutes.to_i
|
||||||
|
|
||||||
DEFAULT_VALUE = 1
|
|
||||||
DEFAULT_VALUE_MESSAGE = 'Keys using the default value for SidekiqStatus detected'
|
|
||||||
|
|
||||||
# Starts tracking of the given job.
|
# Starts tracking of the given job.
|
||||||
#
|
#
|
||||||
# jid - The Sidekiq job ID
|
# jid - The Sidekiq job ID
|
||||||
# expire - The expiration time of the Redis key.
|
# expire - The expiration time of the Redis key.
|
||||||
def self.set(jid, expire = DEFAULT_EXPIRATION, value: DEFAULT_VALUE)
|
def self.set(jid, expire = DEFAULT_EXPIRATION)
|
||||||
|
return unless expire
|
||||||
|
|
||||||
Sidekiq.redis do |redis|
|
Sidekiq.redis do |redis|
|
||||||
redis.set(key_for(jid), value, ex: expire)
|
redis.set(key_for(jid), 1, ex: expire)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -94,17 +93,10 @@ module Gitlab
|
||||||
return [] if job_ids.empty?
|
return [] if job_ids.empty?
|
||||||
|
|
||||||
keys = job_ids.map { |jid| key_for(jid) }
|
keys = job_ids.map { |jid| key_for(jid) }
|
||||||
results = Sidekiq.redis { |redis| redis.mget(*keys) }
|
|
||||||
|
|
||||||
if Feature.enabled?(:log_implicit_sidekiq_status_calls, default_enabled: :yaml)
|
Sidekiq
|
||||||
to_log = keys.zip(results).select do |_key, result|
|
.redis { |redis| redis.mget(*keys) }
|
||||||
result == DEFAULT_VALUE.to_s
|
.map { |result| !result.nil? }
|
||||||
end.map(&:first)
|
|
||||||
|
|
||||||
Sidekiq.logger.info(message: DEFAULT_VALUE_MESSAGE, keys: to_log) if to_log.any?
|
|
||||||
end
|
|
||||||
|
|
||||||
results.map { |result| !result.nil? }
|
|
||||||
end
|
end
|
||||||
|
|
||||||
# Returns the JIDs that are completed
|
# Returns the JIDs that are completed
|
||||||
|
|
|
@ -4,10 +4,14 @@ module Gitlab
|
||||||
module SidekiqStatus
|
module SidekiqStatus
|
||||||
class ClientMiddleware
|
class ClientMiddleware
|
||||||
def call(_, job, _, _)
|
def call(_, job, _, _)
|
||||||
status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION
|
status_expiration = job['status_expiration']
|
||||||
value = job['status_expiration'] ? 2 : Gitlab::SidekiqStatus::DEFAULT_VALUE
|
|
||||||
|
unless ::Feature.enabled?(:opt_in_sidekiq_status, default_enabled: :yaml)
|
||||||
|
status_expiration ||= Gitlab::SidekiqStatus::DEFAULT_EXPIRATION
|
||||||
|
end
|
||||||
|
|
||||||
|
Gitlab::SidekiqStatus.set(job['jid'], status_expiration)
|
||||||
|
|
||||||
Gitlab::SidekiqStatus.set(job['jid'], status_expiration, value: value)
|
|
||||||
yield
|
yield
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,55 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'spec_helper'
|
||||||
|
|
||||||
|
RSpec.describe Mutations::Clusters::AgentTokens::Revoke do
|
||||||
|
let_it_be(:token) { create(:cluster_agent_token) }
|
||||||
|
let_it_be(:user) { create(:user) }
|
||||||
|
|
||||||
|
let(:mutation) do
|
||||||
|
described_class.new(
|
||||||
|
object: double,
|
||||||
|
context: { current_user: user },
|
||||||
|
field: double
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
it { expect(described_class.graphql_name).to eq('ClusterAgentTokenRevoke') }
|
||||||
|
it { expect(described_class).to require_graphql_authorizations(:admin_cluster) }
|
||||||
|
|
||||||
|
describe '#resolve' do
|
||||||
|
let(:global_id) { token.to_global_id }
|
||||||
|
|
||||||
|
subject { mutation.resolve(id: global_id) }
|
||||||
|
|
||||||
|
context 'user does not have permission' do
|
||||||
|
it 'does not revoke the token' do
|
||||||
|
expect { subject }.to raise_error(Gitlab::Graphql::Errors::ResourceNotAvailable)
|
||||||
|
|
||||||
|
expect(token.reload).not_to be_revoked
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'user has permission' do
|
||||||
|
before do
|
||||||
|
token.agent.project.add_maintainer(user)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'revokes the token' do
|
||||||
|
subject
|
||||||
|
|
||||||
|
expect(token.reload).to be_revoked
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'supplied ID is invalid' do
|
||||||
|
let(:global_id) { token.id }
|
||||||
|
|
||||||
|
it 'raises a coercion error' do
|
||||||
|
expect { subject }.to raise_error(::GraphQL::CoercionError)
|
||||||
|
|
||||||
|
expect(token.reload).not_to be_revoked
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -8,7 +8,7 @@ RSpec.describe Gitlab::Import::SetAsyncJid do
|
||||||
it 'sets the JID in Redis' do
|
it 'sets the JID in Redis' do
|
||||||
expect(Gitlab::SidekiqStatus)
|
expect(Gitlab::SidekiqStatus)
|
||||||
.to receive(:set)
|
.to receive(:set)
|
||||||
.with("async-import/project-import-state/#{project.id}", Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2)
|
.with("async-import/project-import-state/#{project.id}", Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION)
|
||||||
.and_call_original
|
.and_call_original
|
||||||
|
|
||||||
described_class.set_jid(project.import_state)
|
described_class.set_jid(project.import_state)
|
||||||
|
|
|
@ -1,24 +1,61 @@
|
||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
require 'fast_spec_helper'
|
# This can use fast_spec_helper when the feature flag stubbing is removed.
|
||||||
|
require 'spec_helper'
|
||||||
|
|
||||||
RSpec.describe Gitlab::SidekiqStatus::ClientMiddleware do
|
RSpec.describe Gitlab::SidekiqStatus::ClientMiddleware, :clean_gitlab_redis_queues do
|
||||||
describe '#call' do
|
describe '#call' do
|
||||||
context 'when the job has status_expiration set' do
|
context 'when opt_in_sidekiq_status is disabled' do
|
||||||
it 'tracks the job in Redis with a value of 2' do
|
before do
|
||||||
expect(Gitlab::SidekiqStatus).to receive(:set).with('123', 1.hour.to_i, value: 2)
|
stub_feature_flags(opt_in_sidekiq_status: false)
|
||||||
|
end
|
||||||
|
|
||||||
described_class.new
|
context 'when the job has status_expiration set' do
|
||||||
.call('Foo', { 'jid' => '123', 'status_expiration' => 1.hour.to_i }, double(:queue), double(:pool)) { nil }
|
it 'tracks the job in Redis' do
|
||||||
|
expect(Gitlab::SidekiqStatus).to receive(:set).with('123', 1.hour.to_i).and_call_original
|
||||||
|
|
||||||
|
described_class.new
|
||||||
|
.call('Foo', { 'jid' => '123', 'status_expiration' => 1.hour.to_i }, double(:queue), double(:pool)) { nil }
|
||||||
|
|
||||||
|
expect(Gitlab::SidekiqStatus.num_running(['123'])).to eq(1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when the job does not have status_expiration set' do
|
||||||
|
it 'tracks the job in Redis' do
|
||||||
|
expect(Gitlab::SidekiqStatus).to receive(:set).with('123', 30.minutes.to_i).and_call_original
|
||||||
|
|
||||||
|
described_class.new
|
||||||
|
.call('Foo', { 'jid' => '123' }, double(:queue), double(:pool)) { nil }
|
||||||
|
|
||||||
|
expect(Gitlab::SidekiqStatus.num_running(['123'])).to eq(1)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'when the job does not have status_expiration set' do
|
context 'when opt_in_sidekiq_status is enabled' do
|
||||||
it 'tracks the job in Redis with a value of 1' do
|
before do
|
||||||
expect(Gitlab::SidekiqStatus).to receive(:set).with('123', Gitlab::SidekiqStatus::DEFAULT_EXPIRATION, value: 1)
|
stub_feature_flags(opt_in_sidekiq_status: true)
|
||||||
|
end
|
||||||
|
|
||||||
described_class.new
|
context 'when the job has status_expiration set' do
|
||||||
.call('Foo', { 'jid' => '123' }, double(:queue), double(:pool)) { nil }
|
it 'tracks the job in Redis' do
|
||||||
|
expect(Gitlab::SidekiqStatus).to receive(:set).with('123', 1.hour.to_i).and_call_original
|
||||||
|
|
||||||
|
described_class.new
|
||||||
|
.call('Foo', { 'jid' => '123', 'status_expiration' => 1.hour.to_i }, double(:queue), double(:pool)) { nil }
|
||||||
|
|
||||||
|
expect(Gitlab::SidekiqStatus.num_running(['123'])).to eq(1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when the job does not have status_expiration set' do
|
||||||
|
it 'does not track the job in Redis' do
|
||||||
|
described_class.new
|
||||||
|
.call('Foo', { 'jid' => '123' }, double(:queue), double(:pool)) { nil }
|
||||||
|
|
||||||
|
expect(Gitlab::SidekiqStatus.num_running(['123'])).to be_zero
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -12,7 +12,7 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_
|
||||||
Sidekiq.redis do |redis|
|
Sidekiq.redis do |redis|
|
||||||
expect(redis.exists(key)).to eq(true)
|
expect(redis.exists(key)).to eq(true)
|
||||||
expect(redis.ttl(key) > 0).to eq(true)
|
expect(redis.ttl(key) > 0).to eq(true)
|
||||||
expect(redis.get(key)).to eq(described_class::DEFAULT_VALUE.to_s)
|
expect(redis.get(key)).to eq('1')
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -24,19 +24,17 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_
|
||||||
Sidekiq.redis do |redis|
|
Sidekiq.redis do |redis|
|
||||||
expect(redis.exists(key)).to eq(true)
|
expect(redis.exists(key)).to eq(true)
|
||||||
expect(redis.ttl(key) > described_class::DEFAULT_EXPIRATION).to eq(true)
|
expect(redis.ttl(key) > described_class::DEFAULT_EXPIRATION).to eq(true)
|
||||||
expect(redis.get(key)).to eq(described_class::DEFAULT_VALUE.to_s)
|
expect(redis.get(key)).to eq('1')
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'allows overriding the default value' do
|
it 'does not store anything with a nil expiry' do
|
||||||
described_class.set('123', value: 2)
|
described_class.set('123', nil)
|
||||||
|
|
||||||
key = described_class.key_for('123')
|
key = described_class.key_for('123')
|
||||||
|
|
||||||
Sidekiq.redis do |redis|
|
Sidekiq.redis do |redis|
|
||||||
expect(redis.exists(key)).to eq(true)
|
expect(redis.exists(key)).to eq(false)
|
||||||
expect(redis.ttl(key) > 0).to eq(true)
|
|
||||||
expect(redis.get(key)).to eq('2')
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -138,33 +136,5 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_
|
||||||
it 'handles an empty array' do
|
it 'handles an empty array' do
|
||||||
expect(described_class.job_status([])).to eq([])
|
expect(described_class.job_status([])).to eq([])
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'when log_implicit_sidekiq_status_calls is enabled' do
|
|
||||||
it 'logs keys that contained the default value' do
|
|
||||||
described_class.set('123', value: 2)
|
|
||||||
described_class.set('456')
|
|
||||||
described_class.set('012')
|
|
||||||
|
|
||||||
expect(Sidekiq.logger).to receive(:info).with(message: described_class::DEFAULT_VALUE_MESSAGE,
|
|
||||||
keys: [described_class.key_for('456'), described_class.key_for('012')])
|
|
||||||
|
|
||||||
expect(described_class.job_status(%w(123 456 789 012))).to eq([true, true, false, true])
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
context 'when log_implicit_sidekiq_status_calls is disabled' do
|
|
||||||
before do
|
|
||||||
stub_feature_flags(log_implicit_sidekiq_status_calls: false)
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'does not perform any logging' do
|
|
||||||
described_class.set('123', value: 2)
|
|
||||||
described_class.set('456')
|
|
||||||
|
|
||||||
expect(Sidekiq.logger).not_to receive(:info)
|
|
||||||
|
|
||||||
expect(described_class.job_status(%w(123 456 789))).to eq([true, true, false])
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -3278,9 +3278,10 @@ RSpec.describe API::MergeRequests do
|
||||||
|
|
||||||
context 'when skip_ci parameter is set' do
|
context 'when skip_ci parameter is set' do
|
||||||
it 'enqueues a rebase of the merge request with skip_ci flag set' do
|
it 'enqueues a rebase of the merge request with skip_ci flag set' do
|
||||||
allow(RebaseWorker).to receive(:with_status).and_return(RebaseWorker)
|
with_status = RebaseWorker.with_status
|
||||||
|
|
||||||
expect(RebaseWorker).to receive(:perform_async).with(merge_request.id, user.id, true).and_call_original
|
expect(RebaseWorker).to receive(:with_status).and_return(with_status)
|
||||||
|
expect(with_status).to receive(:perform_async).with(merge_request.id, user.id, true).and_call_original
|
||||||
|
|
||||||
Sidekiq::Testing.fake! do
|
Sidekiq::Testing.fake! do
|
||||||
expect do
|
expect do
|
||||||
|
|
|
@ -50,17 +50,6 @@ RSpec.describe Deployments::ArchiveInProjectService do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'when deployments_archive feature flag is disabled' do
|
|
||||||
before do
|
|
||||||
stub_feature_flags(deployments_archive: false)
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'does not do anything' do
|
|
||||||
expect(subject[:status]).to eq(:error)
|
|
||||||
expect(subject[:message]).to eq('Feature flag is not enabled')
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def deployment_refs_exist?
|
def deployment_refs_exist?
|
||||||
deployment_refs.map { |path| project.repository.ref_exists?(path) }
|
deployment_refs.map { |path| project.repository.ref_exists?(path) }
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in New Issue