Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-06-06 00:07:52 +00:00
parent 3796649110
commit 7be95ced1d
28 changed files with 509 additions and 149 deletions

View file

@ -0,0 +1,29 @@
# frozen_string_literal: true
class CleanupOrphanedRoutes < Gitlab::Database::Migration[2.0]
MIGRATION = 'CleanupOrphanedRoutes'
DELAY_INTERVAL = 2.minutes
BATCH_SIZE = 100_000
MAX_BATCH_SIZE = 100_000
SUB_BATCH_SIZE = 100
disable_ddl_transaction!
restrict_gitlab_migration gitlab_schema: :gitlab_main
def up
queue_batched_background_migration(
MIGRATION,
:routes,
:id,
job_interval: DELAY_INTERVAL,
batch_size: BATCH_SIZE,
max_batch_size: MAX_BATCH_SIZE,
sub_batch_size: SUB_BATCH_SIZE,
gitlab_schema: :gitlab_main
)
end
def down
delete_batched_background_migration(MIGRATION, :routes, :id, [])
end
end

View file

@ -0,0 +1 @@
af9fd2e51f9781c16528cdb929328346a87c7199275f101efb1c21b7d56b4255

View file

@ -1131,7 +1131,7 @@ Particular attention should be shown to:
#### Use TCP for existing GitLab instances
When adding Gitaly Cluster to an existing Gitaly instance, the existing Gitaly storage
must use a TCP address. If `gitaly_address` is not specified, then a Unix socket is used,
must be listening on TCP/TLS. If `gitaly_address` is not specified, then a Unix socket is used,
which prevents the communication with the cluster.
For example:
@ -1140,7 +1140,7 @@ For example:
git_data_dirs({
'default' => { 'gitaly_address' => 'tcp://old-gitaly.internal:8075' },
'cluster' => {
'gitaly_address' => 'tcp://<PRAEFECT_LOADBALANCER_HOST>:2305',
'gitaly_address' => 'tls://<PRAEFECT_LOADBALANCER_HOST>:3305',
'gitaly_token' => '<praefect_external_token>'
}
})

View file

@ -391,6 +391,38 @@ Plan.default.actual_limits.update!(ci_active_jobs: 500)
Set the limit to `0` to disable it.
### Number of pipelines running concurrently
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32823) in GitLab 12.5.
The total number of pipelines running concurrently can be limited per project.
When enabled, the limit is checked each time a new pipeline is created.
Without a concurrent pipelines limit, a sudden flood of triggered pipelines could
overwhelm the instance resources.
If a new pipeline would cause the total number of pipelines to exceed the limit,
the pipeline fails with a `The pipeline activity limit was exceeded.` error.
On [GitLab Premium](https://about.gitlab.com/pricing/) self-managed or
higher installations, this limit is defined under a `default` plan that affects all
projects. This limit is disabled (`0`) by default. GitLab SaaS subscribers have different
limits [defined per plan](../user/gitlab_com/index.md#gitlab-cicd), and they affect
all projects under that plan.
To set this limit for a self-managed installation, enable the **Maximum number of active pipelines per project**
[setting in the Admin Area](../user/admin_area/settings/continuous_integration.md#set-cicd-limits).
Alternatively, you can run the following in the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session):
```ruby
# If limits don't exist for the default plan, you can create one with:
# Plan.default.create_limits!
Plan.default.actual_limits.update!(ci_active_pipelines: 100)
```
Set the limit to `0` to disable it.
### Maximum time jobs can run
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16777) in GitLab 12.3.

View file

@ -306,8 +306,10 @@ To create a stub instrumentation for a Service Ping metric, you can use a dedica
The generator takes the class name as an argument and the following options:
- `--type=TYPE` Required. Indicates the metric type. It must be one of: `database`, `generic`, `redis`.
- `--operation` Required for `database` type. It must be one of: `count`, `distinct_count`, `estimate_batch_distinct_count`, `sum`.
- `--type=TYPE` Required. Indicates the metric type. It must be one of: `database`, `generic`, `redis`, `numbers`.
- `--operation` Required for `database` & `numebers` type.
- For `database` it must be one of: `count`, `distinct_count`, `estimate_batch_distinct_count`, `sum`.
- For `numbers` it must be: `add`.
- `--ee` Indicates if the metric is for EE.
```shell

View file

@ -105,8 +105,8 @@ the tiers are no longer mentioned in GitLab documentation:
- [External groups](../integration/saml.md#external-groups)
- [Required groups](../integration/saml.md#required-groups)
- Search:
- [Filtering merge requests by approvers](../user/search/index.md#filtering-merge-requests-by-approvers)
- [Filtering merge requests by "approved by"](../user/search/index.md#filtering-merge-requests-by-approved-by)
- [Filtering merge requests by approvers](../user/project/merge_requests/index.md#filter-merge-requests-by-approvers)
- [Filtering merge requests by "approved by"](../user/project/merge_requests/index.md#filter-merge-requests-by-approved-by)
- [Advanced Search (Elasticsearch)](../user/search/advanced_search.md)
- [Service Desk](../user/project/service_desk.md)
- [Storage usage statistics](../user/usage_quotas.md#storage-usage-statistics)

View file

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

View file

@ -564,6 +564,28 @@ To add an issue to an [iteration](../../group/iterations/index.md):
Alternatively, you can use the `/iteration` [quick action](../quick_actions.md#issues-merge-requests-and-epics).
## View all issues assigned to you
To view all issues assigned to you:
1. On the top bar, put your cursor in the **Search** box.
1. From the dropdown list, select **Issues assigned to me**.
Or:
- To use a [keyboard shortcut](../../shortcuts.md), press <kbd>Shift</kbd> + <kbd>i</kbd>.
- On the top bar, on the top right, select **{issues}** **Issues**.
### Filter issues by ID
> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39908) in GitLab 12.1.
1. On the top bar, select **Menu > Projects** and find your project.
1. On the left sidebar, select **Issues > List**.
1. In the **Search** box, type the issue ID. For example, enter filter `#10` to return only issue 10.
![filter issues by specific ID](img/issue_search_by_id_v15_0.png)
## Copy issue reference
To refer to an issue elsewhere in GitLab, you can use its full URL or a short reference, which looks like

View file

@ -16,8 +16,7 @@ Labels are a key part of [issue boards](issue_board.md). With labels you can:
- Categorize [epics](../group/epics/index.md), issues, and merge requests using colors and descriptive titles like
`bug`, `feature request`, or `docs`.
- Dynamically filter and manage [epics](../group/epics/index.md), issues, and merge requests.
- [Search lists of issues, merge requests, and epics](../search/index.md#search-issues-and-merge-requests),
as well as [issue boards](../search/index.md#issue-boards).
- Search lists of issues, merge requests, and epics, as well as issue boards.
## Types of labels
@ -334,7 +333,7 @@ For example, filtering by the `platform::*` label returns issues that have `plat
`platform::Android`, or `platform::Linux` labels.
NOTE:
Filtering by scoped labels not available on the [issues or merge requests dashboard pages](../search/index.md#search-issues-and-merge-requests).
Filtering by scoped labels not available on the issues or merge requests dashboard pages.
### Scoped labels examples

View file

Before

Width:  |  Height:  |  Size: 7.7 KiB

After

Width:  |  Height:  |  Size: 7.7 KiB

View file

Before

Width:  |  Height:  |  Size: 4.2 KiB

After

Width:  |  Height:  |  Size: 4.2 KiB

View file

@ -48,10 +48,75 @@ To view all merge requests assigned to you:
1. On the top bar, put your cursor in the **Search** box.
1. From the dropdown list, select **Merge requests assigned to me**.
Or, to use a [keyboard shortcut](../../shortcuts.md), press <kbd>Shift</kbd> + <kbd>m</kbd>.
Or:
You can [search and filter](../../search/index.md#filter-issue-and-merge-request-lists),
the results, or select a merge request to begin a review.
- To use a [keyboard shortcut](../../shortcuts.md), press <kbd>Shift</kbd> + <kbd>m</kbd>.
- On the top bar, on the top right, select **{merge-request-open}** **Merge requests**.
Then select one of the following:
- [Review requests](reviews/index.md).
- Merge requests assigned.
### Filter merge requests by ID
> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39908) in GitLab 12.1.
You can filter the **Merge Request** list to find merge requests by their ID.
For example, enter filter `#30` to return only merge request 30.
### Filter merge requests by approvers **(PREMIUM)**
> Moved to GitLab Premium in 13.9.
To filter merge requests by an individual eligible approver ([Code owner](../code_owners.md)), you can type (or select from
the dropdown list) **Approver** and select the user.
![Filter MRs by an approver](img/filter_approver_merge_requests_v14_6.png)
### Filter merge requests by "approved by" **(PREMIUM)**
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30335) in GitLab 13.0.
> - Moved to GitLab Premium in 13.9.
To filter merge requests already approved by a specific individual, you can type (or select from
the dropdown list) **Approved-By** and select the user.
![Filter MRs by approved by](img/filter_approved_by_merge_requests_v14_6.png)
### Filter merge requests by reviewer
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47605) in GitLab 13.7.
To filter review requested merge requests for a specific individual, you can type (or select from
the dropdown list) **Reviewer** and select the user.
### Filter merge requests by environment or deployment date
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44041) in GitLab 13.6.
To filter merge requests by deployment data, such as the environment or a date,
you can type (or select from the dropdown list) the following:
- Environment
- Deployed-before
- Deployed-after
NOTE:
Projects using a [fast-forward merge method](methods/index.md#fast-forward-merge)
do not return results, as this method does not create a merge commit.
When filtering by an environment, a dropdown list presents all environments that
you can choose from:
![Filter MRs by their environment](img/filtering_merge_requests_by_environment_v14_6.png)
When filtering by `Deployed-before` or `Deployed-after`, the date refers to when
the deployment to an environment (triggered by the merge commit) completed successfully.
You must enter the deploy date manually. Deploy dates
use the format `YYYY-MM-DD`, and must be quoted if you wish to specify
both a date and time (`"YYYY-MM-DD HH:MM"`):
![Filter MRs by a deploy date](img/filtering_merge_requests_by_date_v14_6.png)
## Add changes to a merge request
@ -84,8 +149,7 @@ a merge request, or:
1. Select **Edit**.
1. Search for the user you want to assign, and select the user.
The merge request is added to the user's
[assigned merge request list](../../search/index.md#search-issues-and-merge-requests).
The merge request is added to the user's assigned merge request list.
### Assign multiple users **(PREMIUM)**

View file

@ -77,7 +77,7 @@ When a fast-forward merge is not possible, the user is given the option to rebas
NOTE:
Projects using the fast-forward merge strategy can't filter merge requests
[by deployment date](../../../search/index.md#filtering-merge-requests-by-environment-or-deployment-date),
[by deployment date](../index.md#filter-merge-requests-by-environment-or-deployment-date),
because no merge commit is created.
When you visit the merge request page with `Fast-forward merge`

View file

@ -109,7 +109,7 @@ Every issue and merge request can be assigned a milestone. The milestones are vi
### Filtering in list pages
From the project and group issue/merge request list pages, you can [filter](../../search/index.md#search-issues-and-merge-requests) by both group and project milestones.
From the project and group issue/merge request list pages, you can filter by both group and project milestones.
### Filtering in issue boards

View file

@ -336,6 +336,29 @@ To view the activity of a project:
1. On the left sidebar, select **Project information > Activity**.
1. Select a tab to view the type of project activity.
## Search in projects
You can search through your projects.
1. On the top bar, select **Menu**.
1. In **Search your projects**, type the project name.
GitLab filters as you type.
You can also look for the projects you [starred](#star-a-project) (**Starred projects**).
You can **Explore** all public and internal projects available in GitLab.com, from which you can filter by visibility,
through **Trending**, best rated with **Most stars**, or **All** of them.
You can sort projects by:
- Name
- Created date
- Updated date
- Owner
You can also choose to hide or show archived projects.
## Leave a project
If you leave a project, you are no longer a project

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.4 KiB

View file

@ -39,57 +39,43 @@ search, or choose a specific group or project.
![basic_search_results](img/basic_search_results_v15_1.png)
## Code search
## Search in code
To search through code or other documents in a project:
1. On the top bar, select **Menu > Projects** and find your project.
1. On the top bar, in the search field, type the string you want to search for.
1. Press **Enter**.
To search through code or other documents in a single project, you can use
the search field on the top-right of your screen while the project page is open.
Code search shows only the first result in the file.
### Git blame from code search
To search across all of GitLab, ask your administrator to enable [advanced search](advanced_search.md).
### View Git blame from code search
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327052) in GitLab 14.7.
You can access Git blame from any line that returned a result from the code search:
After you find search results, you can view who made the last change to the line
where the results were found.
![code search results](img/code_search_git_blame_v15_1.png)
1. From the code search result, hover over the line number.
1. On the left, select **View blame**.
## SHA search
![code search results](img/code_search_git_blame_v15_1.png)
You can quickly access a commit from the project dashboard by entering the SHA
into the search field on the top right of the screen. If a single result is found, you are
redirected to the commit result and given the option to return to the search results page.
## Search for a SHA
You can search for a commit SHA.
1. On the top bar, select **Menu > Projects** and find your project.
1. On the top bar, in the search field, type the SHA.
If a single result is returned, GitLab redirects to the commit result
and gives you the option to return to the search results page.
![project SHA search redirect](img/project_search_sha_redirect.png)
## Search issues and merge requests
To search through issues and merge requests in multiple projects, on the top bar, select the **Issues** or **Merge requests** links.
The numbers indicate how many issues, merge requests, and to-do items are assigned to you:
![issues and MRs dashboard links](img/dashboard_links_v14_6.png)
- **{issues}** **Issues**: Issues assigned to you.
- **{merge-request-open}** **Merge requests**: Open [merge requests](../project/merge_requests/index.md).
Select the icon to show a dropdown list of merge request filters:
- [Attention requests](../project/merge_requests/index.md#request-attention-to-a-merge-request) (**{attention-solid}**) for you.
- [Review requests](../project/merge_requests/reviews/index.md) for you.
- Merge requests assigned to you.
- **{todo-done}** **To-do items**: The [to-do items](../todos.md) assigned to you.
You can search through **Open**, **Closed**, or **All** issues.
You can also filter the results using the search and filter field, as described in
[Filter issue and merge request lists](#filter-issue-and-merge-request-lists).
### Issues and MRs assigned to you or created by you
GitLab shows shortcuts to issues and merge requests created by you or assigned to you
in the search field in the upper right corner:
![shortcut to your issues and merge requests](img/issues_mrs_shortcut_v14_6.png)
### Filter issue and merge request lists
## Filter issue and merge request lists
> - Filtering by epics was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195704) in GitLab 12.9.
> - Filtering by child epics was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9029) in GitLab 13.0.
@ -130,7 +116,7 @@ groups:
GitLab displays the results on-screen, but you can also
[retrieve them as an RSS feed](#retrieve-search-results-as-feed).
### Searching for specific terms
## Searching for specific terms
You can filter issues and merge requests by specific terms included in titles or descriptions.
@ -143,7 +129,7 @@ You can filter issues and merge requests by specific terms included in titles or
issues for `included in titles` is same as `included titles`
- Search is limited to 4096 characters and 64 terms per query.
### Retrieve search results as feed
## Retrieve search results as feed
> Feeds for merge requests were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66336) in GitLab 14.3.
@ -158,68 +144,6 @@ RSS feed of search results:
The URL of the result contains both a feed token, and your search query.
You can add this URL to your feed reader.
### Filtering by ID
> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39908) in GitLab 12.1.
You can filter the **Issues** list to individual instances by their ID. For example, enter filter `#10` to return only issue 10. The same applies to the **Merge requests** list. Enter filter `#30` to return only merge request 30.
![filter issues by specific ID](img/issue_search_by_id_v15_0.png)
### Filtering merge requests by approvers **(PREMIUM)**
> Moved to GitLab Premium in 13.9.
To filter merge requests by an individual eligible approver ([Code owner](../project/code_owners.md)), you can type (or select from
the dropdown list) **Approver** and select the user.
![Filter MRs by an approver](img/filter_approver_merge_requests_v14_6.png)
### Filtering merge requests by "approved by" **(PREMIUM)**
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30335) in GitLab 13.0.
> - Moved to GitLab Premium in 13.9.
To filter merge requests already approved by a specific individual, you can type (or select from
the dropdown list) **Approved-By** and select the user.
![Filter MRs by approved by](img/filter_approved_by_merge_requests_v14_6.png)
### Filtering merge requests by reviewer
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47605) in GitLab 13.7.
To filter review requested merge requests for a specific individual, you can type (or select from
the dropdown list) **Reviewer** and select the user.
### Filtering merge requests by environment or deployment date
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44041) in GitLab 13.6.
To filter merge requests by deployment data, such as the environment or a date,
you can type (or select from the dropdown list) the following:
- Environment
- Deployed-before
- Deployed-after
NOTE:
Projects using a [fast-forward merge method](../project/merge_requests/fast_forward_merge.md)
do not return results, as this method does not create a merge commit.
When filtering by an environment, a dropdown list presents all environments that
you can choose from:
![Filter MRs by their environment](img/filtering_merge_requests_by_environment_v14_6.png)
When filtering by `Deployed-before` or `Deployed-after`, the date refers to when
the deployment to an environment (triggered by the merge commit) completed successfully.
You must enter the deploy date manually. Deploy dates
use the format `YYYY-MM-DD`, and must be quoted if you wish to specify
both a date and time (`"YYYY-MM-DD HH:MM"`):
![Filter MRs by a deploy date](img/filtering_merge_requests_by_date_v14_6.png)
## Filters autocomplete
GitLab provides many filters across many pages (issues, merge requests, epics,
@ -251,35 +175,9 @@ Some filters can be added multiple times. These include but are not limited to a
![multiple assignees filtering](img/multiple_assignees.png)
## To-Do List
You can search your [To-Do List](../todos.md) by "to do" and "done".
You can filter to-do items per project, author, type, and action.
Also, you can sort them by [**Label priority**](../../user/project/labels.md#set-label-priority),
**Last created**, and **Oldest created**.
## Projects
You can search through your projects from the top bar, by selecting **Menu > Projects**.
On the field **Filter by name**, type the project or group name you want to find, and GitLab
filters them for you as you type.
You can also look for the projects you [starred](../project/working_with_projects.md#star-a-project) (**Starred projects**).
You can **Explore** all public and internal projects available in GitLab.com, from which you can filter by visibility,
through **Trending**, best rated with **Most stars**, or **All** of them.
You can also sort them by:
- Name
- Created date
- Updated date
- Owner
You can also choose to hide or show archived projects.
## Groups
Similarly to [projects search](#projects), you can search through your groups from
You can search through your groups from
the left menu, by clicking the menu bar, then **Groups**.
On the field **Filter by name**, type the group name you want to find, and GitLab

View file

@ -23,6 +23,14 @@ To access your To-Do List:
On the top bar, in the top right, select To-Do List (**{task-done}**).
## Search the To-Do List
You can search your To-Do List by `to do` and `done`.
You can filter to-do items per project, author, type, and action.
Also, you can sort them by [**Label priority**](project/labels.md#set-label-priority),
**Last created**, and **Oldest created**.
## Actions that create to-do items
Many to-do items are created automatically.

View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
module Gitlab
module Usage
module Metrics
module Instrumentations
class <%= class_name %>Metric < NumbersMetric
operation :<%= operation%>
data do |time_frame|
[
# Insert numbers here
]
end
end
end
end
end
end

View file

@ -12,10 +12,13 @@ module Gitlab
ALLOWED_SUPERCLASSES = {
generic: 'Generic',
database: 'Database',
redis: 'Redis'
redis: 'Redis',
numbers: 'Numbers'
}.freeze
ALLOWED_OPERATIONS = %w(count distinct_count estimate_batch_distinct_count sum).freeze
ALLOWED_DATABASE_OPERATIONS = %w(count distinct_count estimate_batch_distinct_count sum).freeze
ALLOWED_NUMBERS_OPERATIONS = %w(add).freeze
ALLOWED_OPERATIONS = ALLOWED_DATABASE_OPERATIONS | ALLOWED_NUMBERS_OPERATIONS
source_root File.expand_path('usage_metric/templates', __dir__)
@ -29,6 +32,7 @@ module Gitlab
validate!
template "database_instrumentation_class.rb.template", file_path if type == 'database'
template "numbers_instrumentation_class.rb.template", file_path if type == 'numbers'
template "generic_instrumentation_class.rb.template", file_path if type == 'generic'
template "instrumentation_class_spec.rb.template", spec_file_path
@ -39,7 +43,8 @@ module Gitlab
def validate!
raise ArgumentError, "Type is required, valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" unless type.present?
raise ArgumentError, "Unknown type '#{type}', valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" if metric_superclass.nil?
raise ArgumentError, "Unknown operation '#{operation}' valid operations are #{ALLOWED_OPERATIONS.join(', ')}" if type == 'database' && !ALLOWED_OPERATIONS.include?(operation)
raise ArgumentError, "Unknown operation '#{operation}' valid operations for database are #{ALLOWED_DATABASE_OPERATIONS.join(', ')}" if type == 'database' && ALLOWED_DATABASE_OPERATIONS.exclude?(operation)
raise ArgumentError, "Unknown operation '#{operation}' valid operations for numbers are #{ALLOWED_NUMBERS_OPERATIONS.join(', ')}" if type == 'numbers' && ALLOWED_NUMBERS_OPERATIONS.exclude?(operation)
end
def ee?

View file

@ -0,0 +1,102 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Removes orphaned routes, i.e. routes that reference a namespace or project that no longer exists.
# This was possible since we were using a polymorphic association source_id, source_type. However since now
# we have project namespaces we can use a FK on routes#namespace_id to avoid orphaned records in routes.
class CleanupOrphanedRoutes < Gitlab::BackgroundMigration::BatchedMigrationJob
include Gitlab::Database::DynamicModelHelpers
def perform
# there should really be no records to fix, there is none gitlab.com, but taking the safer route, just in case.
fix_missing_namespace_id_routes
cleanup_orphaned_routes
end
private
def fix_missing_namespace_id_routes
non_orphaned_namespace_routes = non_orphaned_namespace_routes_scoped_to_range(batch_column, start_id, end_id)
non_orphaned_project_routes = non_orphaned_project_routes_scoped_to_range(batch_column, start_id, end_id)
update_namespace_id(batch_column, non_orphaned_namespace_routes, sub_batch_size)
update_namespace_id(batch_column, non_orphaned_project_routes, sub_batch_size)
end
def cleanup_orphaned_routes
orphaned_namespace_routes = orphaned_namespace_routes_scoped_to_range(batch_column, start_id, end_id)
orphaned_project_routes = orphaned_project_routes_scoped_to_range(batch_column, start_id, end_id)
cleanup_relations(batch_column, orphaned_namespace_routes, pause_ms, sub_batch_size)
cleanup_relations(batch_column, orphaned_project_routes, pause_ms, sub_batch_size)
end
def update_namespace_id(batch_column, non_orphaned_namespace_routes, sub_batch_size)
non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
batch_metrics.time_operation(:fix_missing_namespace_id) do
ApplicationRecord.connection.execute <<~SQL
WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
#{sub_batch.to_sql}
)
UPDATE routes
SET namespace_id = route_and_ns.namespace_id
FROM route_and_ns
WHERE id = route_and_ns.route_id
SQL
end
end
end
def cleanup_relations(batch_column, orphaned_namespace_routes, pause_ms, sub_batch_size)
orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
batch_metrics.time_operation(:cleanup_orphaned_routes) do
sub_batch.delete_all
end
end
end
def orphaned_namespace_routes_scoped_to_range(source_key_column, start_id, stop_id)
Gitlab::BackgroundMigration::Route.joins("LEFT OUTER JOIN namespaces ON source_id = namespaces.id")
.where(source_key_column => start_id..stop_id)
.where(source_type: 'Namespace')
.where(namespace_id: nil)
.where(namespaces: { id: nil })
end
def orphaned_project_routes_scoped_to_range(source_key_column, start_id, stop_id)
Gitlab::BackgroundMigration::Route.joins("LEFT OUTER JOIN projects ON source_id = projects.id")
.where(source_key_column => start_id..stop_id)
.where(source_type: 'Project')
.where(namespace_id: nil)
.where(projects: { id: nil })
end
def non_orphaned_namespace_routes_scoped_to_range(source_key_column, start_id, stop_id)
Gitlab::BackgroundMigration::Route.joins("LEFT OUTER JOIN namespaces ON source_id = namespaces.id")
.where(source_key_column => start_id..stop_id)
.where(source_type: 'Namespace')
.where(namespace_id: nil)
.where.not(namespaces: { id: nil })
.select("routes.id, namespaces.id")
end
def non_orphaned_project_routes_scoped_to_range(source_key_column, start_id, stop_id)
Gitlab::BackgroundMigration::Route.joins("LEFT OUTER JOIN projects ON source_id = projects.id")
.where(source_key_column => start_id..stop_id)
.where(source_type: 'Project')
.where(namespace_id: nil)
.where.not(projects: { id: nil })
.select("routes.id, projects.project_namespace_id")
end
end
# Isolated route model for the migration
class Route < ApplicationRecord
include EachBatch
self.table_name = 'routes'
self.inheritance_column = :_type_disabled
end
end
end

View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
module Gitlab
module Usage
module Metrics
module Instrumentations
class CountFooMetric < NumbersMetric
operation :add
data do |time_frame|
[
# Insert numbers here
]
end
end
end
end
end
end

View file

@ -32,6 +32,7 @@ RSpec.describe Gitlab::UsageMetricGenerator, :silence_stdout do
let(:sample_metric_dir) { 'lib/generators/gitlab/usage_metric_generator' }
let(:generic_sample_metric) { fixture_file(File.join(sample_metric_dir, 'sample_generic_metric.rb')) }
let(:database_sample_metric) { fixture_file(File.join(sample_metric_dir, 'sample_database_metric.rb')) }
let(:numbers_sample_metric) { fixture_file(File.join(sample_metric_dir, 'sample_numbers_metric.rb')) }
let(:sample_spec) { fixture_file(File.join(sample_metric_dir, 'sample_metric_test.rb')) }
it 'creates CE metric instrumentation files using the template' do
@ -63,6 +64,17 @@ RSpec.describe Gitlab::UsageMetricGenerator, :silence_stdout do
end
end
context 'for numbers type' do
let(:options) { { 'type' => 'numbers', 'operation' => 'add' } }
it 'creates the metric instrumentation file using the template' do
described_class.new(args, options).invoke_all
expect_generated_file(ce_temp_dir, 'count_foo_metric.rb', numbers_sample_metric)
expect_generated_file(spec_ce_temp_dir, 'count_foo_metric_spec.rb', sample_spec)
end
end
context 'with type option missing' do
let(:options) { {} }
@ -94,5 +106,21 @@ RSpec.describe Gitlab::UsageMetricGenerator, :silence_stdout do
expect { described_class.new(args, options).invoke_all }.to raise_error(ArgumentError, /Unknown operation 'sleep'/)
end
end
context 'without operation for numbers metric' do
let(:options) { { 'type' => 'numbers' } }
it 'raises an ArgumentError' do
expect { described_class.new(args, options).invoke_all }.to raise_error(ArgumentError, /Unknown operation ''/)
end
end
context 'with wrong operation for numbers metric' do
let(:options) { { 'type' => 'numbers', 'operation' => 'sleep' } }
it 'raises an ArgumentError' do
expect { described_class.new(args, options).invoke_all }.to raise_error(ArgumentError, /Unknown operation 'sleep'/)
end
end
end
end

View file

@ -0,0 +1,79 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::CleanupOrphanedRoutes do
let!(:namespaces) { table(:namespaces) }
let!(:projects) { table(:projects) }
let!(:routes) { table(:routes) }
let!(:namespace1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') }
let!(:namespace2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: namespace1.id, path: 'space2') }
let!(:namespace3) { namespaces.create!(name: 'batchtest3', type: 'Group', parent_id: namespace2.id, path: 'space3') }
let!(:proj_namespace1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace1.id) }
let!(:proj_namespace2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: namespace2.id) }
let!(:proj_namespace3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: namespace3.id) }
# rubocop:disable Layout/LineLength
let!(:proj1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace1.id, project_namespace_id: proj_namespace1.id) }
let!(:proj2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: namespace2.id, project_namespace_id: proj_namespace2.id) }
let!(:proj3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: namespace3.id, project_namespace_id: proj_namespace3.id) }
# valid namespace routes with not null namespace_id
let!(:namespace_route1) { routes.create!(path: 'space1', source_id: namespace1.id, source_type: 'Namespace', namespace_id: namespace1.id) }
# valid namespace routes with null namespace_id
let!(:namespace_route2) { routes.create!(path: 'space1/space2', source_id: namespace2.id, source_type: 'Namespace') }
let!(:namespace_route3) { routes.create!(path: 'space1/space3', source_id: namespace3.id, source_type: 'Namespace') }
# invalid/orphaned namespace route
let!(:orphaned_namespace_route_a) { routes.create!(path: 'space1/space4', source_id: non_existing_record_id, source_type: 'Namespace') }
let!(:orphaned_namespace_route_b) { routes.create!(path: 'space1/space5', source_id: non_existing_record_id - 1, source_type: 'Namespace') }
# valid project routes with not null namespace_id
let!(:proj_route1) { routes.create!(path: 'space1/proj1', source_id: proj1.id, source_type: 'Project', namespace_id: proj_namespace1.id) }
# valid project routes with null namespace_id
let!(:proj_route2) { routes.create!(path: 'space1/space2/proj2', source_id: proj2.id, source_type: 'Project') }
let!(:proj_route3) { routes.create!(path: 'space1/space3/proj3', source_id: proj3.id, source_type: 'Project') }
# invalid/orphaned namespace route
let!(:orphaned_project_route_a) { routes.create!(path: 'space1/space3/proj5', source_id: non_existing_record_id, source_type: 'Project') }
let!(:orphaned_project_route_b) { routes.create!(path: 'space1/space3/proj6', source_id: non_existing_record_id - 1, source_type: 'Project') }
# rubocop:enable Layout/LineLength
let!(:migration_attrs) do
{
start_id: Route.minimum(:id),
end_id: Route.maximum(:id),
batch_table: :routes,
batch_column: :id,
sub_batch_size: 100,
pause_ms: 0,
connection: ApplicationRecord.connection
}
end
let!(:migration) { described_class.new(**migration_attrs) }
subject(:perform_migration) { migration.perform }
it 'cleans orphaned routes', :aggregate_failures do
all_route_ids = Route.pluck(:id)
orphaned_route_ids = [
orphaned_namespace_route_a, orphaned_namespace_route_b, orphaned_project_route_a, orphaned_project_route_b
].pluck(:id)
remaining_routes = (all_route_ids - orphaned_route_ids).sort
expect { perform_migration }.to change { Route.pluck(:id) }.to contain_exactly(*remaining_routes)
expect(Route.all).to all(have_attributes(namespace_id: be_present))
# expect that routes that had namespace_id set did not change namespace_id
expect(namespace_route1.reload.namespace_id).to eq(namespace1.id)
expect(proj_route1.reload.namespace_id).to eq(proj_namespace1.id)
end
it 'tracks timings of queries' do
expect(migration.batch_metrics.timings).to be_empty
expect { perform_migration }.to change { migration.batch_metrics.timings }
end
end

View file

@ -0,0 +1,30 @@
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe CleanupOrphanedRoutes, :migration do
let(:migration) { described_class::MIGRATION }
describe '#up' do
it 'schedules background jobs' do
migrate!
expect(migration).to have_scheduled_batched_migration(
table_name: :routes,
column_name: :id,
interval: described_class::DELAY_INTERVAL,
gitlab_schema: :gitlab_main
)
end
end
describe '#down' do
it 'deletes all batched migration records' do
migrate!
schema_migrate_down!
expect(migration).not_to have_scheduled_batched_migration
end
end
end